Blog/wwwroot/jsonql-js/lexer.js 5.2 K · 197 lines · raw · history

1 import { SqlSyntaxError } from './errors.js';
2
3 /** @typedef {{type: string, value: any, pos: number, line: number, col: number}} Token */
4
5 const KEYWORDS = new Set([
6 'AND', 'OR', 'NOT', 'IS', 'NULL', 'LIKE', 'IN', 'BETWEEN', 'TRUE', 'FALSE',
7 'SELECT', 'FROM', 'WHERE', 'AS',
8 'ORDER', 'BY', 'ASC', 'DESC', 'LIMIT',
9 'JOIN', 'INNER', 'LEFT', 'CROSS', 'ON', 'UNNEST',
10 ]);
11
12 const TWO_CHAR_PUNCT = new Set(['!=', '<>', '<=', '>=', '||']);
13 const ONE_CHAR_PUNCT = new Set(['(', ')', '[', ']', '{', '}', ':', ',', '.', '+', '-', '*', '/', '%', '=', '<', '>']);
14
15 /**
16 * Tokenizes an expression string into a flat array of tokens, terminated by an `eof` token.
17 * @param {string} source
18 * @returns {Token[]}
19 */
20 export function tokenize(source) {
21 const tokens = [];
22 const n = source.length;
23 let i = 0;
24 let line = 1;
25 let col = 1;
26
27 function advance(count = 1) {
28 for (let k = 0; k < count; k++) {
29 if (source[i] === '\n') {
30 line++;
31 col = 1;
32 } else {
33 col++;
34 }
35 i++;
36 }
37 }
38
39 /** @param {string} message @param {number} atPos @param {number} atLine @param {number} atCol */
40 function fail(message, atPos, atLine, atCol) {
41 throw new SqlSyntaxError(message, { pos: atPos, line: atLine, col: atCol });
42 }
43
44 while (i < n) {
45 const ch = source[i];
46
47 if (ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n') {
48 advance();
49 continue;
50 }
51
52 if (ch === '-' && source[i + 1] === '-') {
53 while (i < n && source[i] !== '\n') advance();
54 continue;
55 }
56
57 if (ch === '/' && source[i + 1] === '*') {
58 const startPos = i, startLine = line, startCol = col;
59 advance(2);
60 let closed = false;
61 while (i < n) {
62 if (source[i] === '*' && source[i + 1] === '/') {
63 advance(2);
64 closed = true;
65 break;
66 }
67 advance();
68 }
69 if (!closed) fail('Unterminated block comment', startPos, startLine, startCol);
70 continue;
71 }
72
73 const startPos = i, startLine = line, startCol = col;
74
75 if (isDigit(ch)) {
76 let s = '';
77 while (i < n && isDigit(source[i])) {
78 s += source[i];
79 advance();
80 }
81 if (source[i] === '.' && isDigit(source[i + 1])) {
82 s += '.';
83 advance();
84 while (i < n && isDigit(source[i])) {
85 s += source[i];
86 advance();
87 }
88 }
89 if (source[i] === 'e' || source[i] === 'E') {
90 let j = i + 1;
91 if (source[j] === '+' || source[j] === '-') j++;
92 if (isDigit(source[j])) {
93 let k = j;
94 while (isDigit(source[k])) k++;
95 s += source.slice(i, k);
96 while (i < k) advance();
97 }
98 }
99 tokens.push({ type: 'number', value: Number(s), pos: startPos, line: startLine, col: startCol });
100 continue;
101 }
102
103 if (ch === "'") {
104 advance();
105 let s = '';
106 let closed = false;
107 while (i < n) {
108 if (source[i] === "'") {
109 if (source[i + 1] === "'") {
110 s += "'";
111 advance(2);
112 continue;
113 }
114 advance();
115 closed = true;
116 break;
117 }
118 s += source[i];
119 advance();
120 }
121 if (!closed) fail('Unterminated string literal', startPos, startLine, startCol);
122 tokens.push({ type: 'string', value: s, pos: startPos, line: startLine, col: startCol });
123 continue;
124 }
125
126 if (ch === '"') {
127 advance();
128 let s = '';
129 let closed = false;
130 while (i < n) {
131 if (source[i] === '"') {
132 if (source[i + 1] === '"') {
133 s += '"';
134 advance(2);
135 continue;
136 }
137 advance();
138 closed = true;
139 break;
140 }
141 s += source[i];
142 advance();
143 }
144 if (!closed) fail('Unterminated quoted identifier', startPos, startLine, startCol);
145 tokens.push({ type: 'ident', value: s, pos: startPos, line: startLine, col: startCol });
146 continue;
147 }
148
149 if (isIdentStart(ch)) {
150 let s = '';
151 while (i < n && isIdentPart(source[i])) {
152 s += source[i];
153 advance();
154 }
155 const upper = s.toUpperCase();
156 if (KEYWORDS.has(upper)) {
157 tokens.push({ type: upper, value: s, pos: startPos, line: startLine, col: startCol });
158 } else {
159 tokens.push({ type: 'ident', value: s, pos: startPos, line: startLine, col: startCol });
160 }
161 continue;
162 }
163
164 const two = source.slice(i, i + 2);
165 if (TWO_CHAR_PUNCT.has(two)) {
166 tokens.push({ type: two, value: two, pos: startPos, line: startLine, col: startCol });
167 advance(2);
168 continue;
169 }
170
171 if (ONE_CHAR_PUNCT.has(ch)) {
172 tokens.push({ type: ch, value: ch, pos: startPos, line: startLine, col: startCol });
173 advance();
174 continue;
175 }
176
177 fail(`Unexpected character ${JSON.stringify(ch)}`, startPos, startLine, startCol);
178 }
179
180 tokens.push({ type: 'eof', value: null, pos: i, line, col });
181 return tokens;
182 }
183
184 /** @param {string} ch */
185 function isDigit(ch) {
186 return ch >= '0' && ch <= '9';
187 }
188
189 /** @param {string} ch */
190 function isIdentStart(ch) {
191 return ch !== undefined && /[A-Za-z_]/.test(ch);
192 }
193
194 /** @param {string} ch */
195 function isIdentPart(ch) {
196 return ch !== undefined && /[A-Za-z0-9_]/.test(ch);
197 }