1/*
2** Lexical analyzer.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4**
5** Major portions taken verbatim or adapted from the Lua interpreter.
6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7*/
8
9#define lj_lex_c
10#define LUA_CORE
11
12#include "lj_obj.h"
13#include "lj_gc.h"
14#include "lj_err.h"
15#include "lj_buf.h"
16#include "lj_str.h"
17#if LJ_HASFFI
18#include "lj_tab.h"
19#include "lj_ctype.h"
20#include "lj_cdata.h"
21#include "lualib.h"
22#endif
23#include "lj_state.h"
24#include "lj_lex.h"
25#include "lj_parse.h"
26#include "lj_char.h"
27#include "lj_strscan.h"
28#include "lj_strfmt.h"
29
30/* Lua lexer token names. */
31static const char *const tokennames[] = {
32#define TKSTR1(name) #name,
33#define TKSTR2(name, sym) #sym,
34TKDEF(TKSTR1, TKSTR2)
35#undef TKSTR1
36#undef TKSTR2
37 NULL
38};
39
40/* -- Buffer handling ----------------------------------------------------- */
41
42#define LEX_EOF (-1)
43#define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
44
45/* Get more input from reader. */
46static LJ_NOINLINE LexChar lex_more(LexState *ls)
47{
48 size_t sz;
49 const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
50 if (p == NULL || sz == 0) return LEX_EOF;
51 if (sz >= LJ_MAX_BUF) {
52 if (sz != ~(size_t)0) lj_err_mem(ls->L);
53 sz = ~(uintptr_t)0 - (uintptr_t)p;
54 if (sz >= LJ_MAX_BUF) sz = LJ_MAX_BUF-1;
55 ls->endmark = 1;
56 }
57 ls->pe = p + sz;
58 ls->p = p + 1;
59 return (LexChar)(uint8_t)p[0];
60}
61
62/* Get next character. */
63static LJ_AINLINE LexChar lex_next(LexState *ls)
64{
65 return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
66}
67
68/* Save character. */
69static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
70{
71 lj_buf_putb(&ls->sb, c);
72}
73
74/* Save previous character and get next character. */
75static LJ_AINLINE LexChar lex_savenext(LexState *ls)
76{
77 lex_save(ls, ls->c);
78 return lex_next(ls);
79}
80
81/* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
82static void lex_newline(LexState *ls)
83{
84 LexChar old = ls->c;
85 lj_assertLS(lex_iseol(ls), "bad usage");
86 lex_next(ls); /* Skip "\n" or "\r". */
87 if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */
88 if (++ls->linenumber >= LJ_MAX_LINE)
89 lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
90}
91
92/* -- Scanner for terminals ----------------------------------------------- */
93
94/* Parse a number literal. */
95static void lex_number(LexState *ls, TValue *tv)
96{
97 StrScanFmt fmt;
98 LexChar c, xp = 'e';
99 lj_assertLS(lj_char_isdigit(ls->c), "bad usage");
100 if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
101 xp = 'p';
102 while (lj_char_isident(ls->c) || ls->c == '.' ||
103 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
104 c = ls->c;
105 lex_savenext(ls);
106 }
107 lex_save(ls, '\0');
108 fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), sbuflen(&ls->sb)-1, tv,
109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
111 if (LJ_DUALNUM && fmt == STRSCAN_INT) {
112 setitype(tv, LJ_TISNUM);
113 } else if (fmt == STRSCAN_NUM) {
114 /* Already in correct format. */
115#if LJ_HASFFI
116 } else if (fmt != STRSCAN_ERROR) {
117 lua_State *L = ls->L;
118 GCcdata *cd;
119 lj_assertLS(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG,
120 "unexpected number format %d", fmt);
121 if (!ctype_ctsG(G(L))) {
122 ptrdiff_t oldtop = savestack(L, L->top);
123 luaopen_ffi(L); /* Load FFI library on-demand. */
124 L->top = restorestack(L, oldtop);
125 }
126 if (fmt == STRSCAN_IMAG) {
127 cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
128 ((double *)cdataptr(cd))[0] = 0;
129 ((double *)cdataptr(cd))[1] = numV(tv);
130 } else {
131 cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
132 *(uint64_t *)cdataptr(cd) = tv->u64;
133 }
134 lj_parse_keepcdata(ls, tv, cd);
135#endif
136 } else {
137 lj_assertLS(fmt == STRSCAN_ERROR,
138 "unexpected number format %d", fmt);
139 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
140 }
141}
142
143/* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
144static int lex_skipeq(LexState *ls)
145{
146 int count = 0;
147 LexChar s = ls->c;
148 lj_assertLS(s == '[' || s == ']', "bad usage");
149 while (lex_savenext(ls) == '=' && count < 0x20000000)
150 count++;
151 return (ls->c == s) ? count : (-count) - 1;
152}
153
154/* Parse a long string or long comment (tv set to NULL). */
155static void lex_longstring(LexState *ls, TValue *tv, int sep)
156{
157 lex_savenext(ls); /* Skip second '['. */
158 if (lex_iseol(ls)) /* Skip initial newline. */
159 lex_newline(ls);
160 for (;;) {
161 switch (ls->c) {
162 case LEX_EOF:
163 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
164 break;
165 case ']':
166 if (lex_skipeq(ls) == sep) {
167 lex_savenext(ls); /* Skip second ']'. */
168 goto endloop;
169 }
170 break;
171 case '\n':
172 case '\r':
173 lex_save(ls, '\n');
174 lex_newline(ls);
175 if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */
176 break;
177 default:
178 lex_savenext(ls);
179 break;
180 }
181 } endloop:
182 if (tv) {
183 GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
184 sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
185 setstrV(ls->L, tv, str);
186 }
187}
188
189/* Parse a string. */
190static void lex_string(LexState *ls, TValue *tv)
191{
192 LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */
193 lex_savenext(ls);
194 while (ls->c != delim) {
195 switch (ls->c) {
196 case LEX_EOF:
197 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
198 continue;
199 case '\n':
200 case '\r':
201 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
202 continue;
203 case '\\': {
204 LexChar c = lex_next(ls); /* Skip the '\\'. */
205 switch (c) {
206 case 'a': c = '\a'; break;
207 case 'b': c = '\b'; break;
208 case 'f': c = '\f'; break;
209 case 'n': c = '\n'; break;
210 case 'r': c = '\r'; break;
211 case 't': c = '\t'; break;
212 case 'v': c = '\v'; break;
213 case 'x': /* Hexadecimal escape '\xXX'. */
214 c = (lex_next(ls) & 15u) << 4;
215 if (!lj_char_isdigit(ls->c)) {
216 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
217 c += 9 << 4;
218 }
219 c += (lex_next(ls) & 15u);
220 if (!lj_char_isdigit(ls->c)) {
221 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
222 c += 9;
223 }
224 break;
225 case 'u': /* Unicode escape '\u{XX...}'. */
226 if (lex_next(ls) != '{') goto err_xesc;
227 lex_next(ls);
228 c = 0;
229 do {
230 c = (c << 4) | (ls->c & 15u);
231 if (!lj_char_isdigit(ls->c)) {
232 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
233 c += 9;
234 }
235 if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */
236 } while (lex_next(ls) != '}');
237 if (c < 0x800) {
238 if (c < 0x80) break;
239 lex_save(ls, 0xc0 | (c >> 6));
240 } else {
241 if (c >= 0x10000) {
242 lex_save(ls, 0xf0 | (c >> 18));
243 lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
244 } else {
245 if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */
246 lex_save(ls, 0xe0 | (c >> 12));
247 }
248 lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
249 }
250 c = 0x80 | (c & 0x3f);
251 break;
252 case 'z': /* Skip whitespace. */
253 lex_next(ls);
254 while (lj_char_isspace(ls->c))
255 if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
256 continue;
257 case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
258 case '\\': case '\"': case '\'': break;
259 case LEX_EOF: continue;
260 default:
261 if (!lj_char_isdigit(c))
262 goto err_xesc;
263 c -= '0'; /* Decimal escape '\ddd'. */
264 if (lj_char_isdigit(lex_next(ls))) {
265 c = c*10 + (ls->c - '0');
266 if (lj_char_isdigit(lex_next(ls))) {
267 c = c*10 + (ls->c - '0');
268 if (c > 255) {
269 err_xesc:
270 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
271 }
272 lex_next(ls);
273 }
274 }
275 lex_save(ls, c);
276 continue;
277 }
278 lex_save(ls, c);
279 lex_next(ls);
280 continue;
281 }
282 default:
283 lex_savenext(ls);
284 break;
285 }
286 }
287 lex_savenext(ls); /* Skip trailing delimiter. */
288 setstrV(ls->L, tv,
289 lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
290}
291
292/* -- Main lexical scanner ------------------------------------------------ */
293
294/* Get next lexical token. */
295static LexToken lex_scan(LexState *ls, TValue *tv)
296{
297 lj_buf_reset(&ls->sb);
298 for (;;) {
299 if (lj_char_isident(ls->c)) {
300 GCstr *s;
301 if (lj_char_isdigit(ls->c)) { /* Numeric literal. */
302 lex_number(ls, tv);
303 return TK_number;
304 }
305 /* Identifier or reserved word. */
306 do {
307 lex_savenext(ls);
308 } while (lj_char_isident(ls->c));
309 s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
310 setstrV(ls->L, tv, s);
311 if (s->reserved > 0) /* Reserved word? */
312 return TK_OFS + s->reserved;
313 return TK_name;
314 }
315 switch (ls->c) {
316 case '\n':
317 case '\r':
318 lex_newline(ls);
319 continue;
320 case ' ':
321 case '\t':
322 case '\v':
323 case '\f':
324 lex_next(ls);
325 continue;
326 case '-':
327 lex_next(ls);
328 if (ls->c != '-') return '-';
329 lex_next(ls);
330 if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */
331 int sep = lex_skipeq(ls);
332 lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */
333 if (sep >= 0) {
334 lex_longstring(ls, NULL, sep);
335 lj_buf_reset(&ls->sb);
336 continue;
337 }
338 }
339 /* Short comment "--.*\n". */
340 while (!lex_iseol(ls) && ls->c != LEX_EOF)
341 lex_next(ls);
342 continue;
343 case '[': {
344 int sep = lex_skipeq(ls);
345 if (sep >= 0) {
346 lex_longstring(ls, tv, sep);
347 return TK_string;
348 } else if (sep == -1) {
349 return '[';
350 } else {
351 lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
352 continue;
353 }
354 }
355 case '=':
356 lex_next(ls);
357 if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
358 case '<':
359 lex_next(ls);
360 if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
361 case '>':
362 lex_next(ls);
363 if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
364 case '~':
365 lex_next(ls);
366 if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
367 case ':':
368 lex_next(ls);
369 if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
370 case '"':
371 case '\'':
372 lex_string(ls, tv);
373 return TK_string;
374 case '.':
375 if (lex_savenext(ls) == '.') {
376 lex_next(ls);
377 if (ls->c == '.') {
378 lex_next(ls);
379 return TK_dots; /* ... */
380 }
381 return TK_concat; /* .. */
382 } else if (!lj_char_isdigit(ls->c)) {
383 return '.';
384 } else {
385 lex_number(ls, tv);
386 return TK_number;
387 }
388 case LEX_EOF:
389 return TK_eof;
390 default: {
391 LexChar c = ls->c;
392 lex_next(ls);
393 return c; /* Single-char tokens (+ - / ...). */
394 }
395 }
396 }
397}
398
399/* -- Lexer API ----------------------------------------------------------- */
400
401/* Setup lexer state. */
402int lj_lex_setup(lua_State *L, LexState *ls)
403{
404 int header = 0;
405 ls->L = L;
406 ls->fs = NULL;
407 ls->pe = ls->p = NULL;
408 ls->vstack = NULL;
409 ls->sizevstack = 0;
410 ls->vtop = 0;
411 ls->bcstack = NULL;
412 ls->sizebcstack = 0;
413 ls->tok = 0;
414 ls->lookahead = TK_eof; /* No look-ahead token. */
415 ls->linenumber = 1;
416 ls->lastline = 1;
417 ls->endmark = 0;
418 lex_next(ls); /* Read-ahead first char. */
419 if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
420 (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
421 ls->p += 2;
422 lex_next(ls);
423 header = 1;
424 }
425 if (ls->c == '#') { /* Skip POSIX #! header line. */
426 do {
427 lex_next(ls);
428 if (ls->c == LEX_EOF) return 0;
429 } while (!lex_iseol(ls));
430 lex_newline(ls);
431 header = 1;
432 }
433 if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */
434 if (header) {
435 /*
436 ** Loading bytecode with an extra header is disabled for security
437 ** reasons. This may circumvent the usual check for bytecode vs.
438 ** Lua code by looking at the first char. Since this is a potential
439 ** security violation no attempt is made to echo the chunkname either.
440 */
441 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
442 lj_err_throw(L, LUA_ERRSYNTAX);
443 }
444 return 1;
445 }
446 return 0;
447}
448
449/* Cleanup lexer state. */
450void lj_lex_cleanup(lua_State *L, LexState *ls)
451{
452 global_State *g = G(L);
453 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
454 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
455 lj_buf_free(g, &ls->sb);
456}
457
458/* Return next lexical token. */
459void lj_lex_next(LexState *ls)
460{
461 ls->lastline = ls->linenumber;
462 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
463 ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */
464 } else { /* Otherwise return lookahead token. */
465 ls->tok = ls->lookahead;
466 ls->lookahead = TK_eof;
467 ls->tokval = ls->lookaheadval;
468 }
469}
470
471/* Look ahead for the next token. */
472LexToken lj_lex_lookahead(LexState *ls)
473{
474 lj_assertLS(ls->lookahead == TK_eof, "double lookahead");
475 ls->lookahead = lex_scan(ls, &ls->lookaheadval);
476 return ls->lookahead;
477}
478
479/* Convert token to string. */
480const char *lj_lex_token2str(LexState *ls, LexToken tok)
481{
482 if (tok > TK_OFS)
483 return tokennames[tok-TK_OFS-1];
484 else if (!lj_char_iscntrl(tok))
485 return lj_strfmt_pushf(ls->L, "%c", tok);
486 else
487 return lj_strfmt_pushf(ls->L, "char(%d)", tok);
488}
489
490/* Lexer error. */
491void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
492{
493 const char *tokstr;
494 va_list argp;
495 if (tok == 0) {
496 tokstr = NULL;
497 } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
498 lex_save(ls, '\0');
499 tokstr = sbufB(&ls->sb);
500 } else {
501 tokstr = lj_lex_token2str(ls, tok);
502 }
503 va_start(argp, em);
504 lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
505 va_end(argp);
506}
507
508/* Initialize strings for reserved words. */
509void lj_lex_init(lua_State *L)
510{
511 uint32_t i;
512 for (i = 0; i < TK_RESERVED; i++) {
513 GCstr *s = lj_str_newz(L, tokennames[i]);
514 fixstring(s); /* Reserved words are never collected. */
515 s->reserved = (uint8_t)(i+1);
516 }
517}
518
519