1 /* |
|
2 ** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $ |
|
3 ** Lexical Analyzer |
|
4 ** See Copyright Notice in lua.h |
|
5 */ |
|
6 |
|
7 |
|
8 #include <ctype.h> |
|
9 #include <locale.h> |
|
10 #include <string.h> |
|
11 |
|
12 #define llex_c |
|
13 #define LUA_CORE |
|
14 |
|
15 #include "lua.h" |
|
16 |
|
17 #include "ldo.h" |
|
18 #include "llex.h" |
|
19 #include "lobject.h" |
|
20 #include "lparser.h" |
|
21 #include "lstate.h" |
|
22 #include "lstring.h" |
|
23 #include "ltable.h" |
|
24 #include "lzio.h" |
|
25 |
|
26 |
|
27 |
|
28 #define next(ls) (ls->current = zgetc(ls->z)) |
|
29 |
|
30 |
|
31 |
|
32 |
|
33 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') |
|
34 |
|
35 |
|
36 /* ORDER RESERVED */ |
|
37 const char *const luaX_tokens [] = { |
|
38 "and", "break", "do", "else", "elseif", |
|
39 "end", "false", "for", "function", "if", |
|
40 "in", "local", "nil", "not", "or", "repeat", |
|
41 "return", "then", "true", "until", "while", |
|
42 "..", "...", "==", ">=", "<=", "~=", |
|
43 "<number>", "<name>", "<string>", "<eof>", |
|
44 NULL |
|
45 }; |
|
46 |
|
47 |
|
48 #define save_and_next(ls) (save(ls, ls->current), next(ls)) |
|
49 |
|
50 |
|
51 static void save (LexState *ls, int c) { |
|
52 Mbuffer *b = ls->buff; |
|
53 if (b->n + 1 > b->buffsize) { |
|
54 size_t newsize; |
|
55 if (b->buffsize >= MAX_SIZET/2) |
|
56 luaX_lexerror(ls, "lexical element too long", 0); |
|
57 newsize = b->buffsize * 2; |
|
58 luaZ_resizebuffer(ls->L, b, newsize); |
|
59 } |
|
60 b->buffer[b->n++] = cast(char, c); |
|
61 } |
|
62 |
|
63 |
|
64 void luaX_init (lua_State *L) { |
|
65 int i; |
|
66 for (i=0; i<NUM_RESERVED; i++) { |
|
67 TString *ts = luaS_new(L, luaX_tokens[i]); |
|
68 luaS_fix(ts); /* reserved words are never collected */ |
|
69 lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN); |
|
70 ts->tsv.reserved = cast_byte(i+1); /* reserved word */ |
|
71 } |
|
72 } |
|
73 |
|
74 |
|
75 #define MAXSRC 80 |
|
76 |
|
77 |
|
78 const char *luaX_token2str (LexState *ls, int token) { |
|
79 if (token < FIRST_RESERVED) { |
|
80 lua_assert(token == cast(unsigned char, token)); |
|
81 return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) : |
|
82 luaO_pushfstring(ls->L, "%c", token); |
|
83 } |
|
84 else |
|
85 return luaX_tokens[token-FIRST_RESERVED]; |
|
86 } |
|
87 |
|
88 |
|
89 static const char *txtToken (LexState *ls, int token) { |
|
90 switch (token) { |
|
91 case TK_NAME: |
|
92 case TK_STRING: |
|
93 case TK_NUMBER: |
|
94 save(ls, '\0'); |
|
95 return luaZ_buffer(ls->buff); |
|
96 default: |
|
97 return luaX_token2str(ls, token); |
|
98 } |
|
99 } |
|
100 |
|
101 |
|
102 void luaX_lexerror (LexState *ls, const char *msg, int token) { |
|
103 char buff[MAXSRC]; |
|
104 luaO_chunkid(buff, getstr(ls->source), MAXSRC); |
|
105 msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg); |
|
106 if (token) |
|
107 luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token)); |
|
108 luaD_throw(ls->L, LUA_ERRSYNTAX); |
|
109 } |
|
110 |
|
111 |
|
112 void luaX_syntaxerror (LexState *ls, const char *msg) { |
|
113 luaX_lexerror(ls, msg, ls->t.token); |
|
114 } |
|
115 |
|
116 |
|
117 TString *luaX_newstring (LexState *ls, const char *str, size_t l) { |
|
118 lua_State *L = ls->L; |
|
119 TString *ts = luaS_newlstr(L, str, l); |
|
120 TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */ |
|
121 if (ttisnil(o)) |
|
122 setbvalue(o, 1); /* make sure `str' will not be collected */ |
|
123 return ts; |
|
124 } |
|
125 |
|
126 |
|
127 static void inclinenumber (LexState *ls) { |
|
128 int old = ls->current; |
|
129 lua_assert(currIsNewline(ls)); |
|
130 next(ls); /* skip `\n' or `\r' */ |
|
131 if (currIsNewline(ls) && ls->current != old) |
|
132 next(ls); /* skip `\n\r' or `\r\n' */ |
|
133 if (++ls->linenumber >= MAX_INT) |
|
134 luaX_syntaxerror(ls, "chunk has too many lines"); |
|
135 } |
|
136 |
|
137 |
|
138 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) { |
|
139 ls->decpoint = '.'; |
|
140 ls->L = L; |
|
141 ls->lookahead.token = TK_EOS; /* no look-ahead token */ |
|
142 ls->z = z; |
|
143 ls->fs = NULL; |
|
144 ls->linenumber = 1; |
|
145 ls->lastline = 1; |
|
146 ls->source = source; |
|
147 luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */ |
|
148 next(ls); /* read first char */ |
|
149 } |
|
150 |
|
151 |
|
152 |
|
153 /* |
|
154 ** ======================================================= |
|
155 ** LEXICAL ANALYZER |
|
156 ** ======================================================= |
|
157 */ |
|
158 |
|
159 |
|
160 |
|
161 static int check_next (LexState *ls, const char *set) { |
|
162 if (!strchr(set, ls->current)) |
|
163 return 0; |
|
164 save_and_next(ls); |
|
165 return 1; |
|
166 } |
|
167 |
|
168 |
|
169 static void buffreplace (LexState *ls, char from, char to) { |
|
170 size_t n = luaZ_bufflen(ls->buff); |
|
171 char *p = luaZ_buffer(ls->buff); |
|
172 while (n--) |
|
173 if (p[n] == from) p[n] = to; |
|
174 } |
|
175 |
|
176 |
|
177 static void trydecpoint (LexState *ls, SemInfo *seminfo) { |
|
178 /* format error: try to update decimal point separator */ |
|
179 //struct lconv *cv = localeconv(); |
|
180 char old = ls->decpoint; |
|
181 //ls->decpoint = (cv ? cv->decimal_point[0] : '.'); |
|
182 ls->decpoint = '.'; |
|
183 buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */ |
|
184 if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) { |
|
185 /* format error with correct decimal point: no more options */ |
|
186 buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */ |
|
187 luaX_lexerror(ls, "malformed number", TK_NUMBER); |
|
188 } |
|
189 } |
|
190 |
|
191 |
|
192 /* LUA_NUMBER */ |
|
193 static void read_numeral (LexState *ls, SemInfo *seminfo) { |
|
194 lua_assert(isdigit(ls->current)); |
|
195 do { |
|
196 save_and_next(ls); |
|
197 } while (isdigit(ls->current) || ls->current == '.'); |
|
198 if (check_next(ls, "Ee")) /* `E'? */ |
|
199 check_next(ls, "+-"); /* optional exponent sign */ |
|
200 while (isalnum(ls->current) || ls->current == '_') |
|
201 save_and_next(ls); |
|
202 save(ls, '\0'); |
|
203 buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ |
|
204 if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */ |
|
205 trydecpoint(ls, seminfo); /* try to update decimal point separator */ |
|
206 } |
|
207 |
|
208 |
|
209 static int skip_sep (LexState *ls) { |
|
210 int count = 0; |
|
211 int s = ls->current; |
|
212 lua_assert(s == '[' || s == ']'); |
|
213 save_and_next(ls); |
|
214 while (ls->current == '=') { |
|
215 save_and_next(ls); |
|
216 count++; |
|
217 } |
|
218 return (ls->current == s) ? count : (-count) - 1; |
|
219 } |
|
220 |
|
221 |
|
222 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { |
|
223 int cont = 0; |
|
224 (void)(cont); /* avoid warnings when `cont' is not used */ |
|
225 save_and_next(ls); /* skip 2nd `[' */ |
|
226 if (currIsNewline(ls)) /* string starts with a newline? */ |
|
227 inclinenumber(ls); /* skip it */ |
|
228 for (;;) { |
|
229 switch (ls->current) { |
|
230 case EOZ: |
|
231 luaX_lexerror(ls, (seminfo) ? "unfinished long string" : |
|
232 "unfinished long comment", TK_EOS); |
|
233 break; /* to avoid warnings */ |
|
234 #if defined(LUA_COMPAT_LSTR) |
|
235 case '[': { |
|
236 if (skip_sep(ls) == sep) { |
|
237 save_and_next(ls); /* skip 2nd `[' */ |
|
238 cont++; |
|
239 #if LUA_COMPAT_LSTR == 1 |
|
240 if (sep == 0) |
|
241 luaX_lexerror(ls, "nesting of [[...]] is deprecated", '['); |
|
242 #endif |
|
243 } |
|
244 break; |
|
245 } |
|
246 #endif |
|
247 case ']': { |
|
248 if (skip_sep(ls) == sep) { |
|
249 save_and_next(ls); /* skip 2nd `]' */ |
|
250 #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2 |
|
251 cont--; |
|
252 if (sep == 0 && cont >= 0) break; |
|
253 #endif |
|
254 goto endloop; |
|
255 } |
|
256 break; |
|
257 } |
|
258 case '\n': |
|
259 case '\r': { |
|
260 save(ls, '\n'); |
|
261 inclinenumber(ls); |
|
262 if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ |
|
263 break; |
|
264 } |
|
265 default: { |
|
266 if (seminfo) save_and_next(ls); |
|
267 else next(ls); |
|
268 } |
|
269 } |
|
270 } endloop: |
|
271 if (seminfo) |
|
272 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), |
|
273 luaZ_bufflen(ls->buff) - 2*(2 + sep)); |
|
274 } |
|
275 |
|
276 |
|
277 static void read_string (LexState *ls, int del, SemInfo *seminfo) { |
|
278 save_and_next(ls); |
|
279 while (ls->current != del) { |
|
280 switch (ls->current) { |
|
281 case EOZ: |
|
282 luaX_lexerror(ls, "unfinished string", TK_EOS); |
|
283 continue; /* to avoid warnings */ |
|
284 case '\n': |
|
285 case '\r': |
|
286 luaX_lexerror(ls, "unfinished string", TK_STRING); |
|
287 continue; /* to avoid warnings */ |
|
288 case '\\': { |
|
289 int c; |
|
290 next(ls); /* do not save the `\' */ |
|
291 switch (ls->current) { |
|
292 case 'a': c = '\a'; break; |
|
293 case 'b': c = '\b'; break; |
|
294 case 'f': c = '\f'; break; |
|
295 case 'n': c = '\n'; break; |
|
296 case 'r': c = '\r'; break; |
|
297 case 't': c = '\t'; break; |
|
298 case 'v': c = '\v'; break; |
|
299 case '\n': /* go through */ |
|
300 case '\r': save(ls, '\n'); inclinenumber(ls); continue; |
|
301 case EOZ: continue; /* will raise an error next loop */ |
|
302 default: { |
|
303 if (!isdigit(ls->current)) |
|
304 save_and_next(ls); /* handles \\, \", \', and \? */ |
|
305 else { /* \xxx */ |
|
306 int i = 0; |
|
307 c = 0; |
|
308 do { |
|
309 c = 10*c + (ls->current-'0'); |
|
310 next(ls); |
|
311 } while (++i<3 && isdigit(ls->current)); |
|
312 if (c > UCHAR_MAX) |
|
313 luaX_lexerror(ls, "escape sequence too large", TK_STRING); |
|
314 save(ls, c); |
|
315 } |
|
316 continue; |
|
317 } |
|
318 } |
|
319 save(ls, c); |
|
320 next(ls); |
|
321 continue; |
|
322 } |
|
323 default: |
|
324 save_and_next(ls); |
|
325 } |
|
326 } |
|
327 save_and_next(ls); /* skip delimiter */ |
|
328 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, |
|
329 luaZ_bufflen(ls->buff) - 2); |
|
330 } |
|
331 |
|
332 |
|
333 static int llex (LexState *ls, SemInfo *seminfo) { |
|
334 luaZ_resetbuffer(ls->buff); |
|
335 for (;;) { |
|
336 switch (ls->current) { |
|
337 case '\n': |
|
338 case '\r': { |
|
339 inclinenumber(ls); |
|
340 continue; |
|
341 } |
|
342 case '-': { |
|
343 next(ls); |
|
344 if (ls->current != '-') return '-'; |
|
345 /* else is a comment */ |
|
346 next(ls); |
|
347 if (ls->current == '[') { |
|
348 int sep = skip_sep(ls); |
|
349 luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ |
|
350 if (sep >= 0) { |
|
351 read_long_string(ls, NULL, sep); /* long comment */ |
|
352 luaZ_resetbuffer(ls->buff); |
|
353 continue; |
|
354 } |
|
355 } |
|
356 /* else short comment */ |
|
357 while (!currIsNewline(ls) && ls->current != EOZ) |
|
358 next(ls); |
|
359 continue; |
|
360 } |
|
361 case '[': { |
|
362 int sep = skip_sep(ls); |
|
363 if (sep >= 0) { |
|
364 read_long_string(ls, seminfo, sep); |
|
365 return TK_STRING; |
|
366 } |
|
367 else if (sep == -1) return '['; |
|
368 else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING); |
|
369 } |
|
370 case '=': { |
|
371 next(ls); |
|
372 if (ls->current != '=') return '='; |
|
373 else { next(ls); return TK_EQ; } |
|
374 } |
|
375 case '<': { |
|
376 next(ls); |
|
377 if (ls->current != '=') return '<'; |
|
378 else { next(ls); return TK_LE; } |
|
379 } |
|
380 case '>': { |
|
381 next(ls); |
|
382 if (ls->current != '=') return '>'; |
|
383 else { next(ls); return TK_GE; } |
|
384 } |
|
385 case '~': { |
|
386 next(ls); |
|
387 if (ls->current != '=') return '~'; |
|
388 else { next(ls); return TK_NE; } |
|
389 } |
|
390 case '"': |
|
391 case '\'': { |
|
392 read_string(ls, ls->current, seminfo); |
|
393 return TK_STRING; |
|
394 } |
|
395 case '.': { |
|
396 save_and_next(ls); |
|
397 if (check_next(ls, ".")) { |
|
398 if (check_next(ls, ".")) |
|
399 return TK_DOTS; /* ... */ |
|
400 else return TK_CONCAT; /* .. */ |
|
401 } |
|
402 else if (!isdigit(ls->current)) return '.'; |
|
403 else { |
|
404 read_numeral(ls, seminfo); |
|
405 return TK_NUMBER; |
|
406 } |
|
407 } |
|
408 case EOZ: { |
|
409 return TK_EOS; |
|
410 } |
|
411 default: { |
|
412 if (isspace(ls->current)) { |
|
413 lua_assert(!currIsNewline(ls)); |
|
414 next(ls); |
|
415 continue; |
|
416 } |
|
417 else if (isdigit(ls->current)) { |
|
418 read_numeral(ls, seminfo); |
|
419 return TK_NUMBER; |
|
420 } |
|
421 else if (isalpha(ls->current) || ls->current == '_') { |
|
422 /* identifier or reserved word */ |
|
423 TString *ts; |
|
424 do { |
|
425 save_and_next(ls); |
|
426 } while (isalnum(ls->current) || ls->current == '_'); |
|
427 ts = luaX_newstring(ls, luaZ_buffer(ls->buff), |
|
428 luaZ_bufflen(ls->buff)); |
|
429 if (ts->tsv.reserved > 0) /* reserved word? */ |
|
430 return ts->tsv.reserved - 1 + FIRST_RESERVED; |
|
431 else { |
|
432 seminfo->ts = ts; |
|
433 return TK_NAME; |
|
434 } |
|
435 } |
|
436 else { |
|
437 int c = ls->current; |
|
438 next(ls); |
|
439 return c; /* single-char tokens (+ - / ...) */ |
|
440 } |
|
441 } |
|
442 } |
|
443 } |
|
444 } |
|
445 |
|
446 |
|
447 void luaX_next (LexState *ls) { |
|
448 ls->lastline = ls->linenumber; |
|
449 if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */ |
|
450 ls->t = ls->lookahead; /* use this one */ |
|
451 ls->lookahead.token = TK_EOS; /* and discharge it */ |
|
452 } |
|
453 else |
|
454 ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */ |
|
455 } |
|
456 |
|
457 |
|
458 void luaX_lookahead (LexState *ls) { |
|
459 lua_assert(ls->lookahead.token == TK_EOS); |
|
460 ls->lookahead.token = llex(ls, &ls->lookahead.seminfo); |
|
461 } |
|
462 |
|