2812
+ − 1 /*
+ − 2 ** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $
+ − 3 ** Lexical Analyzer
+ − 4 ** See Copyright Notice in lua.h
+ − 5 */
+ − 6
+ − 7
+ − 8 #include <ctype.h>
+ − 9 #include <locale.h>
+ − 10 #include <string.h>
+ − 11
+ − 12 #define llex_c
+ − 13 #define LUA_CORE
+ − 14
+ − 15 #include "lua.h"
+ − 16
+ − 17 #include "ldo.h"
+ − 18 #include "llex.h"
+ − 19 #include "lobject.h"
+ − 20 #include "lparser.h"
+ − 21 #include "lstate.h"
+ − 22 #include "lstring.h"
+ − 23 #include "ltable.h"
+ − 24 #include "lzio.h"
+ − 25
+ − 26
+ − 27
+ − 28 #define next(ls) (ls->current = zgetc(ls->z))
+ − 29
+ − 30
+ − 31
+ − 32
+ − 33 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
+ − 34
+ − 35
+ − 36 /* ORDER RESERVED */
+ − 37 const char *const luaX_tokens [] = {
+ − 38 "and", "break", "do", "else", "elseif",
+ − 39 "end", "false", "for", "function", "if",
+ − 40 "in", "local", "nil", "not", "or", "repeat",
+ − 41 "return", "then", "true", "until", "while",
+ − 42 "..", "...", "==", ">=", "<=", "~=",
+ − 43 "<number>", "<name>", "<string>", "<eof>",
+ − 44 NULL
+ − 45 };
+ − 46
+ − 47
+ − 48 #define save_and_next(ls) (save(ls, ls->current), next(ls))
+ − 49
+ − 50
+ − 51 static void save (LexState *ls, int c) {
+ − 52 Mbuffer *b = ls->buff;
+ − 53 if (b->n + 1 > b->buffsize) {
+ − 54 size_t newsize;
+ − 55 if (b->buffsize >= MAX_SIZET/2)
+ − 56 luaX_lexerror(ls, "lexical element too long", 0);
+ − 57 newsize = b->buffsize * 2;
+ − 58 luaZ_resizebuffer(ls->L, b, newsize);
+ − 59 }
+ − 60 b->buffer[b->n++] = cast(char, c);
+ − 61 }
+ − 62
+ − 63
+ − 64 void luaX_init (lua_State *L) {
+ − 65 int i;
+ − 66 for (i=0; i<NUM_RESERVED; i++) {
+ − 67 TString *ts = luaS_new(L, luaX_tokens[i]);
+ − 68 luaS_fix(ts); /* reserved words are never collected */
+ − 69 lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN);
+ − 70 ts->tsv.reserved = cast_byte(i+1); /* reserved word */
+ − 71 }
+ − 72 }
+ − 73
+ − 74
+ − 75 #define MAXSRC 80
+ − 76
+ − 77
+ − 78 const char *luaX_token2str (LexState *ls, int token) {
+ − 79 if (token < FIRST_RESERVED) {
+ − 80 lua_assert(token == cast(unsigned char, token));
+ − 81 return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
+ − 82 luaO_pushfstring(ls->L, "%c", token);
+ − 83 }
+ − 84 else
+ − 85 return luaX_tokens[token-FIRST_RESERVED];
+ − 86 }
+ − 87
+ − 88
+ − 89 static const char *txtToken (LexState *ls, int token) {
+ − 90 switch (token) {
+ − 91 case TK_NAME:
+ − 92 case TK_STRING:
+ − 93 case TK_NUMBER:
+ − 94 save(ls, '\0');
+ − 95 return luaZ_buffer(ls->buff);
+ − 96 default:
+ − 97 return luaX_token2str(ls, token);
+ − 98 }
+ − 99 }
+ − 100
+ − 101
+ − 102 void luaX_lexerror (LexState *ls, const char *msg, int token) {
+ − 103 char buff[MAXSRC];
+ − 104 luaO_chunkid(buff, getstr(ls->source), MAXSRC);
+ − 105 msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
+ − 106 if (token)
+ − 107 luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
+ − 108 luaD_throw(ls->L, LUA_ERRSYNTAX);
+ − 109 }
+ − 110
+ − 111
+ − 112 void luaX_syntaxerror (LexState *ls, const char *msg) {
+ − 113 luaX_lexerror(ls, msg, ls->t.token);
+ − 114 }
+ − 115
+ − 116
+ − 117 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
+ − 118 lua_State *L = ls->L;
+ − 119 TString *ts = luaS_newlstr(L, str, l);
+ − 120 TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */
+ − 121 if (ttisnil(o))
+ − 122 setbvalue(o, 1); /* make sure `str' will not be collected */
+ − 123 return ts;
+ − 124 }
+ − 125
+ − 126
+ − 127 static void inclinenumber (LexState *ls) {
+ − 128 int old = ls->current;
+ − 129 lua_assert(currIsNewline(ls));
+ − 130 next(ls); /* skip `\n' or `\r' */
+ − 131 if (currIsNewline(ls) && ls->current != old)
+ − 132 next(ls); /* skip `\n\r' or `\r\n' */
+ − 133 if (++ls->linenumber >= MAX_INT)
+ − 134 luaX_syntaxerror(ls, "chunk has too many lines");
+ − 135 }
+ − 136
+ − 137
+ − 138 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
+ − 139 ls->decpoint = '.';
+ − 140 ls->L = L;
+ − 141 ls->lookahead.token = TK_EOS; /* no look-ahead token */
+ − 142 ls->z = z;
+ − 143 ls->fs = NULL;
+ − 144 ls->linenumber = 1;
+ − 145 ls->lastline = 1;
+ − 146 ls->source = source;
+ − 147 luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
+ − 148 next(ls); /* read first char */
+ − 149 }
+ − 150
+ − 151
+ − 152
+ − 153 /*
+ − 154 ** =======================================================
+ − 155 ** LEXICAL ANALYZER
+ − 156 ** =======================================================
+ − 157 */
+ − 158
+ − 159
+ − 160
+ − 161 static int check_next (LexState *ls, const char *set) {
+ − 162 if (!strchr(set, ls->current))
+ − 163 return 0;
+ − 164 save_and_next(ls);
+ − 165 return 1;
+ − 166 }
+ − 167
+ − 168
+ − 169 static void buffreplace (LexState *ls, char from, char to) {
+ − 170 size_t n = luaZ_bufflen(ls->buff);
+ − 171 char *p = luaZ_buffer(ls->buff);
+ − 172 while (n--)
+ − 173 if (p[n] == from) p[n] = to;
+ − 174 }
+ − 175
+ − 176
+ − 177 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
+ − 178 /* format error: try to update decimal point separator */
+ − 179 struct lconv *cv = localeconv();
+ − 180 char old = ls->decpoint;
+ − 181 ls->decpoint = (cv ? cv->decimal_point[0] : '.');
+ − 182 buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */
+ − 183 if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
+ − 184 /* format error with correct decimal point: no more options */
+ − 185 buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */
+ − 186 luaX_lexerror(ls, "malformed number", TK_NUMBER);
+ − 187 }
+ − 188 }
+ − 189
+ − 190
+ − 191 /* LUA_NUMBER */
+ − 192 static void read_numeral (LexState *ls, SemInfo *seminfo) {
+ − 193 lua_assert(isdigit(ls->current));
+ − 194 do {
+ − 195 save_and_next(ls);
+ − 196 } while (isdigit(ls->current) || ls->current == '.');
+ − 197 if (check_next(ls, "Ee")) /* `E'? */
+ − 198 check_next(ls, "+-"); /* optional exponent sign */
+ − 199 while (isalnum(ls->current) || ls->current == '_')
+ − 200 save_and_next(ls);
+ − 201 save(ls, '\0');
+ − 202 buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */
+ − 203 if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */
+ − 204 trydecpoint(ls, seminfo); /* try to update decimal point separator */
+ − 205 }
+ − 206
+ − 207
+ − 208 static int skip_sep (LexState *ls) {
+ − 209 int count = 0;
+ − 210 int s = ls->current;
+ − 211 lua_assert(s == '[' || s == ']');
+ − 212 save_and_next(ls);
+ − 213 while (ls->current == '=') {
+ − 214 save_and_next(ls);
+ − 215 count++;
+ − 216 }
+ − 217 return (ls->current == s) ? count : (-count) - 1;
+ − 218 }
+ − 219
+ − 220
+ − 221 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
+ − 222 int cont = 0;
+ − 223 (void)(cont); /* avoid warnings when `cont' is not used */
+ − 224 save_and_next(ls); /* skip 2nd `[' */
+ − 225 if (currIsNewline(ls)) /* string starts with a newline? */
+ − 226 inclinenumber(ls); /* skip it */
+ − 227 for (;;) {
+ − 228 switch (ls->current) {
+ − 229 case EOZ:
+ − 230 luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
+ − 231 "unfinished long comment", TK_EOS);
+ − 232 break; /* to avoid warnings */
+ − 233 #if defined(LUA_COMPAT_LSTR)
+ − 234 case '[': {
+ − 235 if (skip_sep(ls) == sep) {
+ − 236 save_and_next(ls); /* skip 2nd `[' */
+ − 237 cont++;
+ − 238 #if LUA_COMPAT_LSTR == 1
+ − 239 if (sep == 0)
+ − 240 luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
+ − 241 #endif
+ − 242 }
+ − 243 break;
+ − 244 }
+ − 245 #endif
+ − 246 case ']': {
+ − 247 if (skip_sep(ls) == sep) {
+ − 248 save_and_next(ls); /* skip 2nd `]' */
+ − 249 #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
+ − 250 cont--;
+ − 251 if (sep == 0 && cont >= 0) break;
+ − 252 #endif
+ − 253 goto endloop;
+ − 254 }
+ − 255 break;
+ − 256 }
+ − 257 case '\n':
+ − 258 case '\r': {
+ − 259 save(ls, '\n');
+ − 260 inclinenumber(ls);
+ − 261 if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
+ − 262 break;
+ − 263 }
+ − 264 default: {
+ − 265 if (seminfo) save_and_next(ls);
+ − 266 else next(ls);
+ − 267 }
+ − 268 }
+ − 269 } endloop:
+ − 270 if (seminfo)
+ − 271 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
+ − 272 luaZ_bufflen(ls->buff) - 2*(2 + sep));
+ − 273 }
+ − 274
+ − 275
+ − 276 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
+ − 277 save_and_next(ls);
+ − 278 while (ls->current != del) {
+ − 279 switch (ls->current) {
+ − 280 case EOZ:
+ − 281 luaX_lexerror(ls, "unfinished string", TK_EOS);
+ − 282 continue; /* to avoid warnings */
+ − 283 case '\n':
+ − 284 case '\r':
+ − 285 luaX_lexerror(ls, "unfinished string", TK_STRING);
+ − 286 continue; /* to avoid warnings */
+ − 287 case '\\': {
+ − 288 int c;
+ − 289 next(ls); /* do not save the `\' */
+ − 290 switch (ls->current) {
+ − 291 case 'a': c = '\a'; break;
+ − 292 case 'b': c = '\b'; break;
+ − 293 case 'f': c = '\f'; break;
+ − 294 case 'n': c = '\n'; break;
+ − 295 case 'r': c = '\r'; break;
+ − 296 case 't': c = '\t'; break;
+ − 297 case 'v': c = '\v'; break;
+ − 298 case '\n': /* go through */
+ − 299 case '\r': save(ls, '\n'); inclinenumber(ls); continue;
+ − 300 case EOZ: continue; /* will raise an error next loop */
+ − 301 default: {
+ − 302 if (!isdigit(ls->current))
+ − 303 save_and_next(ls); /* handles \\, \", \', and \? */
+ − 304 else { /* \xxx */
+ − 305 int i = 0;
+ − 306 c = 0;
+ − 307 do {
+ − 308 c = 10*c + (ls->current-'0');
+ − 309 next(ls);
+ − 310 } while (++i<3 && isdigit(ls->current));
+ − 311 if (c > UCHAR_MAX)
+ − 312 luaX_lexerror(ls, "escape sequence too large", TK_STRING);
+ − 313 save(ls, c);
+ − 314 }
+ − 315 continue;
+ − 316 }
+ − 317 }
+ − 318 save(ls, c);
+ − 319 next(ls);
+ − 320 continue;
+ − 321 }
+ − 322 default:
+ − 323 save_and_next(ls);
+ − 324 }
+ − 325 }
+ − 326 save_and_next(ls); /* skip delimiter */
+ − 327 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
+ − 328 luaZ_bufflen(ls->buff) - 2);
+ − 329 }
+ − 330
+ − 331
+ − 332 static int llex (LexState *ls, SemInfo *seminfo) {
+ − 333 luaZ_resetbuffer(ls->buff);
+ − 334 for (;;) {
+ − 335 switch (ls->current) {
+ − 336 case '\n':
+ − 337 case '\r': {
+ − 338 inclinenumber(ls);
+ − 339 continue;
+ − 340 }
+ − 341 case '-': {
+ − 342 next(ls);
+ − 343 if (ls->current != '-') return '-';
+ − 344 /* else is a comment */
+ − 345 next(ls);
+ − 346 if (ls->current == '[') {
+ − 347 int sep = skip_sep(ls);
+ − 348 luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */
+ − 349 if (sep >= 0) {
+ − 350 read_long_string(ls, NULL, sep); /* long comment */
+ − 351 luaZ_resetbuffer(ls->buff);
+ − 352 continue;
+ − 353 }
+ − 354 }
+ − 355 /* else short comment */
+ − 356 while (!currIsNewline(ls) && ls->current != EOZ)
+ − 357 next(ls);
+ − 358 continue;
+ − 359 }
+ − 360 case '[': {
+ − 361 int sep = skip_sep(ls);
+ − 362 if (sep >= 0) {
+ − 363 read_long_string(ls, seminfo, sep);
+ − 364 return TK_STRING;
+ − 365 }
+ − 366 else if (sep == -1) return '[';
+ − 367 else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
+ − 368 }
+ − 369 case '=': {
+ − 370 next(ls);
+ − 371 if (ls->current != '=') return '=';
+ − 372 else { next(ls); return TK_EQ; }
+ − 373 }
+ − 374 case '<': {
+ − 375 next(ls);
+ − 376 if (ls->current != '=') return '<';
+ − 377 else { next(ls); return TK_LE; }
+ − 378 }
+ − 379 case '>': {
+ − 380 next(ls);
+ − 381 if (ls->current != '=') return '>';
+ − 382 else { next(ls); return TK_GE; }
+ − 383 }
+ − 384 case '~': {
+ − 385 next(ls);
+ − 386 if (ls->current != '=') return '~';
+ − 387 else { next(ls); return TK_NE; }
+ − 388 }
+ − 389 case '"':
+ − 390 case '\'': {
+ − 391 read_string(ls, ls->current, seminfo);
+ − 392 return TK_STRING;
+ − 393 }
+ − 394 case '.': {
+ − 395 save_and_next(ls);
+ − 396 if (check_next(ls, ".")) {
+ − 397 if (check_next(ls, "."))
+ − 398 return TK_DOTS; /* ... */
+ − 399 else return TK_CONCAT; /* .. */
+ − 400 }
+ − 401 else if (!isdigit(ls->current)) return '.';
+ − 402 else {
+ − 403 read_numeral(ls, seminfo);
+ − 404 return TK_NUMBER;
+ − 405 }
+ − 406 }
+ − 407 case EOZ: {
+ − 408 return TK_EOS;
+ − 409 }
+ − 410 default: {
+ − 411 if (isspace(ls->current)) {
+ − 412 lua_assert(!currIsNewline(ls));
+ − 413 next(ls);
+ − 414 continue;
+ − 415 }
+ − 416 else if (isdigit(ls->current)) {
+ − 417 read_numeral(ls, seminfo);
+ − 418 return TK_NUMBER;
+ − 419 }
+ − 420 else if (isalpha(ls->current) || ls->current == '_') {
+ − 421 /* identifier or reserved word */
+ − 422 TString *ts;
+ − 423 do {
+ − 424 save_and_next(ls);
+ − 425 } while (isalnum(ls->current) || ls->current == '_');
+ − 426 ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
+ − 427 luaZ_bufflen(ls->buff));
+ − 428 if (ts->tsv.reserved > 0) /* reserved word? */
+ − 429 return ts->tsv.reserved - 1 + FIRST_RESERVED;
+ − 430 else {
+ − 431 seminfo->ts = ts;
+ − 432 return TK_NAME;
+ − 433 }
+ − 434 }
+ − 435 else {
+ − 436 int c = ls->current;
+ − 437 next(ls);
+ − 438 return c; /* single-char tokens (+ - / ...) */
+ − 439 }
+ − 440 }
+ − 441 }
+ − 442 }
+ − 443 }
+ − 444
+ − 445
+ − 446 void luaX_next (LexState *ls) {
+ − 447 ls->lastline = ls->linenumber;
+ − 448 if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
+ − 449 ls->t = ls->lookahead; /* use this one */
+ − 450 ls->lookahead.token = TK_EOS; /* and discharge it */
+ − 451 }
+ − 452 else
+ − 453 ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
+ − 454 }
+ − 455
+ − 456
+ − 457 void luaX_lookahead (LexState *ls) {
+ − 458 lua_assert(ls->lookahead.token == TK_EOS);
+ − 459 ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
+ − 460 }
+ − 461