1 /* 2 * Copyright (C) 2020-2021 Jo-Philipp Wich <jo@mein.io> 3 * 4 * Permission to use, copy, modify, and/or distribute this software for any 5 * purpose with or without fee is hereby granted, provided that the above 6 * copyright notice and this permission notice appear in all copies. 7 * 8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 */ 16 17 #include <ctype.h> 18 19 #include "ucode/internal/lexer.h" 20 #include "ucode/internal/vallist.h" 21 22 struct keyword { 23 unsigned type; 24 const char *pat; 25 unsigned plen; 26 }; 27 28 #define dec(o) \ 29 ((o) - '') 30 31 #define hex(x) \ 32 (((x) >= 'a') ? (10 + (x) - 'a') : \ 33 (((x) >= 'A') ? (10 + (x) - 'A') : dec(x))) 34 35 #ifndef NO_COMPILE 36 37 static const struct keyword reserved_words[] = { 38 { TK_ENDFUNC, "endfunction", 11 }, 39 { TK_CONTINUE, "continue", 8 }, 40 { TK_ENDWHILE, "endwhile", 8 }, 41 { TK_FUNC, "function", 8 }, 42 { TK_DEFAULT, "default", 7 }, 43 { TK_DELETE, "delete", 6 }, 44 { TK_RETURN, "return", 6 }, 45 { TK_ENDFOR, "endfor", 6 }, 46 { TK_SWITCH, "switch", 6 }, 47 { TK_IMPORT, "import", 6 }, 48 { TK_EXPORT, "export", 6 }, 49 { TK_ENDIF, "endif", 5 }, 50 { TK_WHILE, "while", 5 }, 51 { TK_BREAK, "break", 5 }, 52 { TK_CATCH, "catch", 5 }, 53 { TK_CONST, "const", 5 }, 54 { TK_FALSE, "false", 5 }, 55 { TK_TRUE, "true", 4 }, 56 { TK_ELIF, "elif", 4 }, 57 { TK_ELSE, "else", 4 }, 58 { TK_THIS, "this", 4 }, 59 { TK_NULL, "null", 4 }, 60 { TK_CASE, "case", 4 }, 61 { TK_TRY, "try", 3 }, 62 { TK_FOR, "for", 3 }, 63 { TK_LOCAL, "let", 3 }, 64 { TK_IF, "if", 2 }, 65 { TK_IN, "in", 2 }, 66 }; 67 68 69 static int 70 fill_buf(uc_lexer_t *lex) { 71 lex->rbuf = xrealloc(lex->rbuf, 128); 72 lex->rlen = fread(lex->rbuf, 1, 128, lex->source->fp); 73 lex->rpos = 0; 74 75 if (!lex->rlen) 76 return EOF; 77 78 lex->rpos++; 79 80 return (int)lex->rbuf[0]; 81 } 82 83 static int 84 update_line(uc_lexer_t *lex, int ch) { 85 if (ch == '\n') 86 uc_source_line_next(lex->source); 87 else if (ch != EOF) 88 uc_source_line_update(lex->source, 1); 89 90 lex->source->off++; 91 92 return ch; 93 } 94 95 static int 96 lookahead_char(uc_lexer_t *lex) { 97 int c; 98 99 if (lex->rpos < lex->rlen) 100 return (int)lex->rbuf[lex->rpos]; 101 102 c = fill_buf(lex); 103 lex->rpos = 0; 104 105 return c; 106 } 107 108 static bool 109 check_char(uc_lexer_t *lex, int ch) { 110 if (lookahead_char(lex) != ch) 111 return false; 112 113 lex->rpos++; 114 115 update_line(lex, ch); 116 117 return true; 118 } 119 120 static int 121 next_char(uc_lexer_t *lex) { 122 int ch = (lex->rpos < lex->rlen) ? (int)lex->rbuf[lex->rpos++] : fill_buf(lex); 123 124 return update_line(lex, ch); 125 } 126 127 static uc_token_t * 128 emit_op(uc_lexer_t *lex, ssize_t pos, int type, uc_value_t *uv) 129 { 130 lex->curr.type = type; 131 lex->curr.uv = uv; 132 133 if (pos < 0) 134 lex->curr.pos = lex->source->off + pos; 135 else 136 lex->curr.pos = (size_t)pos; 137 138 lex->curr.end = lex->source->off; 139 140 return &lex->curr; 141 } 142 143 static uc_token_t * 144 emit_buffer(uc_lexer_t *lex, ssize_t pos, int type, const char *strip_trailing_chars) { 145 uc_token_t *rv = NULL; 146 char *p; 147 148 if (lex->buffer.count) { 149 if (strip_trailing_chars) 150 for (p = uc_vector_last(&lex->buffer); 151 p && strchr(strip_trailing_chars, *p); 152 lex->buffer.count--, p = uc_vector_last(&lex->buffer)); 153 154 rv = emit_op(lex, pos, type, ucv_string_new_length(uc_vector_first(&lex->buffer), lex->buffer.count)); 155 156 uc_vector_clear(&lex->buffer); 157 } 158 else if (type != TK_TEXT) { 159 rv = emit_op(lex, pos, type, ucv_string_new_length("", 0)); 160 } 161 162 return rv; 163 } 164 165 166 static uc_token_t * 167 parse_comment(uc_lexer_t *lex, int kind) 168 { 169 size_t off = lex->source->off - 1; 170 int ch; 171 172 uc_vector_push(&lex->buffer, '/'); 173 174 /* consume the second opening character so that the opening star of a block 175 * comment cannot double as the closing star of an empty comment */ 176 uc_vector_push(&lex->buffer, next_char(lex)); 177 178 while (true) { 179 ch = next_char(lex); 180 181 uc_vector_push(&lex->buffer, ch); 182 183 if (kind == '/' && (ch == '\n' || ch == EOF)) 184 break; 185 186 if (kind == '*' && ch == '*' && check_char(lex, '/')) { 187 uc_vector_push(&lex->buffer, '/'); 188 break; 189 } 190 191 if (ch == EOF) { 192 lex->state = UC_LEX_EOF; 193 194 return emit_op(lex, lex->lastoff, TK_ERROR, ucv_string_new("Unterminated comment")); 195 } 196 } 197 198 return emit_buffer(lex, off, TK_COMMENT, NULL); 199 } 200 201 static void 202 append_utf8(uc_lexer_t *lex, int code) { 203 char ustr[8], *up; 204 int rem; 205 206 up = ustr; 207 rem = sizeof(ustr); 208 209 if (utf8enc(&up, &rem, code)) 210 for (up = ustr; rem < (int)sizeof(ustr); rem++) 211 uc_vector_push(&lex->buffer, *up++); 212 } 213 214 static uc_token_t * 215 parse_escape(uc_lexer_t *lex, const char *regex_macros) 216 { 217 int code, ch, i; 218 const char *p; 219 220 /* unicode escape sequence */ 221 if (check_char(lex, 'u')) { 222 for (i = 0, code = 0; i < 4; i++) { 223 ch = next_char(lex); 224 225 if (!isxdigit(ch)) 226 return emit_op(lex, -1, TK_ERROR, ucv_string_new("Invalid escape sequence")); 227 228 code = code * 16 + hex(ch); 229 } 230 231 /* is a leading surrogate value */ 232 if ((code & 0xFC00) == 0xD800) { 233 /* found a subsequent leading surrogate, ignore and emit replacement char for previous one */ 234 if (lex->lead_surrogate) 235 append_utf8(lex, 0xFFFD); 236 237 /* store surrogate value and advance to next escape sequence */ 238 lex->lead_surrogate = code; 239 } 240 241 /* is a trailing surrogate value */ 242 else if ((code & 0xFC00) == 0xDC00) { 243 /* found a trailing surrogate following a leading one, combine and encode */ 244 if (lex->lead_surrogate) { 245 code = 0x10000 + ((lex->lead_surrogate & 0x3FF) << 10) + (code & 0x3FF); 246 lex->lead_surrogate = 0; 247 } 248 249 /* trailing surrogate not following a leading one, ignore and use replacement char */ 250 else { 251 code = 0xFFFD; 252 } 253 254 append_utf8(lex, code); 255 } 256 257 /* is a normal codepoint */ 258 else { 259 append_utf8(lex, code); 260 } 261 } 262 263 /* hex escape sequence */ 264 else if (check_char(lex, 'x')) { 265 for (i = 0, code = 0; i < 2; i++) { 266 ch = next_char(lex); 267 268 if (!isxdigit(ch)) 269 return emit_op(lex, -1, TK_ERROR, ucv_string_new("Invalid escape sequence")); 270 271 code = code * 16 + hex(ch); 272 } 273 274 uc_vector_push(&lex->buffer, code); 275 } 276 277 /* octal or letter */ 278 else { 279 /* try to parse octal sequence... */ 280 for (i = 0, code = 0, ch = lookahead_char(lex); 281 i < 3 && ch >= '' && ch <= '7'; 282 i++, next_char(lex), ch = lookahead_char(lex)) { 283 code = code * 8 + dec(ch); 284 } 285 286 if (i) { 287 if (code > 255) 288 return emit_op(lex, -3, TK_ERROR, ucv_string_new("Invalid escape sequence")); 289 290 uc_vector_push(&lex->buffer, code); 291 } 292 293 /* ... no octal sequence, handle potential regex macros */ 294 else if (strchr(regex_macros, ch)) { 295 ch = next_char(lex); 296 297 switch (ch) { 298 case 'd': p = "[[:digit:]]"; break; 299 case 'D': p = "[^[:digit:]]"; break; 300 case 'w': p = "[[:alnum:]_]"; break; 301 case 'W': p = "[^[:alnum:]_]"; break; 302 case 's': p = "[[:space:]]"; break; 303 case 'S': p = "[^[:space:]]"; break; 304 default: p = NULL; 305 } 306 307 if (p) { 308 while (*p) 309 uc_vector_push(&lex->buffer, *p++); 310 } 311 else { 312 uc_vector_push(&lex->buffer, '\\'); 313 uc_vector_push(&lex->buffer, ch); 314 } 315 } 316 317 /* ... handle other escape */ 318 else { 319 ch = next_char(lex); 320 321 switch (ch) { 322 case 'a': uc_vector_push(&lex->buffer, '\a'); break; 323 case 'b': uc_vector_push(&lex->buffer, '\b'); break; 324 case 'e': uc_vector_push(&lex->buffer, '\033'); break; 325 case 'f': uc_vector_push(&lex->buffer, '\f'); break; 326 case 'n': uc_vector_push(&lex->buffer, '\n'); break; 327 case 'r': uc_vector_push(&lex->buffer, '\r'); break; 328 case 't': uc_vector_push(&lex->buffer, '\t'); break; 329 case 'v': uc_vector_push(&lex->buffer, '\v'); break; 330 331 case EOF: 332 return emit_op(lex, -2, TK_ERROR, ucv_string_new("Unterminated string")); 333 334 default: 335 uc_vector_push(&lex->buffer, ch); 336 } 337 } 338 } 339 340 return NULL; 341 } 342 343 static uc_token_t * 344 parse_string(uc_lexer_t *lex, int kind) 345 { 346 uc_token_t *err, *tok; 347 unsigned type; 348 int code, ch; 349 size_t off; 350 351 if (kind == '`') 352 type = TK_TEMPLATE; 353 else if (kind == '/') 354 type = TK_REGEXP; 355 else 356 type = TK_STRING; 357 358 off = lex->source->off - 1; 359 360 for (ch = next_char(lex); ch != EOF; ch = next_char(lex)) { 361 switch (ch) { 362 /* placeholder */ 363 case '$': 364 if (type == TK_TEMPLATE && check_char(lex, '{')) { 365 lex->state = UC_LEX_PLACEHOLDER_START; 366 367 tok = emit_buffer(lex, off, type, NULL); 368 tok->end -= 2; 369 370 return tok; 371 } 372 373 uc_vector_push(&lex->buffer, '$'); 374 break; 375 376 /* regexp bracket expression */ 377 case '[': 378 uc_vector_push(&lex->buffer, '['); 379 380 if (type == TK_REGEXP) { 381 /* skip leading negation (^) */ 382 if (check_char(lex, '^')) 383 uc_vector_push(&lex->buffer, '^'); 384 385 /* skip leading `]` - it is literal and not closing the bracket expr */ 386 if (check_char(lex, ']')) 387 uc_vector_push(&lex->buffer, ']'); 388 389 /* read until closing `]` */ 390 for (ch = next_char(lex); ch != EOF; ch = next_char(lex)) { 391 if (ch == '\\') { 392 err = parse_escape(lex, "^"); 393 394 if (err) 395 return err; 396 397 continue; 398 } 399 400 uc_vector_push(&lex->buffer, ch); 401 402 if (ch == ']') 403 break; 404 405 /* skip nested char classes / equivalence classes / collating chars */ 406 if (ch == '[') { 407 code = lookahead_char(lex); 408 409 if (code == ':' || code == '.' || code == '=') { 410 uc_vector_push(&lex->buffer, code); 411 next_char(lex); 412 413 for (ch = next_char(lex); ch != EOF; ch = next_char(lex)) { 414 if (ch == '\\') { 415 err = parse_escape(lex, ""); 416 417 if (err) 418 return err; 419 420 continue; 421 } 422 423 uc_vector_push(&lex->buffer, ch); 424 425 if (ch == code && check_char(lex, ']')) { 426 uc_vector_push(&lex->buffer, ']'); 427 break; 428 } 429 } 430 } 431 } 432 } 433 } 434 435 break; 436 437 /* escape sequence */ 438 case '\\': 439 err = parse_escape(lex, 440 (type == TK_REGEXP) ? "^bBdDsSwW<>.[$()|*+?{\\" : ""); 441 442 if (err) 443 return err; 444 445 break; 446 447 /* other character */ 448 default: 449 /* terminating delimitter */ 450 if (ch == kind) 451 return emit_buffer(lex, off, type, NULL); 452 453 uc_vector_push(&lex->buffer, ch); 454 } 455 } 456 457 // FIXME 458 lex->state = UC_LEX_EOF; 459 460 return emit_op(lex, lex->lastoff, TK_ERROR, ucv_string_new("Unterminated string")); 461 } 462 463 464 /* 465 * Parses a regexp literal from the given buffer. 466 * 467 * Returns a negative value on error, otherwise the amount of consumed 468 * characters from the given buffer. 469 * 470 * Error values: 471 * -UC_ERROR_UNTERMINATED_STRING Unterminated regexp 472 * -UC_ERROR_INVALID_ESCAPE Invalid escape sequence 473 * -UC_ERROR_OVERLONG_STRING Regexp literal too long 474 * -UC_ERROR_INVALID_REGEXP Could not compile regexp 475 */ 476 477 enum { 478 UC_LEX_PARSE_REGEX_INIT, 479 UC_LEX_PARSE_REGEX_PATTERN, 480 UC_LEX_PARSE_REGEX_FLAGS 481 }; 482 483 static uc_token_t * 484 parse_regexp(uc_lexer_t *lex) 485 { 486 bool is_reg_global = false, is_reg_icase = false, is_reg_newline = false; 487 uc_token_t *rv; 488 size_t len; 489 char *s; 490 491 rv = parse_string(lex, '/'); 492 493 if (rv->type == TK_REGEXP) { 494 while (true) { 495 if (check_char(lex, 'g')) 496 is_reg_global = true; 497 else if (check_char(lex, 'i')) 498 is_reg_icase = true; 499 else if (check_char(lex, 's')) 500 is_reg_newline = true; 501 else 502 break; 503 } 504 505 /* encode the flags byte followed by the raw pattern bytes; a plain 506 * printf conversion cannot be used as the pattern may contain NUL */ 507 len = ucv_string_length(rv->uv); 508 s = xalloc(len + 1); 509 s[0] = (is_reg_global << 0) | (is_reg_icase << 1) | (is_reg_newline << 2); 510 memcpy(s + 1, ucv_string_get(rv->uv), len); 511 512 ucv_free(rv->uv, false); 513 rv->uv = ucv_string_new_length(s, len + 1); 514 free(s); 515 } 516 517 return rv; 518 } 519 520 521 /* 522 * Parses a label from the given buffer. 523 * 524 * Returns a negative value on error, otherwise the amount of consumed 525 * characters from the given buffer. 526 * 527 * Error values: 528 * -UC_ERROR_OVERLONG_STRING Label too long 529 */ 530 531 static uc_token_t * 532 parse_label(uc_lexer_t *lex, int ch) 533 { 534 const struct keyword *word; 535 size_t i, len; 536 537 while (true) { 538 uc_vector_push(&lex->buffer, ch); 539 ch = lookahead_char(lex); 540 541 if (!isalnum(ch) && ch != '_') 542 break; 543 544 next_char(lex); 545 } 546 547 len = lex->buffer.count; 548 549 if (!lex->no_keyword) { 550 for (i = 0, word = &reserved_words[0]; i < ARRAY_SIZE(reserved_words); i++, word = &reserved_words[i]) { 551 if (lex->buffer.count == word->plen && !strncmp(uc_vector_first(&lex->buffer), word->pat, word->plen)) { 552 uc_vector_clear(&lex->buffer); 553 554 return emit_op(lex, -len, word->type, NULL); 555 } 556 } 557 } 558 559 return emit_buffer(lex, -len, TK_LABEL, NULL); 560 } 561 562 563 /* 564 * Parses a number literal from the given buffer. 565 * 566 * Returns a negative value on error, otherwise the amount of consumed 567 * characters from the given buffer. 568 * 569 * Error values: 570 * -UC_ERROR_INVALID_ESCAPE Invalid number character 571 */ 572 573 static inline bool 574 is_numeric_char(uc_lexer_t *lex, char c) 575 { 576 char prev = lex->buffer.count ? *uc_vector_last(&lex->buffer) : 0; 577 578 switch (c|32) { 579 case '.': 580 case '': 581 case '1': 582 case '2': 583 case '3': 584 case '4': 585 case '5': 586 case '6': 587 case '7': 588 case '8': 589 case '9': 590 return true; 591 592 case 'a': 593 case 'b': 594 case 'c': 595 case 'd': 596 case 'e': 597 case 'f': 598 case 'o': 599 case 'x': 600 /* require previous char, a number literal cannot start with these */ 601 return prev != 0; 602 603 case '+': 604 case '-': 605 /* sign is only allowed after a decimal exponent char; in a hex 606 * literal `e`/`E` is a digit, not an exponent marker */ 607 if ((prev|32) != 'e') 608 return false; 609 610 if (lex->buffer.count >= 2) { 611 char *b = uc_vector_first(&lex->buffer); 612 613 if (b[0] == '' && (b[1]|32) == 'x') 614 return false; 615 } 616 617 return true; 618 } 619 620 return false; 621 } 622 623 static uc_token_t * 624 parse_number(uc_lexer_t *lex, int ch) 625 { 626 uc_value_t *nv = NULL; 627 size_t len; 628 char *e; 629 630 while (true) { 631 uc_vector_push(&lex->buffer, ch); 632 ch = lookahead_char(lex); 633 634 if (!is_numeric_char(lex, ch)) 635 break; 636 637 next_char(lex); 638 } 639 640 len = lex->buffer.count; 641 642 uc_vector_push(&lex->buffer, '\0'); 643 644 nv = uc_number_parse_octal(uc_vector_first(&lex->buffer), &e); 645 646 uc_vector_clear(&lex->buffer); 647 648 switch (ucv_type(nv)) { 649 case UC_DOUBLE: 650 return emit_op(lex, -len, TK_DOUBLE, nv); 651 652 case UC_INTEGER: 653 return emit_op(lex, -len, TK_NUMBER, nv); 654 655 default: 656 return emit_op(lex, -len, TK_ERROR, ucv_string_new("Invalid number literal")); 657 } 658 } 659 660 static uc_token_t * 661 lex_find_token(uc_lexer_t *lex) 662 { 663 bool tpl = !(lex->config && lex->config->raw_mode); 664 int ch = next_char(lex); 665 666 while (isspace(ch)) 667 ch = next_char(lex); 668 669 switch (ch) { 670 case '~': 671 return emit_op(lex, -1, TK_COMPL, NULL); 672 673 case '}': 674 if (tpl && check_char(lex, '}')) 675 return emit_op(lex, -2, TK_REXP, NULL); 676 677 return emit_op(lex, -1, TK_RBRACE, NULL); 678 679 case '|': 680 if (check_char(lex, '|')) { 681 if (check_char(lex, '=')) 682 return emit_op(lex, -3, TK_ASOR, NULL); 683 684 return emit_op(lex, -2, TK_OR, NULL); 685 } 686 687 if (check_char(lex, '=')) 688 return emit_op(lex, -2, TK_ASBOR, NULL); 689 690 return emit_op(lex, -1, TK_BOR, NULL); 691 692 case '{': 693 if (tpl && check_char(lex, '{')) 694 return emit_op(lex, -2, TK_LEXP, NULL); 695 696 if (tpl && check_char(lex, '%')) 697 return emit_op(lex, -2, TK_LSTM, NULL); 698 699 return emit_op(lex, -1, TK_LBRACE, NULL); 700 701 case '^': 702 if (check_char(lex, '=')) 703 return emit_op(lex, -2, TK_ASBXOR, NULL); 704 705 return emit_op(lex, -1, TK_BXOR, NULL); 706 707 case '[': 708 return emit_op(lex, -1, TK_LBRACK, NULL); 709 710 case ']': 711 return emit_op(lex, -1, TK_RBRACK, NULL); 712 713 case '?': 714 if (check_char(lex, '?')) { 715 if (check_char(lex, '=')) 716 return emit_op(lex, -3, TK_ASNULLISH, NULL); 717 718 return emit_op(lex, -2, TK_NULLISH, NULL); 719 } 720 721 if (check_char(lex, '.')) { 722 if (check_char(lex, '[')) 723 return emit_op(lex, -3, TK_QLBRACK, NULL); 724 725 if (check_char(lex, '(')) 726 return emit_op(lex, -3, TK_QLPAREN, NULL); 727 728 return emit_op(lex, -2, TK_QDOT, NULL); 729 } 730 731 return emit_op(lex, -1, TK_QMARK, NULL); 732 733 case '>': 734 if (check_char(lex, '>')) { 735 if (check_char(lex, '=')) 736 return emit_op(lex, -3, TK_ASRIGHT, NULL); 737 738 return emit_op(lex, -2, TK_RSHIFT, NULL); 739 } 740 741 if (check_char(lex, '=')) 742 return emit_op(lex, -2, TK_GE, NULL); 743 744 return emit_op(lex, -1, TK_GT, NULL); 745 746 case '=': 747 if (check_char(lex, '=')) { 748 if (check_char(lex, '=')) 749 return emit_op(lex, -3, TK_EQS, NULL); 750 751 return emit_op(lex, -2, TK_EQ, NULL); 752 } 753 754 if (check_char(lex, '>')) 755 return emit_op(lex, -2, TK_ARROW, NULL); 756 757 return emit_op(lex, -1, TK_ASSIGN, NULL); 758 759 case '<': 760 if (check_char(lex, '<')) { 761 if (check_char(lex, '=')) 762 return emit_op(lex, -3, TK_ASLEFT, NULL); 763 764 return emit_op(lex, -2, TK_LSHIFT, NULL); 765 } 766 767 if (check_char(lex, '=')) 768 return emit_op(lex, -2, TK_LE, NULL); 769 770 return emit_op(lex, -1, TK_LT, NULL); 771 772 case ';': 773 return emit_op(lex, -1, TK_SCOL, NULL); 774 775 case ':': 776 return emit_op(lex, -1, TK_COLON, NULL); 777 778 case '/': 779 ch = lookahead_char(lex); 780 lex->lastoff = lex->source->off - 1; 781 782 if (ch == '/' || ch == '*') 783 return parse_comment(lex, ch); 784 785 if (lex->no_regexp) { 786 if (check_char(lex, '=')) 787 return emit_op(lex, -2, TK_ASDIV, NULL); 788 789 return emit_op(lex, -1, TK_DIV, NULL); 790 } 791 792 return parse_regexp(lex); 793 794 case '.': 795 if (check_char(lex, '.')) { 796 if (check_char(lex, '.')) 797 return emit_op(lex, -3, TK_ELLIP, NULL); 798 799 /* The sequence ".." cannot be a valid */ 800 return emit_op(lex, -2, TK_ERROR, ucv_string_new("Unexpected character")); 801 } 802 803 return emit_op(lex, -1, TK_DOT, NULL); 804 805 case '-': 806 if (tpl && check_char(lex, '}')) { 807 if (check_char(lex, '}')) { 808 lex->modifier = MINUS; 809 810 return emit_op(lex, -3, TK_REXP, NULL); 811 } 812 813 /* The sequence "-}" cannot be a valid */ 814 return emit_op(lex, -1, TK_ERROR, ucv_string_new("Unexpected character")); 815 } 816 817 if (tpl && check_char(lex, '%')) { 818 if (check_char(lex, '}')) { 819 lex->modifier = MINUS; 820 821 return emit_op(lex, -3, TK_RSTM, NULL); 822 } 823 824 /* The sequence "-%" cannot be a valid */ 825 return emit_op(lex, -1, TK_ERROR, ucv_string_new("Unexpected character")); 826 } 827 828 if (check_char(lex, '=')) 829 return emit_op(lex, -2, TK_ASSUB, NULL); 830 831 if (check_char(lex, '-')) 832 return emit_op(lex, -2, TK_DEC, NULL); 833 834 return emit_op(lex, -1, TK_SUB, NULL); 835 836 case ',': 837 return emit_op(lex, -1, TK_COMMA, NULL); 838 839 case '+': 840 if (check_char(lex, '=')) 841 return emit_op(lex, -2, TK_ASADD, NULL); 842 843 if (check_char(lex, '+')) 844 return emit_op(lex, -2, TK_INC, NULL); 845 846 return emit_op(lex, -1, TK_ADD, NULL); 847 848 case '*': 849 if (check_char(lex, '*')) { 850 if (check_char(lex, '=')) 851 return emit_op(lex, -3, TK_ASEXP, NULL); 852 853 return emit_op(lex, -2, TK_EXP, NULL); 854 } 855 856 if (check_char(lex, '=')) 857 return emit_op(lex, -2, TK_ASMUL, NULL); 858 859 return emit_op(lex, -1, TK_MUL, NULL); 860 861 case '(': 862 return emit_op(lex, -1, TK_LPAREN, NULL); 863 864 case ')': 865 return emit_op(lex, -1, TK_RPAREN, NULL); 866 867 case '\'': 868 case '"': 869 case '`': 870 lex->lastoff = lex->source->off - 1; 871 872 return parse_string(lex, ch); 873 874 case '&': 875 if (check_char(lex, '&')) { 876 if (check_char(lex, '=')) 877 return emit_op(lex, -3, TK_ASAND, NULL); 878 879 return emit_op(lex, -2, TK_AND, NULL); 880 } 881 882 if (check_char(lex, '=')) 883 return emit_op(lex, -2, TK_ASBAND, NULL); 884 885 return emit_op(lex, -1, TK_BAND, NULL); 886 887 case '%': 888 if (tpl && check_char(lex, '}')) 889 return emit_op(lex, -2, TK_RSTM, NULL); 890 891 if (check_char(lex, '=')) 892 return emit_op(lex, -2, TK_ASMOD, NULL); 893 894 return emit_op(lex, -1, TK_MOD, NULL); 895 896 case '!': 897 if (check_char(lex, '=')) { 898 if (check_char(lex, '=')) 899 return emit_op(lex, -3, TK_NES, NULL); 900 901 return emit_op(lex, -2, TK_NE, NULL); 902 } 903 904 return emit_op(lex, -1, TK_NOT, NULL); 905 906 case EOF: 907 return emit_op(lex, -1, TK_EOF, NULL); 908 909 default: 910 if (isalpha(ch) || ch == '_') 911 return parse_label(lex, ch); 912 913 if (isdigit(ch)) 914 return parse_number(lex, ch); 915 916 return emit_op(lex, -1, TK_ERROR, ucv_string_new("Unexpected character")); 917 } 918 } 919 920 static uc_token_t * 921 lex_step(uc_lexer_t *lex) 922 { 923 const char *strip = NULL; 924 uc_token_t *tok; 925 size_t *nest; 926 int ch; 927 928 while (lex->state != UC_LEX_EOF) { 929 switch (lex->state) { 930 case UC_LEX_IDENTIFY_BLOCK: 931 ch = next_char(lex); 932 933 /* previous block had strip trailing whitespace flag, skip leading whitespace */ 934 if (lex->modifier == MINUS) { 935 while (isspace(ch)) 936 ch = next_char(lex); 937 938 lex->modifier = UNSPEC; 939 } 940 941 /* previous block was a statement block and trim_blocks is enabled, skip leading newline */ 942 else if (lex->modifier == NEWLINE) { 943 if (ch == '\n') 944 ch = next_char(lex); 945 946 lex->modifier = UNSPEC; 947 } 948 949 /* scan forward through buffer to identify block start token */ 950 while (ch != EOF) { 951 if (ch == '{') { 952 ch = next_char(lex); 953 954 switch (ch) { 955 /* found start of comment block */ 956 case '#': 957 lex->state = UC_LEX_BLOCK_COMMENT; 958 lex->block = COMMENT; 959 960 if (check_char(lex, '-')) 961 strip = " \n\t\v\f\r"; 962 963 break; 964 965 /* found start of expression block */ 966 case '{': 967 lex->state = UC_LEX_BLOCK_EXPRESSION_EMIT_TAG; 968 969 if (check_char(lex, '-')) 970 strip = " \n\t\v\f\r"; 971 972 break; 973 974 /* found start of statement block */ 975 case '%': 976 lex->state = UC_LEX_BLOCK_STATEMENT_EMIT_TAG; 977 978 if (check_char(lex, '-')) 979 strip = " \n\t\v\f\r"; 980 else if (check_char(lex, '+')) 981 strip = NULL; 982 else if (lex->config && lex->config->lstrip_blocks) 983 strip = " \t\v\f\r"; 984 985 break; 986 987 default: 988 /* not a start tag, remember char and move on */ 989 uc_vector_push(&lex->buffer, '{'); 990 continue; 991 } 992 993 break; 994 } 995 996 uc_vector_push(&lex->buffer, ch); 997 ch = next_char(lex); 998 } 999 1000 if (ch == EOF) 1001 lex->state = UC_LEX_EOF; 1002 1003 /* push out leading text */ 1004 tok = emit_buffer(lex, lex->lastoff, TK_TEXT, strip); 1005 lex->lastoff = lex->source->off - 2; 1006 1007 if (!tok) 1008 continue; 1009 1010 tok->end -= 2; 1011 1012 return tok; 1013 1014 1015 case UC_LEX_BLOCK_COMMENT: 1016 ch = next_char(lex); 1017 1018 /* scan forward through buffer to identify end token */ 1019 while (ch != EOF) { 1020 if (ch == '-' && check_char(lex, '#') && check_char(lex, '}')) { 1021 lex->modifier = MINUS; 1022 break; 1023 } 1024 1025 if (ch == '#' && check_char(lex, '}')) 1026 break; 1027 1028 ch = next_char(lex); 1029 } 1030 1031 if (ch == EOF) { 1032 lex->state = UC_LEX_EOF; 1033 1034 return emit_op(lex, lex->lastoff, TK_ERROR, ucv_string_new("Unterminated template block")); 1035 } 1036 1037 tok = emit_op(lex, lex->lastoff, TK_COMMENT, NULL); 1038 1039 lex->lastoff = lex->source->off; 1040 lex->state = UC_LEX_IDENTIFY_BLOCK; 1041 1042 return tok; 1043 1044 case UC_LEX_BLOCK_EXPRESSION_EMIT_TAG: 1045 lex->state = UC_LEX_IDENTIFY_TOKEN; 1046 lex->block = EXPRESSION; 1047 1048 return emit_op(lex, lex->source->off - 2, TK_LEXP, NULL); 1049 1050 case UC_LEX_BLOCK_STATEMENT_EMIT_TAG: 1051 lex->state = UC_LEX_IDENTIFY_TOKEN; 1052 lex->block = STATEMENTS; 1053 1054 return emit_op(lex, lex->source->off - 2, TK_LSTM, NULL); 1055 1056 case UC_LEX_IDENTIFY_TOKEN: 1057 do { tok = lex_find_token(lex); } while (tok == NULL); 1058 1059 /* disallow nesting blocks */ 1060 if (tok->type == TK_LSTM || tok->type == TK_LEXP) 1061 return emit_op(lex, -2, TK_ERROR, ucv_string_new("Template blocks may not be nested")); 1062 1063 /* found end of statement block */ 1064 if (lex->block == STATEMENTS && tok->type == TK_RSTM) { 1065 /* strip newline after statement block? */ 1066 if (lex->modifier == UNSPEC && lex->config && lex->config->trim_blocks) 1067 lex->modifier = NEWLINE; 1068 1069 lex->lastoff = lex->source->off; 1070 lex->state = UC_LEX_IDENTIFY_BLOCK; 1071 lex->block = NONE; 1072 1073 tok = emit_op(lex, -2, TK_RSTM, NULL); 1074 } 1075 1076 /* found end of expression block */ 1077 else if (lex->block == EXPRESSION && tok->type == TK_REXP) { 1078 lex->lastoff = lex->source->off; 1079 lex->state = UC_LEX_IDENTIFY_BLOCK; 1080 lex->block = NONE; 1081 } 1082 1083 /* track opening braces */ 1084 else if (tok->type == TK_LBRACE && lex->templates.count > 0) { 1085 nest = uc_vector_last(&lex->templates); 1086 (*nest)++; 1087 } 1088 1089 /* check end of placeholder expression */ 1090 else if (tok->type == TK_RBRACE && lex->templates.count > 0) { 1091 nest = uc_vector_last(&lex->templates); 1092 1093 if (*nest == 0) { 1094 lex->templates.count--; 1095 lex->state = UC_LEX_PLACEHOLDER_END; 1096 } 1097 else { 1098 (*nest)--; 1099 } 1100 } 1101 1102 /* premature EOF? */ 1103 else if (tok->type == TK_EOF && lex->block != STATEMENTS) { 1104 lex->state = UC_LEX_EOF; 1105 1106 return emit_op(lex, -2, TK_ERROR, ucv_string_new("Unterminated template block")); 1107 } 1108 1109 return tok; 1110 1111 1112 case UC_LEX_PLACEHOLDER_START: 1113 lex->state = UC_LEX_IDENTIFY_TOKEN; 1114 1115 uc_vector_push(&lex->templates, 0); 1116 1117 return emit_op(lex, -2, TK_PLACEH, NULL); 1118 1119 1120 case UC_LEX_PLACEHOLDER_END: 1121 lex->state = UC_LEX_IDENTIFY_TOKEN; 1122 1123 tok = parse_string(lex, '`'); 1124 tok->pos++; 1125 1126 return tok; 1127 1128 1129 case UC_LEX_EOF: 1130 break; 1131 } 1132 } 1133 1134 return emit_op(lex, lex->source->off, TK_EOF, NULL); 1135 } 1136 1137 void 1138 uc_lexer_init(uc_lexer_t *lex, uc_parse_config_t *config, uc_source_t *source) 1139 { 1140 lex->state = UC_LEX_IDENTIFY_BLOCK; 1141 1142 lex->config = config; 1143 lex->source = uc_source_get(source); 1144 1145 lex->block = NONE; 1146 lex->modifier = UNSPEC; 1147 1148 lex->rlen = 0; 1149 lex->rpos = 0; 1150 lex->rbuf = NULL; 1151 1152 lex->buffer.count = 0; 1153 lex->buffer.entries = NULL; 1154 1155 lex->lead_surrogate = 0; 1156 1157 lex->lastoff = 0; 1158 1159 lex->templates.count = 0; 1160 lex->templates.entries = NULL; 1161 1162 if (config && config->raw_mode) { 1163 lex->state = UC_LEX_IDENTIFY_TOKEN; 1164 lex->block = STATEMENTS; 1165 } 1166 } 1167 1168 void 1169 uc_lexer_free(uc_lexer_t *lex) 1170 { 1171 uc_vector_clear(&lex->buffer); 1172 uc_vector_clear(&lex->templates); 1173 1174 uc_source_put(lex->source); 1175 1176 free(lex->rbuf); 1177 } 1178 1179 uc_token_t * 1180 uc_lexer_next_token(uc_lexer_t *lex) 1181 { 1182 uc_token_t *rv = NULL; 1183 1184 rv = lex_step(lex); 1185 1186 if (rv && rv->type != TK_COMMENT) { 1187 lex->no_keyword = false; 1188 lex->no_regexp = false; 1189 } 1190 1191 return rv; 1192 } 1193 1194 const char * 1195 uc_tokenname(unsigned type) 1196 { 1197 static char buf[sizeof("'endfunction'")]; 1198 const char *tokennames[] = { 1199 [TK_LEXP] = "'{{'", 1200 [TK_REXP] = "'}}'", 1201 [TK_LSTM] = "'{%'", 1202 [TK_RSTM] = "'%}'", 1203 [TK_COMMA] = "','", 1204 [TK_ASSIGN] = "'='", 1205 [TK_ASADD] = "'+='", 1206 [TK_ASSUB] = "'-='", 1207 [TK_ASMUL] = "'*='", 1208 [TK_ASDIV] = "'/='", 1209 [TK_ASMOD] = "'%='", 1210 [TK_ASLEFT] = "'<<='", 1211 [TK_ASRIGHT] = "'>>='", 1212 [TK_ASBAND] = "'&='", 1213 [TK_ASBXOR] = "'^='", 1214 [TK_ASBOR] = "'|='", 1215 [TK_QMARK] = "'?'", 1216 [TK_COLON] = "':'", 1217 [TK_OR] = "'||'", 1218 [TK_AND] = "'&&'", 1219 [TK_BOR] = "'|'", 1220 [TK_BXOR] = "'^'", 1221 [TK_BAND] = "'&'", 1222 [TK_EQS] = "'==='", 1223 [TK_NES] = "'!=='", 1224 [TK_EQ] = "'=='", 1225 [TK_NE] = "'!='", 1226 [TK_LT] = "'<'", 1227 [TK_LE] = "'<='", 1228 [TK_GT] = "'>'", 1229 [TK_GE] = "'>='", 1230 [TK_LSHIFT] = "'<<'", 1231 [TK_RSHIFT] = "'>>'", 1232 [TK_ADD] = "'+'", 1233 [TK_SUB] = "'-'", 1234 [TK_MUL] = "'*'", 1235 [TK_DIV] = "'/'", 1236 [TK_MOD] = "'%'", 1237 [TK_EXP] = "'**'", 1238 [TK_NOT] = "'!'", 1239 [TK_COMPL] = "'~'", 1240 [TK_INC] = "'++'", 1241 [TK_DEC] = "'--'", 1242 [TK_DOT] = "'.'", 1243 [TK_LBRACK] = "'['", 1244 [TK_RBRACK] = "']'", 1245 [TK_LPAREN] = "'('", 1246 [TK_RPAREN] = "')'", 1247 [TK_LBRACE] = "'{'", 1248 [TK_RBRACE] = "'}'", 1249 [TK_SCOL] = "';'", 1250 [TK_ELLIP] = "'...'", 1251 [TK_ARROW] = "'=>'", 1252 [TK_QLBRACK] = "'?.['", 1253 [TK_QLPAREN] = "'?.('", 1254 [TK_QDOT] = "'?.'", 1255 [TK_ASEXP] = "'**='", 1256 [TK_ASAND] = "'&&='", 1257 [TK_ASOR] = "'||='", 1258 [TK_ASNULLISH] = "'\?\?='", 1259 [TK_NULLISH] = "'\?\?'", 1260 [TK_PLACEH] = "'${'", 1261 1262 [TK_TEXT] = "Text", 1263 [TK_LABEL] = "Label", 1264 [TK_NUMBER] = "Number", 1265 [TK_DOUBLE] = "Double", 1266 [TK_STRING] = "String", 1267 [TK_REGEXP] = "Regexp", 1268 [TK_TEMPLATE] = "Template", 1269 [TK_ERROR] = "Error", 1270 [TK_EOF] = "End of file", 1271 }; 1272 1273 size_t i; 1274 1275 for (i = 0; i < ARRAY_SIZE(reserved_words); i++) { 1276 if (reserved_words[i].type != type) 1277 continue; 1278 1279 snprintf(buf, sizeof(buf), "'%s'", reserved_words[i].pat); 1280 1281 return buf; 1282 } 1283 1284 return tokennames[type] ? tokennames[type] : "?"; 1285 } 1286 1287 bool 1288 uc_lexer_is_keyword(uc_value_t *label) 1289 { 1290 size_t i; 1291 1292 if (ucv_type(label) != UC_STRING) 1293 return false; 1294 1295 for (i = 0; i < ARRAY_SIZE(reserved_words); i++) 1296 if (!strcmp(reserved_words[i].pat, ucv_string_get(label))) 1297 return true; 1298 1299 return false; 1300 } 1301 1302 #endif /* NO_COMPILE */ 1303 1304 /* 1305 * Stores the given codepoint as a utf8 multibyte sequence into the given 1306 * output buffer and subtracts the required amount of bytes from the given 1307 * length pointer. 1308 * 1309 * Returns false if the multibyte sequence would not fit into the buffer, 1310 * otherwise true. 1311 */ 1312 1313 bool 1314 utf8enc(char **out, int *rem, int code) 1315 { 1316 if (code >= 0 && code <= 0x7F) { 1317 if (*rem < 1) 1318 return false; 1319 1320 *(*out)++ = code; (*rem)--; 1321 1322 return true; 1323 } 1324 else if (code > 0 && code <= 0x7FF) { 1325 if (*rem < 2) 1326 return false; 1327 1328 *(*out)++ = ((code >> 6) & 0x1F) | 0xC0; (*rem)--; 1329 *(*out)++ = ( code & 0x3F) | 0x80; (*rem)--; 1330 1331 return true; 1332 } 1333 else if (code > 0 && code <= 0xFFFF) { 1334 if (*rem < 3) 1335 return false; 1336 1337 *(*out)++ = ((code >> 12) & 0x0F) | 0xE0; (*rem)--; 1338 *(*out)++ = ((code >> 6) & 0x3F) | 0x80; (*rem)--; 1339 *(*out)++ = ( code & 0x3F) | 0x80; (*rem)--; 1340 1341 return true; 1342 } 1343 else if (code > 0 && code <= 0x10FFFF) { 1344 if (*rem < 4) 1345 return false; 1346 1347 *(*out)++ = ((code >> 18) & 0x07) | 0xF0; (*rem)--; 1348 *(*out)++ = ((code >> 12) & 0x3F) | 0x80; (*rem)--; 1349 *(*out)++ = ((code >> 6) & 0x3F) | 0x80; (*rem)--; 1350 *(*out)++ = ( code & 0x3F) | 0x80; (*rem)--; 1351 1352 return true; 1353 } 1354 1355 return true; 1356 } 1357
This page was automatically generated by LXR 0.3.1. • OpenWrt