1// lex.cc -- Go frontend lexer. 2 3// Copyright 2009 The Go Authors. All rights reserved. 4// Use of this source code is governed by a BSD-style 5// license that can be found in the LICENSE file. 6 7#include "go-system.h" 8 9#include "lex.h" 10 11// Manage mapping from keywords to the Keyword codes. 12 13class Keywords 14{ 15 public: 16 // The structure which maps keywords to codes. 17 struct Mapping 18 { 19 // Keyword string. 20 const char* keystring; 21 // Keyword code. 22 Keyword keycode; 23 }; 24 25 // Return the parsecode corresponding to KEYSTRING, or 26 // KEYWORD_INVALID if it is not a keyword. 27 Keyword 28 keyword_to_code(const char* keyword, size_t len) const; 29 30 // Return the string for a keyword. 31 const char* 32 keyword_to_string(Keyword) const; 33 34 private: 35 static const Mapping mapping_[]; 36 static const int count_; 37}; 38 39// Mapping from keyword string to keyword code. This array must be 40// kept in sorted order, and the order must match the Keyword enum. 41// Strings are looked up using bsearch. 42 43const Keywords::Mapping 44Keywords::mapping_[] = 45{ 46 { NULL, KEYWORD_INVALID }, 47 { "__asm__", KEYWORD_ASM }, 48 { "break", KEYWORD_BREAK }, 49 { "case", KEYWORD_CASE }, 50 { "chan", KEYWORD_CHAN }, 51 { "const", KEYWORD_CONST }, 52 { "continue", KEYWORD_CONTINUE }, 53 { "default", KEYWORD_DEFAULT }, 54 { "defer", KEYWORD_DEFER }, 55 { "else", KEYWORD_ELSE }, 56 { "fallthrough", KEYWORD_FALLTHROUGH }, 57 { "for", KEYWORD_FOR }, 58 { "func", KEYWORD_FUNC }, 59 { "go", KEYWORD_GO }, 60 { "goto", KEYWORD_GOTO }, 61 { "if", KEYWORD_IF }, 62 { "import", KEYWORD_IMPORT }, 63 { "interface", KEYWORD_INTERFACE }, 64 { "map", KEYWORD_MAP }, 65 { "package", KEYWORD_PACKAGE }, 66 { "range", KEYWORD_RANGE }, 67 { "return", KEYWORD_RETURN }, 68 { "select", KEYWORD_SELECT }, 69 { "struct", KEYWORD_STRUCT }, 70 { "switch", KEYWORD_SWITCH }, 71 { "type", KEYWORD_TYPE }, 72 { "var", KEYWORD_VAR } 73}; 74 75// Number of entries in the map. 76 77const int Keywords::count_ = 78 sizeof(Keywords::mapping_) / sizeof(Keywords::mapping_[0]); 79 80// Comparison function passed to bsearch. 81 82extern "C" 83{ 84 85struct Keywords_search_key 86{ 87 const char* str; 88 size_t len; 89}; 90 91static int 92keyword_compare(const void* keyv, const void* mapv) 93{ 94 const Keywords_search_key* key = 95 static_cast<const Keywords_search_key*>(keyv); 96 const Keywords::Mapping* map = 97 static_cast<const Keywords::Mapping*>(mapv); 98 if (map->keystring == NULL) 99 return 1; 100 int i = strncmp(key->str, map->keystring, key->len); 101 if (i != 0) 102 return i; 103 if (map->keystring[key->len] != '\0') 104 return -1; 105 return 0; 106} 107 108} // End extern "C". 109 110// Convert a string to a keyword code. Return KEYWORD_INVALID if the 111// string is not a keyword. 112 113Keyword 114Keywords::keyword_to_code(const char* keyword, size_t len) const 115{ 116 Keywords_search_key key; 117 key.str = keyword; 118 key.len = len; 119 void* mapv = bsearch(&key, 120 this->mapping_, 121 this->count_, 122 sizeof(this->mapping_[0]), 123 keyword_compare); 124 if (mapv == NULL) 125 return KEYWORD_INVALID; 126 Mapping* map = static_cast<Mapping*>(mapv); 127 return map->keycode; 128} 129 130// Convert a keyword code to a string. 131 132const char* 133Keywords::keyword_to_string(Keyword code) const 134{ 135 go_assert(code > KEYWORD_INVALID && code < this->count_); 136 const Mapping* map = &this->mapping_[code]; 137 go_assert(map->keycode == code); 138 return map->keystring; 139} 140 141// There is one instance of the Keywords class. 142 143static Keywords keywords; 144 145// Class Token. 146 147// Make a general token. 148 149Token::Token(Classification classification, Location location) 150 : classification_(classification), location_(location) 151{ 152} 153 154// Destroy a token. 155 156Token::~Token() 157{ 158 this->clear(); 159} 160 161// Clear a token--release memory. 162 163void 164Token::clear() 165{ 166 if (this->classification_ == TOKEN_INTEGER 167 || this->classification_ == TOKEN_CHARACTER) 168 mpz_clear(this->u_.integer_value); 169 else if (this->classification_ == TOKEN_FLOAT 170 || this->classification_ == TOKEN_IMAGINARY) 171 mpfr_clear(this->u_.float_value); 172} 173 174// Construct a token. 175 176Token::Token(const Token& tok) 177 : classification_(tok.classification_), location_(tok.location_) 178{ 179 switch (this->classification_) 180 { 181 case TOKEN_INVALID: 182 case TOKEN_EOF: 183 break; 184 case TOKEN_KEYWORD: 185 this->u_.keyword = tok.u_.keyword; 186 break; 187 case TOKEN_IDENTIFIER: 188 case TOKEN_STRING: 189 this->u_.string_value = tok.u_.string_value; 190 break; 191 case TOKEN_OPERATOR: 192 this->u_.op = tok.u_.op; 193 break; 194 case TOKEN_CHARACTER: 195 case TOKEN_INTEGER: 196 mpz_init_set(this->u_.integer_value, tok.u_.integer_value); 197 break; 198 case TOKEN_FLOAT: 199 case TOKEN_IMAGINARY: 200 mpfr_init_set(this->u_.float_value, tok.u_.float_value, GMP_RNDN); 201 break; 202 default: 203 go_unreachable(); 204 } 205} 206 207// Assign to a token. 208 209Token& 210Token::operator=(const Token& tok) 211{ 212 this->clear(); 213 this->classification_ = tok.classification_; 214 this->location_ = tok.location_; 215 switch (tok.classification_) 216 { 217 case TOKEN_INVALID: 218 case TOKEN_EOF: 219 break; 220 case TOKEN_KEYWORD: 221 this->u_.keyword = tok.u_.keyword; 222 break; 223 case TOKEN_IDENTIFIER: 224 this->u_.identifier_value.name = tok.u_.identifier_value.name; 225 this->u_.identifier_value.is_exported = 226 tok.u_.identifier_value.is_exported; 227 break; 228 case TOKEN_STRING: 229 this->u_.string_value = tok.u_.string_value; 230 break; 231 case TOKEN_OPERATOR: 232 this->u_.op = tok.u_.op; 233 break; 234 case TOKEN_CHARACTER: 235 case TOKEN_INTEGER: 236 mpz_init_set(this->u_.integer_value, tok.u_.integer_value); 237 break; 238 case TOKEN_FLOAT: 239 case TOKEN_IMAGINARY: 240 mpfr_init_set(this->u_.float_value, tok.u_.float_value, GMP_RNDN); 241 break; 242 default: 243 go_unreachable(); 244 } 245 return *this; 246} 247 248// Print the token for debugging. 249 250void 251Token::print(FILE* file) const 252{ 253 switch (this->classification_) 254 { 255 case TOKEN_INVALID: 256 fprintf(file, "invalid"); 257 break; 258 case TOKEN_EOF: 259 fprintf(file, "EOF"); 260 break; 261 case TOKEN_KEYWORD: 262 fprintf(file, "keyword %s", keywords.keyword_to_string(this->u_.keyword)); 263 break; 264 case TOKEN_IDENTIFIER: 265 fprintf(file, "identifier \"%s\"", this->u_.string_value->c_str()); 266 break; 267 case TOKEN_STRING: 268 fprintf(file, "quoted string \"%s\"", this->u_.string_value->c_str()); 269 break; 270 case TOKEN_CHARACTER: 271 fprintf(file, "character "); 272 mpz_out_str(file, 10, this->u_.integer_value); 273 break; 274 case TOKEN_INTEGER: 275 fprintf(file, "integer "); 276 mpz_out_str(file, 10, this->u_.integer_value); 277 break; 278 case TOKEN_FLOAT: 279 fprintf(file, "float "); 280 mpfr_out_str(file, 10, 0, this->u_.float_value, GMP_RNDN); 281 break; 282 case TOKEN_IMAGINARY: 283 fprintf(file, "imaginary "); 284 mpfr_out_str(file, 10, 0, this->u_.float_value, GMP_RNDN); 285 break; 286 case TOKEN_OPERATOR: 287 fprintf(file, "operator "); 288 switch (this->u_.op) 289 { 290 case OPERATOR_INVALID: 291 fprintf(file, "invalid"); 292 break; 293 case OPERATOR_OROR: 294 fprintf(file, "||"); 295 break; 296 case OPERATOR_ANDAND: 297 fprintf(file, "&&"); 298 break; 299 case OPERATOR_EQEQ: 300 fprintf(file, "=="); 301 break; 302 case OPERATOR_NOTEQ: 303 fprintf(file, "!="); 304 break; 305 case OPERATOR_LT: 306 fprintf(file, "<"); 307 break; 308 case OPERATOR_LE: 309 fprintf(file, "<="); 310 break; 311 case OPERATOR_GT: 312 fprintf(file, ">"); 313 break; 314 case OPERATOR_GE: 315 fprintf(file, ">="); 316 break; 317 case OPERATOR_PLUS: 318 fprintf(file, "+"); 319 break; 320 case OPERATOR_MINUS: 321 fprintf(file, "-"); 322 break; 323 case OPERATOR_OR: 324 fprintf(file, "|"); 325 break; 326 case OPERATOR_XOR: 327 fprintf(file, "^"); 328 break; 329 case OPERATOR_MULT: 330 fprintf(file, "*"); 331 break; 332 case OPERATOR_DIV: 333 fprintf(file, "/"); 334 break; 335 case OPERATOR_MOD: 336 fprintf(file, "%%"); 337 break; 338 case OPERATOR_LSHIFT: 339 fprintf(file, "<<"); 340 break; 341 case OPERATOR_RSHIFT: 342 fprintf(file, ">>"); 343 break; 344 case OPERATOR_AND: 345 fprintf(file, "&"); 346 break; 347 case OPERATOR_BITCLEAR: 348 fprintf(file, "&^"); 349 break; 350 case OPERATOR_NOT: 351 fprintf(file, "!"); 352 break; 353 case OPERATOR_CHANOP: 354 fprintf(file, "<-"); 355 break; 356 case OPERATOR_EQ: 357 fprintf(file, "="); 358 break; 359 case OPERATOR_PLUSEQ: 360 fprintf(file, "+="); 361 break; 362 case OPERATOR_MINUSEQ: 363 fprintf(file, "-="); 364 break; 365 case OPERATOR_OREQ: 366 fprintf(file, "|="); 367 break; 368 case OPERATOR_XOREQ: 369 fprintf(file, "^="); 370 break; 371 case OPERATOR_MULTEQ: 372 fprintf(file, "*="); 373 break; 374 case OPERATOR_DIVEQ: 375 fprintf(file, "/="); 376 break; 377 case OPERATOR_MODEQ: 378 fprintf(file, "%%="); 379 break; 380 case OPERATOR_LSHIFTEQ: 381 fprintf(file, "<<="); 382 break; 383 case OPERATOR_RSHIFTEQ: 384 fprintf(file, ">>="); 385 break; 386 case OPERATOR_ANDEQ: 387 fprintf(file, "&="); 388 break; 389 case OPERATOR_BITCLEAREQ: 390 fprintf(file, "&^="); 391 break; 392 case OPERATOR_PLUSPLUS: 393 fprintf(file, "++"); 394 break; 395 case OPERATOR_MINUSMINUS: 396 fprintf(file, "--"); 397 break; 398 case OPERATOR_COLON: 399 fprintf(file, ":"); 400 break; 401 case OPERATOR_COLONEQ: 402 fprintf(file, ":="); 403 break; 404 case OPERATOR_SEMICOLON: 405 fprintf(file, ";"); 406 break; 407 case OPERATOR_DOT: 408 fprintf(file, "."); 409 break; 410 case OPERATOR_COMMA: 411 fprintf(file, ","); 412 break; 413 case OPERATOR_LPAREN: 414 fprintf(file, "("); 415 break; 416 case OPERATOR_RPAREN: 417 fprintf(file, ")"); 418 break; 419 case OPERATOR_LCURLY: 420 fprintf(file, "{"); 421 break; 422 case OPERATOR_RCURLY: 423 fprintf(file, "}"); 424 break; 425 case OPERATOR_LSQUARE: 426 fprintf(file, "["); 427 break; 428 case OPERATOR_RSQUARE: 429 fprintf(file, "]"); 430 break; 431 default: 432 go_unreachable(); 433 } 434 break; 435 default: 436 go_unreachable(); 437 } 438} 439 440// Class Lex. 441 442Lex::Lex(const char* input_file_name, FILE* input_file, Linemap* linemap) 443 : input_file_name_(input_file_name), input_file_(input_file), 444 linemap_(linemap), linebuf_(NULL), linebufsize_(120), linesize_(0), 445 lineoff_(0), lineno_(0), add_semi_at_eol_(false), saw_nointerface_(false), 446 extern_() 447{ 448 this->linebuf_ = new char[this->linebufsize_]; 449 this->linemap_->start_file(input_file_name, 0); 450} 451 452Lex::~Lex() 453{ 454 delete[] this->linebuf_; 455} 456 457// Read a new line from the file. 458 459ssize_t 460Lex::get_line() 461{ 462 char* buf = this->linebuf_; 463 size_t size = this->linebufsize_; 464 465 FILE* file = this->input_file_; 466 size_t cur = 0; 467 while (true) 468 { 469 int c = getc(file); 470 if (c == EOF) 471 { 472 if (cur == 0) 473 return -1; 474 break; 475 } 476 if (cur + 1 >= size) 477 { 478 size_t ns = 2 * size + 1; 479 if (ns < size || static_cast<ssize_t>(ns) < 0) 480 error_at(this->location(), "out of memory"); 481 char* nb = new char[ns]; 482 memcpy(nb, buf, cur); 483 delete[] buf; 484 buf = nb; 485 size = ns; 486 } 487 buf[cur] = c; 488 ++cur; 489 490 if (c == '\n') 491 break; 492 } 493 494 buf[cur] = '\0'; 495 496 this->linebuf_ = buf; 497 this->linebufsize_ = size; 498 499 return cur; 500} 501 502// See if we need to read a new line. Return true if there is a new 503// line, false if we are at EOF. 504 505bool 506Lex::require_line() 507{ 508 if (this->lineoff_ < this->linesize_) 509 return true; 510 511 ssize_t got = this->get_line(); 512 if (got < 0) 513 return false; 514 ++this->lineno_; 515 this->linesize_= got; 516 this->lineoff_ = 0; 517 518 this->linemap_->start_line(this->lineno_, this->linesize_); 519 520 return true; 521} 522 523// Get the current location. 524 525Location 526Lex::location() const 527{ 528 return this->linemap_->get_location(this->lineoff_ + 1); 529} 530 531// Get a location slightly before the current one. This is used for 532// slightly more efficient handling of operator tokens. 533 534Location 535Lex::earlier_location(int chars) const 536{ 537 return this->linemap_->get_location(this->lineoff_ + 1 - chars); 538} 539 540// Get the next token. 541 542Token 543Lex::next_token() 544{ 545 bool saw_cpp_comment = false; 546 while (true) 547 { 548 if (!this->require_line()) 549 { 550 bool add_semi_at_eol = this->add_semi_at_eol_; 551 this->add_semi_at_eol_ = false; 552 if (add_semi_at_eol) 553 return this->make_operator(OPERATOR_SEMICOLON, 1); 554 return this->make_eof_token(); 555 } 556 557 if (!saw_cpp_comment) 558 this->extern_.clear(); 559 saw_cpp_comment = false; 560 561 const char* p = this->linebuf_ + this->lineoff_; 562 const char* pend = this->linebuf_ + this->linesize_; 563 564 while (p < pend) 565 { 566 unsigned char cc = *p; 567 switch (cc) 568 { 569 case ' ': case '\t': case '\r': 570 ++p; 571 // Skip whitespace quickly. 572 while (*p == ' ' || *p == '\t' || *p == '\r') 573 ++p; 574 break; 575 576 case '\n': 577 { 578 ++p; 579 bool add_semi_at_eol = this->add_semi_at_eol_; 580 this->add_semi_at_eol_ = false; 581 if (add_semi_at_eol) 582 { 583 this->lineoff_ = p - this->linebuf_; 584 return this->make_operator(OPERATOR_SEMICOLON, 1); 585 } 586 } 587 break; 588 589 case '/': 590 if (p[1] == '/') 591 { 592 this->lineoff_ = p + 2 - this->linebuf_; 593 this->skip_cpp_comment(); 594 p = pend; 595 if (p[-1] == '\n' && this->add_semi_at_eol_) 596 --p; 597 saw_cpp_comment = true; 598 } 599 else if (p[1] == '*') 600 { 601 this->lineoff_ = p + 2 - this->linebuf_; 602 Location location = this->location(); 603 if (!this->skip_c_comment()) 604 return Token::make_invalid_token(location); 605 p = this->linebuf_ + this->lineoff_; 606 pend = this->linebuf_ + this->linesize_; 607 } 608 else if (p[1] == '=') 609 { 610 this->add_semi_at_eol_ = false; 611 this->lineoff_ = p + 2 - this->linebuf_; 612 return this->make_operator(OPERATOR_DIVEQ, 2); 613 } 614 else 615 { 616 this->add_semi_at_eol_ = false; 617 this->lineoff_ = p + 1 - this->linebuf_; 618 return this->make_operator(OPERATOR_DIV, 1); 619 } 620 break; 621 622 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 623 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 624 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 625 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 626 case 'Y': case 'Z': 627 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 628 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 629 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 630 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 631 case 'y': case 'z': 632 case '_': 633 this->lineoff_ = p - this->linebuf_; 634 return this->gather_identifier(); 635 636 case '0': case '1': case '2': case '3': case '4': 637 case '5': case '6': case '7': case '8': case '9': 638 this->add_semi_at_eol_ = true; 639 this->lineoff_ = p - this->linebuf_; 640 return this->gather_number(); 641 642 case '\'': 643 this->add_semi_at_eol_ = true; 644 this->lineoff_ = p - this->linebuf_; 645 return this->gather_character(); 646 647 case '"': 648 this->add_semi_at_eol_ = true; 649 this->lineoff_ = p - this->linebuf_; 650 return this->gather_string(); 651 652 case '`': 653 this->add_semi_at_eol_ = true; 654 this->lineoff_ = p - this->linebuf_; 655 return this->gather_raw_string(); 656 657 case '<': 658 case '>': 659 case '&': 660 if (p + 2 < pend) 661 { 662 this->add_semi_at_eol_ = false; 663 Operator op = this->three_character_operator(cc, p[1], p[2]); 664 if (op != OPERATOR_INVALID) 665 { 666 this->lineoff_ = p + 3 - this->linebuf_; 667 return this->make_operator(op, 3); 668 } 669 } 670 // Fall through. 671 case '|': 672 case '=': 673 case '!': 674 case '+': 675 case '-': 676 case '^': 677 case '*': 678 // '/' handled above. 679 case '%': 680 case ':': 681 case ';': 682 case ',': 683 case '(': case ')': 684 case '{': case '}': 685 case '[': case ']': 686 { 687 this->add_semi_at_eol_ = false; 688 Operator op = this->two_character_operator(cc, p[1]); 689 int chars; 690 if (op != OPERATOR_INVALID) 691 { 692 ++p; 693 chars = 2; 694 } 695 else 696 { 697 op = this->one_character_operator(cc); 698 chars = 1; 699 } 700 this->lineoff_ = p + 1 - this->linebuf_; 701 return this->make_operator(op, chars); 702 } 703 704 case '.': 705 if (p[1] >= '0' && p[1] <= '9') 706 { 707 this->add_semi_at_eol_ = true; 708 this->lineoff_ = p - this->linebuf_; 709 return this->gather_number(); 710 } 711 if (p[1] == '.' && p[2] == '.') 712 { 713 this->add_semi_at_eol_ = false; 714 this->lineoff_ = p + 3 - this->linebuf_; 715 return this->make_operator(OPERATOR_ELLIPSIS, 3); 716 } 717 this->add_semi_at_eol_ = false; 718 this->lineoff_ = p + 1 - this->linebuf_; 719 return this->make_operator(OPERATOR_DOT, 1); 720 721 default: 722 { 723 unsigned int ci; 724 bool issued_error; 725 this->lineoff_ = p - this->linebuf_; 726 const char *pnext = this->advance_one_utf8_char(p, &ci, 727 &issued_error); 728 729 // Ignore byte order mark at start of file. 730 if (ci == 0xfeff) 731 { 732 p = pnext; 733 break; 734 } 735 736 if (Lex::is_unicode_letter(ci)) 737 return this->gather_identifier(); 738 739 if (!issued_error) 740 error_at(this->location(), 741 "invalid character 0x%x in input file", 742 ci); 743 744 p = pend; 745 746 break; 747 } 748 } 749 } 750 751 this->lineoff_ = p - this->linebuf_; 752 } 753} 754 755// Fetch one UTF-8 character from a string. Set *VALUE to the value. 756// Return the number of bytes read from the string. Returns 0 if the 757// string does not point to a valid UTF-8 character. 758 759int 760Lex::fetch_char(const char* p, unsigned int* value) 761{ 762 unsigned char c = *p; 763 if (c <= 0x7f) 764 { 765 *value = c; 766 return 1; 767 } 768 else if ((c & 0xe0) == 0xc0 769 && (p[1] & 0xc0) == 0x80) 770 { 771 *value = (((c & 0x1f) << 6) 772 + (p[1] & 0x3f)); 773 if (*value <= 0x7f) 774 { 775 *value = 0xfffd; 776 return 0; 777 } 778 return 2; 779 } 780 else if ((c & 0xf0) == 0xe0 781 && (p[1] & 0xc0) == 0x80 782 && (p[2] & 0xc0) == 0x80) 783 { 784 *value = (((c & 0xf) << 12) 785 + ((p[1] & 0x3f) << 6) 786 + (p[2] & 0x3f)); 787 if (*value <= 0x7ff) 788 { 789 *value = 0xfffd; 790 return 0; 791 } 792 return 3; 793 } 794 else if ((c & 0xf8) == 0xf0 795 && (p[1] & 0xc0) == 0x80 796 && (p[2] & 0xc0) == 0x80 797 && (p[3] & 0xc0) == 0x80) 798 { 799 *value = (((c & 0x7) << 18) 800 + ((p[1] & 0x3f) << 12) 801 + ((p[2] & 0x3f) << 6) 802 + (p[3] & 0x3f)); 803 if (*value <= 0xffff) 804 { 805 *value = 0xfffd; 806 return 0; 807 } 808 return 4; 809 } 810 else 811 { 812 /* Invalid encoding. Return the Unicode replacement 813 character. */ 814 *value = 0xfffd; 815 return 0; 816 } 817} 818 819// Advance one UTF-8 character. Return the pointer beyond the 820// character. Set *VALUE to the value. Set *ISSUED_ERROR if an error 821// was issued. 822 823const char* 824Lex::advance_one_utf8_char(const char* p, unsigned int* value, 825 bool* issued_error) 826{ 827 *issued_error = false; 828 829 if (*p == '\0') 830 { 831 error_at(this->location(), "invalid NUL byte"); 832 *issued_error = true; 833 *value = 0; 834 return p + 1; 835 } 836 837 int adv = Lex::fetch_char(p, value); 838 if (adv == 0) 839 { 840 error_at(this->location(), "invalid UTF-8 encoding"); 841 *issued_error = true; 842 return p + 1; 843 } 844 845 // Warn about byte order mark, except at start of file. 846 if (*value == 0xfeff && (this->lineno_ != 1 || this->lineoff_ != 0)) 847 { 848 error_at(this->location(), "Unicode (UTF-8) BOM in middle of file"); 849 *issued_error = true; 850 } 851 852 return p + adv; 853} 854 855// Pick up an identifier. 856 857Token 858Lex::gather_identifier() 859{ 860 const char* pstart = this->linebuf_ + this->lineoff_; 861 const char* p = pstart; 862 const char* pend = this->linebuf_ + this->linesize_; 863 bool is_first = true; 864 bool is_exported = false; 865 bool has_non_ascii_char = false; 866 std::string buf; 867 while (p < pend) 868 { 869 unsigned char cc = *p; 870 if (cc <= 0x7f) 871 { 872 if ((cc < 'A' || cc > 'Z') 873 && (cc < 'a' || cc > 'z') 874 && cc != '_' 875 && (cc < '0' || cc > '9')) 876 { 877 // Check for an invalid character here, as we get better 878 // error behaviour if we swallow them as part of the 879 // identifier we are building. 880 if ((cc >= ' ' && cc < 0x7f) 881 || cc == '\t' 882 || cc == '\r' 883 || cc == '\n') 884 break; 885 886 this->lineoff_ = p - this->linebuf_; 887 error_at(this->location(), 888 "invalid character 0x%x in identifier", 889 cc); 890 if (!has_non_ascii_char) 891 { 892 buf.assign(pstart, p - pstart); 893 has_non_ascii_char = true; 894 } 895 if (!Lex::is_invalid_identifier(buf)) 896 buf.append("$INVALID$"); 897 } 898 ++p; 899 if (is_first) 900 { 901 is_exported = cc >= 'A' && cc <= 'Z'; 902 is_first = false; 903 } 904 if (has_non_ascii_char) 905 buf.push_back(cc); 906 } 907 else 908 { 909 unsigned int ci; 910 bool issued_error; 911 this->lineoff_ = p - this->linebuf_; 912 const char* pnext = this->advance_one_utf8_char(p, &ci, 913 &issued_error); 914 bool is_invalid = false; 915 if (!Lex::is_unicode_letter(ci) && !Lex::is_unicode_digit(ci)) 916 { 917 // There is no valid place for a non-ASCII character 918 // other than an identifier, so we get better error 919 // handling behaviour if we swallow this character after 920 // giving an error. 921 if (!issued_error) 922 error_at(this->location(), 923 "invalid character 0x%x in identifier", 924 ci); 925 is_invalid = true; 926 } 927 if (is_first) 928 { 929 is_exported = Lex::is_unicode_uppercase(ci); 930 is_first = false; 931 } 932 if (!has_non_ascii_char) 933 { 934 buf.assign(pstart, p - pstart); 935 has_non_ascii_char = true; 936 } 937 if (is_invalid && !Lex::is_invalid_identifier(buf)) 938 buf.append("$INVALID$"); 939 buf.append(p, pnext - p); 940 p = pnext; 941 } 942 } 943 Location location = this->location(); 944 this->add_semi_at_eol_ = true; 945 this->lineoff_ = p - this->linebuf_; 946 if (has_non_ascii_char) 947 return Token::make_identifier_token(buf, is_exported, location); 948 else 949 { 950 Keyword code = keywords.keyword_to_code(pstart, p - pstart); 951 if (code == KEYWORD_INVALID) 952 return Token::make_identifier_token(std::string(pstart, p - pstart), 953 is_exported, location); 954 else 955 { 956 switch (code) 957 { 958 case KEYWORD_BREAK: 959 case KEYWORD_CONTINUE: 960 case KEYWORD_FALLTHROUGH: 961 case KEYWORD_RETURN: 962 break; 963 default: 964 this->add_semi_at_eol_ = false; 965 break; 966 } 967 return Token::make_keyword_token(code, location); 968 } 969 } 970} 971 972// Return whether C is a hex digit. 973 974bool 975Lex::is_hex_digit(char c) 976{ 977 return ((c >= '0' && c <= '9') 978 || (c >= 'A' && c <= 'F') 979 || (c >= 'a' && c <= 'f')); 980} 981 982// Return whether an exponent could start at P. 983 984bool 985Lex::could_be_exponent(const char* p, const char* pend) 986{ 987 if (*p != 'e' && *p != 'E') 988 return false; 989 ++p; 990 if (p >= pend) 991 return false; 992 if (*p == '+' || *p == '-') 993 { 994 ++p; 995 if (p >= pend) 996 return false; 997 } 998 return *p >= '0' && *p <= '9'; 999} 1000 1001// Pick up a number. 1002 1003Token 1004Lex::gather_number() 1005{ 1006 const char* pstart = this->linebuf_ + this->lineoff_; 1007 const char* p = pstart; 1008 const char* pend = this->linebuf_ + this->linesize_; 1009 1010 Location location = this->location(); 1011 1012 bool neg = false; 1013 if (*p == '+') 1014 ++p; 1015 else if (*p == '-') 1016 { 1017 ++p; 1018 neg = true; 1019 } 1020 1021 const char* pnum = p; 1022 if (*p == '0') 1023 { 1024 int base; 1025 if ((p[1] == 'x' || p[1] == 'X') 1026 && Lex::is_hex_digit(p[2])) 1027 { 1028 base = 16; 1029 p += 2; 1030 pnum = p; 1031 while (p < pend) 1032 { 1033 if (!Lex::is_hex_digit(*p)) 1034 break; 1035 ++p; 1036 } 1037 } 1038 else 1039 { 1040 base = 8; 1041 pnum = p; 1042 while (p < pend) 1043 { 1044 if (*p < '0' || *p > '7') 1045 break; 1046 ++p; 1047 } 1048 } 1049 1050 // A partial token that looks like an octal literal might actually be the 1051 // beginning of a floating-point or imaginary literal. 1052 if (base == 16 || (*p != '.' && *p != 'i' && !Lex::could_be_exponent(p, pend))) 1053 { 1054 std::string s(pnum, p - pnum); 1055 mpz_t val; 1056 int r = mpz_init_set_str(val, s.c_str(), base); 1057 go_assert(r == 0); 1058 1059 if (neg) 1060 mpz_neg(val, val); 1061 1062 this->lineoff_ = p - this->linebuf_; 1063 Token ret = Token::make_integer_token(val, location); 1064 mpz_clear(val); 1065 return ret; 1066 } 1067 } 1068 1069 while (p < pend) 1070 { 1071 if (*p < '0' || *p > '9') 1072 break; 1073 ++p; 1074 } 1075 1076 if (*p != '.' && *p != 'i' && !Lex::could_be_exponent(p, pend)) 1077 { 1078 std::string s(pnum, p - pnum); 1079 mpz_t val; 1080 int r = mpz_init_set_str(val, s.c_str(), 10); 1081 go_assert(r == 0); 1082 1083 if (neg) 1084 mpz_neg(val, val); 1085 1086 this->lineoff_ = p - this->linebuf_; 1087 Token ret = Token::make_integer_token(val, location); 1088 mpz_clear(val); 1089 return ret; 1090 } 1091 1092 if (*p != 'i') 1093 { 1094 bool dot = *p == '.'; 1095 1096 ++p; 1097 1098 if (!dot) 1099 { 1100 if (*p == '+' || *p == '-') 1101 ++p; 1102 } 1103 1104 while (p < pend) 1105 { 1106 if (*p < '0' || *p > '9') 1107 break; 1108 ++p; 1109 } 1110 1111 if (dot && Lex::could_be_exponent(p, pend)) 1112 { 1113 ++p; 1114 if (*p == '+' || *p == '-') 1115 ++p; 1116 while (p < pend) 1117 { 1118 if (*p < '0' || *p > '9') 1119 break; 1120 ++p; 1121 } 1122 } 1123 } 1124 1125 std::string s(pnum, p - pnum); 1126 mpfr_t val; 1127 int r = mpfr_init_set_str(val, s.c_str(), 10, GMP_RNDN); 1128 go_assert(r == 0); 1129 1130 if (neg) 1131 mpfr_neg(val, val, GMP_RNDN); 1132 1133 bool is_imaginary = *p == 'i'; 1134 if (is_imaginary) 1135 ++p; 1136 1137 this->lineoff_ = p - this->linebuf_; 1138 if (is_imaginary) 1139 { 1140 Token ret = Token::make_imaginary_token(val, location); 1141 mpfr_clear(val); 1142 return ret; 1143 } 1144 else 1145 { 1146 Token ret = Token::make_float_token(val, location); 1147 mpfr_clear(val); 1148 return ret; 1149 } 1150} 1151 1152// Advance one character, possibly escaped. Return the pointer beyond 1153// the character. Set *VALUE to the character. Set *IS_CHARACTER if 1154// this is a character (e.g., 'a' or '\u1234') rather than a byte 1155// value (e.g., '\001'). 1156 1157const char* 1158Lex::advance_one_char(const char* p, bool is_single_quote, unsigned int* value, 1159 bool* is_character) 1160{ 1161 *value = 0; 1162 *is_character = true; 1163 if (*p != '\\') 1164 { 1165 bool issued_error; 1166 const char* ret = this->advance_one_utf8_char(p, value, &issued_error); 1167 if (is_single_quote 1168 && (*value == '\'' || *value == '\n') 1169 && !issued_error) 1170 error_at(this->location(), "invalid character literal"); 1171 return ret; 1172 } 1173 else 1174 { 1175 ++p; 1176 switch (*p) 1177 { 1178 case '0': case '1': case '2': case '3': 1179 case '4': case '5': case '6': case '7': 1180 *is_character = false; 1181 if (p[1] >= '0' && p[1] <= '7' 1182 && p[2] >= '0' && p[2] <= '7') 1183 { 1184 *value = ((Lex::octal_value(p[0]) << 6) 1185 + (Lex::octal_value(p[1]) << 3) 1186 + Lex::octal_value(p[2])); 1187 if (*value > 255) 1188 { 1189 error_at(this->location(), "invalid octal constant"); 1190 *value = 255; 1191 } 1192 return p + 3; 1193 } 1194 error_at(this->location(), "invalid octal character"); 1195 return (p[1] >= '0' && p[1] <= '7' 1196 ? p + 2 1197 : p + 1); 1198 1199 case 'x': 1200 case 'X': 1201 *is_character = false; 1202 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])) 1203 { 1204 *value = (hex_value(p[1]) << 4) + hex_value(p[2]); 1205 return p + 3; 1206 } 1207 error_at(this->location(), "invalid hex character"); 1208 return (Lex::is_hex_digit(p[1]) 1209 ? p + 2 1210 : p + 1); 1211 1212 case 'a': 1213 *value = '\a'; 1214 return p + 1; 1215 case 'b': 1216 *value = '\b'; 1217 return p + 1; 1218 case 'f': 1219 *value = '\f'; 1220 return p + 1; 1221 case 'n': 1222 *value = '\n'; 1223 return p + 1; 1224 case 'r': 1225 *value = '\r'; 1226 return p + 1; 1227 case 't': 1228 *value = '\t'; 1229 return p + 1; 1230 case 'v': 1231 *value = '\v'; 1232 return p + 1; 1233 case '\\': 1234 *value = '\\'; 1235 return p + 1; 1236 case '\'': 1237 if (!is_single_quote) 1238 error_at(this->location(), "invalid quoted character"); 1239 *value = '\''; 1240 return p + 1; 1241 case '"': 1242 if (is_single_quote) 1243 error_at(this->location(), "invalid quoted character"); 1244 *value = '"'; 1245 return p + 1; 1246 1247 case 'u': 1248 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]) 1249 && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4])) 1250 { 1251 *value = ((hex_value(p[1]) << 12) 1252 + (hex_value(p[2]) << 8) 1253 + (hex_value(p[3]) << 4) 1254 + hex_value(p[4])); 1255 if (*value >= 0xd800 && *value < 0xe000) 1256 { 1257 error_at(this->location(), 1258 "invalid unicode code point 0x%x", 1259 *value); 1260 // Use the replacement character. 1261 *value = 0xfffd; 1262 } 1263 return p + 5; 1264 } 1265 error_at(this->location(), "invalid little unicode code point"); 1266 return p + 1; 1267 1268 case 'U': 1269 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]) 1270 && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4]) 1271 && Lex::is_hex_digit(p[5]) && Lex::is_hex_digit(p[6]) 1272 && Lex::is_hex_digit(p[7]) && Lex::is_hex_digit(p[8])) 1273 { 1274 *value = ((hex_value(p[1]) << 28) 1275 + (hex_value(p[2]) << 24) 1276 + (hex_value(p[3]) << 20) 1277 + (hex_value(p[4]) << 16) 1278 + (hex_value(p[5]) << 12) 1279 + (hex_value(p[6]) << 8) 1280 + (hex_value(p[7]) << 4) 1281 + hex_value(p[8])); 1282 if (*value > 0x10ffff 1283 || (*value >= 0xd800 && *value < 0xe000)) 1284 { 1285 error_at(this->location(), "invalid unicode code point 0x%x", 1286 *value); 1287 // Use the replacement character. 1288 *value = 0xfffd; 1289 } 1290 return p + 9; 1291 } 1292 error_at(this->location(), "invalid big unicode code point"); 1293 return p + 1; 1294 1295 default: 1296 error_at(this->location(), "invalid character after %<\\%>"); 1297 *value = *p; 1298 return p + 1; 1299 } 1300 } 1301} 1302 1303// Append V to STR. IS_CHARACTER is true for a character which should 1304// be stored in UTF-8, false for a general byte value which should be 1305// stored directly. 1306 1307void 1308Lex::append_char(unsigned int v, bool is_character, std::string* str, 1309 Location location) 1310{ 1311 char buf[4]; 1312 size_t len; 1313 if (v <= 0x7f || !is_character) 1314 { 1315 buf[0] = v; 1316 len = 1; 1317 } 1318 else if (v <= 0x7ff) 1319 { 1320 buf[0] = 0xc0 + (v >> 6); 1321 buf[1] = 0x80 + (v & 0x3f); 1322 len = 2; 1323 } 1324 else 1325 { 1326 if (v > 0x10ffff) 1327 { 1328 warning_at(location, 0, 1329 "unicode code point 0x%x out of range in string", v); 1330 // Turn it into the "replacement character". 1331 v = 0xfffd; 1332 } 1333 if (v >= 0xd800 && v < 0xe000) 1334 { 1335 warning_at(location, 0, 1336 "unicode code point 0x%x is invalid surrogate pair", v); 1337 v = 0xfffd; 1338 } 1339 if (v <= 0xffff) 1340 { 1341 buf[0] = 0xe0 + (v >> 12); 1342 buf[1] = 0x80 + ((v >> 6) & 0x3f); 1343 buf[2] = 0x80 + (v & 0x3f); 1344 len = 3; 1345 } 1346 else 1347 { 1348 buf[0] = 0xf0 + (v >> 18); 1349 buf[1] = 0x80 + ((v >> 12) & 0x3f); 1350 buf[2] = 0x80 + ((v >> 6) & 0x3f); 1351 buf[3] = 0x80 + (v & 0x3f); 1352 len = 4; 1353 } 1354 } 1355 str->append(buf, len); 1356} 1357 1358// Pick up a character literal. 1359 1360Token 1361Lex::gather_character() 1362{ 1363 ++this->lineoff_; 1364 const char* pstart = this->linebuf_ + this->lineoff_; 1365 const char* p = pstart; 1366 1367 unsigned int value; 1368 bool is_character; 1369 p = this->advance_one_char(p, true, &value, &is_character); 1370 1371 if (*p != '\'') 1372 { 1373 error_at(this->location(), "unterminated character constant"); 1374 this->lineoff_ = p - this->linebuf_; 1375 return this->make_invalid_token(); 1376 } 1377 1378 mpz_t val; 1379 mpz_init_set_ui(val, value); 1380 1381 Location location = this->location(); 1382 this->lineoff_ = p + 1 - this->linebuf_; 1383 Token ret = Token::make_character_token(val, location); 1384 mpz_clear(val); 1385 return ret; 1386} 1387 1388// Pick up a quoted string. 1389 1390Token 1391Lex::gather_string() 1392{ 1393 const char* pstart = this->linebuf_ + this->lineoff_ + 1; 1394 const char* p = pstart; 1395 const char* pend = this->linebuf_ + this->linesize_; 1396 1397 std::string value; 1398 while (*p != '"') 1399 { 1400 Location loc = this->location(); 1401 unsigned int c; 1402 bool is_character; 1403 this->lineoff_ = p - this->linebuf_; 1404 p = this->advance_one_char(p, false, &c, &is_character); 1405 if (p >= pend) 1406 { 1407 error_at(this->location(), "unterminated string"); 1408 --p; 1409 break; 1410 } 1411 Lex::append_char(c, is_character, &value, loc); 1412 } 1413 1414 Location location = this->location(); 1415 this->lineoff_ = p + 1 - this->linebuf_; 1416 return Token::make_string_token(value, location); 1417} 1418 1419// Pick up a raw string. 1420 1421Token 1422Lex::gather_raw_string() 1423{ 1424 const char* p = this->linebuf_ + this->lineoff_ + 1; 1425 const char* pend = this->linebuf_ + this->linesize_; 1426 Location location = this->location(); 1427 1428 std::string value; 1429 while (true) 1430 { 1431 while (p < pend) 1432 { 1433 if (*p == '`') 1434 { 1435 this->lineoff_ = p + 1 - this->linebuf_; 1436 return Token::make_string_token(value, location); 1437 } 1438 Location loc = this->location(); 1439 unsigned int c; 1440 bool issued_error; 1441 this->lineoff_ = p - this->linebuf_; 1442 p = this->advance_one_utf8_char(p, &c, &issued_error); 1443 // "Carriage return characters ('\r') inside raw string literals 1444 // are discarded from the raw string value." 1445 if (c != '\r') 1446 Lex::append_char(c, true, &value, loc); 1447 } 1448 this->lineoff_ = p - this->linebuf_; 1449 if (!this->require_line()) 1450 { 1451 error_at(location, "unterminated raw string"); 1452 return Token::make_string_token(value, location); 1453 } 1454 p = this->linebuf_ + this->lineoff_; 1455 pend = this->linebuf_ + this->linesize_; 1456 } 1457} 1458 1459// If C1 C2 C3 are a three character operator, return the code. 1460 1461Operator 1462Lex::three_character_operator(char c1, char c2, char c3) 1463{ 1464 if (c3 == '=') 1465 { 1466 if (c1 == '<' && c2 == '<') 1467 return OPERATOR_LSHIFTEQ; 1468 else if (c1 == '>' && c2 == '>') 1469 return OPERATOR_RSHIFTEQ; 1470 else if (c1 == '&' && c2 == '^') 1471 return OPERATOR_BITCLEAREQ; 1472 } 1473 return OPERATOR_INVALID; 1474} 1475 1476// If C1 C2 are a two character operator, return the code. 1477 1478Operator 1479Lex::two_character_operator(char c1, char c2) 1480{ 1481 switch (c1) 1482 { 1483 case '|': 1484 if (c2 == '|') 1485 return OPERATOR_OROR; 1486 else if (c2 == '=') 1487 return OPERATOR_OREQ; 1488 break; 1489 case '&': 1490 if (c2 == '&') 1491 return OPERATOR_ANDAND; 1492 else if (c2 == '^') 1493 return OPERATOR_BITCLEAR; 1494 else if (c2 == '=') 1495 return OPERATOR_ANDEQ; 1496 break; 1497 case '^': 1498 if (c2 == '=') 1499 return OPERATOR_XOREQ; 1500 break; 1501 case '=': 1502 if (c2 == '=') 1503 return OPERATOR_EQEQ; 1504 break; 1505 case '!': 1506 if (c2 == '=') 1507 return OPERATOR_NOTEQ; 1508 break; 1509 case '<': 1510 if (c2 == '=') 1511 return OPERATOR_LE; 1512 else if (c2 == '<') 1513 return OPERATOR_LSHIFT; 1514 else if (c2 == '-') 1515 return OPERATOR_CHANOP; 1516 break; 1517 case '>': 1518 if (c2 == '=') 1519 return OPERATOR_GE; 1520 else if (c2 == '>') 1521 return OPERATOR_RSHIFT; 1522 break; 1523 case '*': 1524 if (c2 == '=') 1525 return OPERATOR_MULTEQ; 1526 break; 1527 case '/': 1528 if (c2 == '=') 1529 return OPERATOR_DIVEQ; 1530 break; 1531 case '%': 1532 if (c2 == '=') 1533 return OPERATOR_MODEQ; 1534 break; 1535 case '+': 1536 if (c2 == '+') 1537 { 1538 this->add_semi_at_eol_ = true; 1539 return OPERATOR_PLUSPLUS; 1540 } 1541 else if (c2 == '=') 1542 return OPERATOR_PLUSEQ; 1543 break; 1544 case '-': 1545 if (c2 == '-') 1546 { 1547 this->add_semi_at_eol_ = true; 1548 return OPERATOR_MINUSMINUS; 1549 } 1550 else if (c2 == '=') 1551 return OPERATOR_MINUSEQ; 1552 break; 1553 case ':': 1554 if (c2 == '=') 1555 return OPERATOR_COLONEQ; 1556 break; 1557 default: 1558 break; 1559 } 1560 return OPERATOR_INVALID; 1561} 1562 1563// If character C is an operator, return the code. 1564 1565Operator 1566Lex::one_character_operator(char c) 1567{ 1568 switch (c) 1569 { 1570 case '<': 1571 return OPERATOR_LT; 1572 case '>': 1573 return OPERATOR_GT; 1574 case '+': 1575 return OPERATOR_PLUS; 1576 case '-': 1577 return OPERATOR_MINUS; 1578 case '|': 1579 return OPERATOR_OR; 1580 case '^': 1581 return OPERATOR_XOR; 1582 case '*': 1583 return OPERATOR_MULT; 1584 case '/': 1585 return OPERATOR_DIV; 1586 case '%': 1587 return OPERATOR_MOD; 1588 case '&': 1589 return OPERATOR_AND; 1590 case '!': 1591 return OPERATOR_NOT; 1592 case '=': 1593 return OPERATOR_EQ; 1594 case ':': 1595 return OPERATOR_COLON; 1596 case ';': 1597 return OPERATOR_SEMICOLON; 1598 case '.': 1599 return OPERATOR_DOT; 1600 case ',': 1601 return OPERATOR_COMMA; 1602 case '(': 1603 return OPERATOR_LPAREN; 1604 case ')': 1605 this->add_semi_at_eol_ = true; 1606 return OPERATOR_RPAREN; 1607 case '{': 1608 return OPERATOR_LCURLY; 1609 case '}': 1610 this->add_semi_at_eol_ = true; 1611 return OPERATOR_RCURLY; 1612 case '[': 1613 return OPERATOR_LSQUARE; 1614 case ']': 1615 this->add_semi_at_eol_ = true; 1616 return OPERATOR_RSQUARE; 1617 default: 1618 return OPERATOR_INVALID; 1619 } 1620} 1621 1622// Skip a C-style comment. 1623 1624bool 1625Lex::skip_c_comment() 1626{ 1627 while (true) 1628 { 1629 if (!this->require_line()) 1630 { 1631 error_at(this->location(), "unterminated comment"); 1632 return false; 1633 } 1634 1635 const char* p = this->linebuf_ + this->lineoff_; 1636 const char* pend = this->linebuf_ + this->linesize_; 1637 1638 while (p < pend) 1639 { 1640 if (p[0] == '*' && p + 1 < pend && p[1] == '/') 1641 { 1642 this->lineoff_ = p + 2 - this->linebuf_; 1643 return true; 1644 } 1645 1646 this->lineoff_ = p - this->linebuf_; 1647 unsigned int c; 1648 bool issued_error; 1649 p = this->advance_one_utf8_char(p, &c, &issued_error); 1650 } 1651 1652 this->lineoff_ = p - this->linebuf_; 1653 } 1654} 1655 1656// Skip a C++-style comment. 1657 1658void 1659Lex::skip_cpp_comment() 1660{ 1661 // Ensure that if EXTERN_ is set, it means that we just saw a 1662 // //extern comment. 1663 this->extern_.clear(); 1664 1665 const char* p = this->linebuf_ + this->lineoff_; 1666 const char* pend = this->linebuf_ + this->linesize_; 1667 1668 // By convention, a C++ comment at the start of the line of the form 1669 // //line FILE:LINENO 1670 // is interpreted as setting the file name and line number of the 1671 // next source line. 1672 1673 if (this->lineoff_ == 2 1674 && pend - p > 5 1675 && memcmp(p, "line ", 5) == 0) 1676 { 1677 p += 5; 1678 while (p < pend && *p == ' ') 1679 ++p; 1680 const char* pcolon = static_cast<const char*>(memchr(p, ':', pend - p)); 1681 if (pcolon != NULL 1682 && pcolon[1] >= '0' 1683 && pcolon[1] <= '9') 1684 { 1685 char* plend; 1686 long lineno = strtol(pcolon + 1, &plend, 10); 1687 if (plend > pcolon + 1 1688 && (plend == pend 1689 || *plend < '0' 1690 || *plend > '9') 1691 && lineno > 0 1692 && lineno < 0x7fffffff) 1693 { 1694 unsigned int filelen = pcolon - p; 1695 char* file = new char[filelen + 1]; 1696 memcpy(file, p, filelen); 1697 file[filelen] = '\0'; 1698 1699 this->linemap_->start_file(file, lineno); 1700 this->lineno_ = lineno - 1; 1701 1702 p = plend; 1703 } 1704 } 1705 } 1706 1707 // As a special gccgo extension, a C++ comment at the start of the 1708 // line of the form 1709 // //extern NAME 1710 // which immediately precedes a function declaration means that the 1711 // external name of the function declaration is NAME. This is 1712 // normally used to permit Go code to call a C function. 1713 if (this->lineoff_ == 2 1714 && pend - p > 7 1715 && memcmp(p, "extern ", 7) == 0) 1716 { 1717 p += 7; 1718 while (p < pend && (*p == ' ' || *p == '\t')) 1719 ++p; 1720 const char* plend = pend; 1721 while (plend > p 1722 && (plend[-1] == ' ' || plend[-1] == '\t' || plend[-1] == '\n')) 1723 --plend; 1724 if (plend > p) 1725 this->extern_ = std::string(p, plend - p); 1726 } 1727 1728 // For field tracking analysis: a //go:nointerface comment means 1729 // that the next interface method should not be stored in the type 1730 // descriptor. This permits it to be discarded if it is not needed. 1731 if (this->lineoff_ == 2 && memcmp(p, "go:nointerface", 14) == 0) 1732 this->saw_nointerface_ = true; 1733 1734 while (p < pend) 1735 { 1736 this->lineoff_ = p - this->linebuf_; 1737 unsigned int c; 1738 bool issued_error; 1739 p = this->advance_one_utf8_char(p, &c, &issued_error); 1740 if (issued_error) 1741 this->extern_.clear(); 1742 } 1743} 1744 1745// The Unicode tables use this struct. 1746 1747struct Unicode_range 1748{ 1749 // The low end of the range. 1750 unsigned int low; 1751 // The high end of the range. 1752 unsigned int high; 1753 // The stride. This entries represents low, low + stride, low + 2 * 1754 // stride, etc., up to high. 1755 unsigned int stride; 1756}; 1757 1758// A table of whitespace characters--Unicode code points classified as 1759// "Space", "C" locale whitespace characters, the "next line" control 1760// character (0085), the line separator (2028), the paragraph 1761// separator (2029), and the "zero-width non-break space" (feff). 1762 1763static const Unicode_range unicode_space[] = 1764{ 1765 { 0x0009, 0x000d, 1 }, 1766 { 0x0020, 0x0020, 1 }, 1767 { 0x0085, 0x0085, 1 }, 1768 { 0x00a0, 0x00a0, 1 }, 1769 { 0x1680, 0x1680, 1 }, 1770 { 0x180e, 0x180e, 1 }, 1771 { 0x2000, 0x200a, 1 }, 1772 { 0x2028, 0x2029, 1 }, 1773 { 0x202f, 0x202f, 1 }, 1774 { 0x205f, 0x205f, 1 }, 1775 { 0x3000, 0x3000, 1 }, 1776 { 0xfeff, 0xfeff, 1 }, 1777}; 1778 1779// A table of Unicode digits--Unicode code points classified as 1780// "Digit". 1781 1782static const Unicode_range unicode_digits[] = 1783{ 1784 { 0x0030, 0x0039, 1}, 1785 { 0x0660, 0x0669, 1}, 1786 { 0x06f0, 0x06f9, 1}, 1787 { 0x07c0, 0x07c9, 1}, 1788 { 0x0966, 0x096f, 1}, 1789 { 0x09e6, 0x09ef, 1}, 1790 { 0x0a66, 0x0a6f, 1}, 1791 { 0x0ae6, 0x0aef, 1}, 1792 { 0x0b66, 0x0b6f, 1}, 1793 { 0x0be6, 0x0bef, 1}, 1794 { 0x0c66, 0x0c6f, 1}, 1795 { 0x0ce6, 0x0cef, 1}, 1796 { 0x0d66, 0x0d6f, 1}, 1797 { 0x0e50, 0x0e59, 1}, 1798 { 0x0ed0, 0x0ed9, 1}, 1799 { 0x0f20, 0x0f29, 1}, 1800 { 0x1040, 0x1049, 1}, 1801 { 0x17e0, 0x17e9, 1}, 1802 { 0x1810, 0x1819, 1}, 1803 { 0x1946, 0x194f, 1}, 1804 { 0x19d0, 0x19d9, 1}, 1805 { 0x1b50, 0x1b59, 1}, 1806 { 0xff10, 0xff19, 1}, 1807 { 0x104a0, 0x104a9, 1}, 1808 { 0x1d7ce, 0x1d7ff, 1}, 1809}; 1810 1811// A table of Unicode letters--Unicode code points classified as 1812// "Letter". 1813 1814static const Unicode_range unicode_letters[] = 1815{ 1816 { 0x0041, 0x005a, 1}, 1817 { 0x0061, 0x007a, 1}, 1818 { 0x00aa, 0x00b5, 11}, 1819 { 0x00ba, 0x00ba, 1}, 1820 { 0x00c0, 0x00d6, 1}, 1821 { 0x00d8, 0x00f6, 1}, 1822 { 0x00f8, 0x02c1, 1}, 1823 { 0x02c6, 0x02d1, 1}, 1824 { 0x02e0, 0x02e4, 1}, 1825 { 0x02ec, 0x02ee, 2}, 1826 { 0x0370, 0x0374, 1}, 1827 { 0x0376, 0x0377, 1}, 1828 { 0x037a, 0x037d, 1}, 1829 { 0x0386, 0x0386, 1}, 1830 { 0x0388, 0x038a, 1}, 1831 { 0x038c, 0x038c, 1}, 1832 { 0x038e, 0x03a1, 1}, 1833 { 0x03a3, 0x03f5, 1}, 1834 { 0x03f7, 0x0481, 1}, 1835 { 0x048a, 0x0523, 1}, 1836 { 0x0531, 0x0556, 1}, 1837 { 0x0559, 0x0559, 1}, 1838 { 0x0561, 0x0587, 1}, 1839 { 0x05d0, 0x05ea, 1}, 1840 { 0x05f0, 0x05f2, 1}, 1841 { 0x0621, 0x064a, 1}, 1842 { 0x066e, 0x066f, 1}, 1843 { 0x0671, 0x06d3, 1}, 1844 { 0x06d5, 0x06d5, 1}, 1845 { 0x06e5, 0x06e6, 1}, 1846 { 0x06ee, 0x06ef, 1}, 1847 { 0x06fa, 0x06fc, 1}, 1848 { 0x06ff, 0x0710, 17}, 1849 { 0x0712, 0x072f, 1}, 1850 { 0x074d, 0x07a5, 1}, 1851 { 0x07b1, 0x07b1, 1}, 1852 { 0x07ca, 0x07ea, 1}, 1853 { 0x07f4, 0x07f5, 1}, 1854 { 0x07fa, 0x07fa, 1}, 1855 { 0x0904, 0x0939, 1}, 1856 { 0x093d, 0x0950, 19}, 1857 { 0x0958, 0x0961, 1}, 1858 { 0x0971, 0x0972, 1}, 1859 { 0x097b, 0x097f, 1}, 1860 { 0x0985, 0x098c, 1}, 1861 { 0x098f, 0x0990, 1}, 1862 { 0x0993, 0x09a8, 1}, 1863 { 0x09aa, 0x09b0, 1}, 1864 { 0x09b2, 0x09b2, 1}, 1865 { 0x09b6, 0x09b9, 1}, 1866 { 0x09bd, 0x09ce, 17}, 1867 { 0x09dc, 0x09dd, 1}, 1868 { 0x09df, 0x09e1, 1}, 1869 { 0x09f0, 0x09f1, 1}, 1870 { 0x0a05, 0x0a0a, 1}, 1871 { 0x0a0f, 0x0a10, 1}, 1872 { 0x0a13, 0x0a28, 1}, 1873 { 0x0a2a, 0x0a30, 1}, 1874 { 0x0a32, 0x0a33, 1}, 1875 { 0x0a35, 0x0a36, 1}, 1876 { 0x0a38, 0x0a39, 1}, 1877 { 0x0a59, 0x0a5c, 1}, 1878 { 0x0a5e, 0x0a5e, 1}, 1879 { 0x0a72, 0x0a74, 1}, 1880 { 0x0a85, 0x0a8d, 1}, 1881 { 0x0a8f, 0x0a91, 1}, 1882 { 0x0a93, 0x0aa8, 1}, 1883 { 0x0aaa, 0x0ab0, 1}, 1884 { 0x0ab2, 0x0ab3, 1}, 1885 { 0x0ab5, 0x0ab9, 1}, 1886 { 0x0abd, 0x0ad0, 19}, 1887 { 0x0ae0, 0x0ae1, 1}, 1888 { 0x0b05, 0x0b0c, 1}, 1889 { 0x0b0f, 0x0b10, 1}, 1890 { 0x0b13, 0x0b28, 1}, 1891 { 0x0b2a, 0x0b30, 1}, 1892 { 0x0b32, 0x0b33, 1}, 1893 { 0x0b35, 0x0b39, 1}, 1894 { 0x0b3d, 0x0b3d, 1}, 1895 { 0x0b5c, 0x0b5d, 1}, 1896 { 0x0b5f, 0x0b61, 1}, 1897 { 0x0b71, 0x0b83, 18}, 1898 { 0x0b85, 0x0b8a, 1}, 1899 { 0x0b8e, 0x0b90, 1}, 1900 { 0x0b92, 0x0b95, 1}, 1901 { 0x0b99, 0x0b9a, 1}, 1902 { 0x0b9c, 0x0b9c, 1}, 1903 { 0x0b9e, 0x0b9f, 1}, 1904 { 0x0ba3, 0x0ba4, 1}, 1905 { 0x0ba8, 0x0baa, 1}, 1906 { 0x0bae, 0x0bb9, 1}, 1907 { 0x0bd0, 0x0bd0, 1}, 1908 { 0x0c05, 0x0c0c, 1}, 1909 { 0x0c0e, 0x0c10, 1}, 1910 { 0x0c12, 0x0c28, 1}, 1911 { 0x0c2a, 0x0c33, 1}, 1912 { 0x0c35, 0x0c39, 1}, 1913 { 0x0c3d, 0x0c3d, 1}, 1914 { 0x0c58, 0x0c59, 1}, 1915 { 0x0c60, 0x0c61, 1}, 1916 { 0x0c85, 0x0c8c, 1}, 1917 { 0x0c8e, 0x0c90, 1}, 1918 { 0x0c92, 0x0ca8, 1}, 1919 { 0x0caa, 0x0cb3, 1}, 1920 { 0x0cb5, 0x0cb9, 1}, 1921 { 0x0cbd, 0x0cde, 33}, 1922 { 0x0ce0, 0x0ce1, 1}, 1923 { 0x0d05, 0x0d0c, 1}, 1924 { 0x0d0e, 0x0d10, 1}, 1925 { 0x0d12, 0x0d28, 1}, 1926 { 0x0d2a, 0x0d39, 1}, 1927 { 0x0d3d, 0x0d3d, 1}, 1928 { 0x0d60, 0x0d61, 1}, 1929 { 0x0d7a, 0x0d7f, 1}, 1930 { 0x0d85, 0x0d96, 1}, 1931 { 0x0d9a, 0x0db1, 1}, 1932 { 0x0db3, 0x0dbb, 1}, 1933 { 0x0dbd, 0x0dbd, 1}, 1934 { 0x0dc0, 0x0dc6, 1}, 1935 { 0x0e01, 0x0e30, 1}, 1936 { 0x0e32, 0x0e33, 1}, 1937 { 0x0e40, 0x0e46, 1}, 1938 { 0x0e81, 0x0e82, 1}, 1939 { 0x0e84, 0x0e84, 1}, 1940 { 0x0e87, 0x0e88, 1}, 1941 { 0x0e8a, 0x0e8d, 3}, 1942 { 0x0e94, 0x0e97, 1}, 1943 { 0x0e99, 0x0e9f, 1}, 1944 { 0x0ea1, 0x0ea3, 1}, 1945 { 0x0ea5, 0x0ea7, 2}, 1946 { 0x0eaa, 0x0eab, 1}, 1947 { 0x0ead, 0x0eb0, 1}, 1948 { 0x0eb2, 0x0eb3, 1}, 1949 { 0x0ebd, 0x0ebd, 1}, 1950 { 0x0ec0, 0x0ec4, 1}, 1951 { 0x0ec6, 0x0ec6, 1}, 1952 { 0x0edc, 0x0edd, 1}, 1953 { 0x0f00, 0x0f00, 1}, 1954 { 0x0f40, 0x0f47, 1}, 1955 { 0x0f49, 0x0f6c, 1}, 1956 { 0x0f88, 0x0f8b, 1}, 1957 { 0x1000, 0x102a, 1}, 1958 { 0x103f, 0x103f, 1}, 1959 { 0x1050, 0x1055, 1}, 1960 { 0x105a, 0x105d, 1}, 1961 { 0x1061, 0x1061, 1}, 1962 { 0x1065, 0x1066, 1}, 1963 { 0x106e, 0x1070, 1}, 1964 { 0x1075, 0x1081, 1}, 1965 { 0x108e, 0x108e, 1}, 1966 { 0x10a0, 0x10c5, 1}, 1967 { 0x10d0, 0x10fa, 1}, 1968 { 0x10fc, 0x10fc, 1}, 1969 { 0x1100, 0x1159, 1}, 1970 { 0x115f, 0x11a2, 1}, 1971 { 0x11a8, 0x11f9, 1}, 1972 { 0x1200, 0x1248, 1}, 1973 { 0x124a, 0x124d, 1}, 1974 { 0x1250, 0x1256, 1}, 1975 { 0x1258, 0x1258, 1}, 1976 { 0x125a, 0x125d, 1}, 1977 { 0x1260, 0x1288, 1}, 1978 { 0x128a, 0x128d, 1}, 1979 { 0x1290, 0x12b0, 1}, 1980 { 0x12b2, 0x12b5, 1}, 1981 { 0x12b8, 0x12be, 1}, 1982 { 0x12c0, 0x12c0, 1}, 1983 { 0x12c2, 0x12c5, 1}, 1984 { 0x12c8, 0x12d6, 1}, 1985 { 0x12d8, 0x1310, 1}, 1986 { 0x1312, 0x1315, 1}, 1987 { 0x1318, 0x135a, 1}, 1988 { 0x1380, 0x138f, 1}, 1989 { 0x13a0, 0x13f4, 1}, 1990 { 0x1401, 0x166c, 1}, 1991 { 0x166f, 0x1676, 1}, 1992 { 0x1681, 0x169a, 1}, 1993 { 0x16a0, 0x16ea, 1}, 1994 { 0x1700, 0x170c, 1}, 1995 { 0x170e, 0x1711, 1}, 1996 { 0x1720, 0x1731, 1}, 1997 { 0x1740, 0x1751, 1}, 1998 { 0x1760, 0x176c, 1}, 1999 { 0x176e, 0x1770, 1}, 2000 { 0x1780, 0x17b3, 1}, 2001 { 0x17d7, 0x17dc, 5}, 2002 { 0x1820, 0x1877, 1}, 2003 { 0x1880, 0x18a8, 1}, 2004 { 0x18aa, 0x18aa, 1}, 2005 { 0x1900, 0x191c, 1}, 2006 { 0x1950, 0x196d, 1}, 2007 { 0x1970, 0x1974, 1}, 2008 { 0x1980, 0x19a9, 1}, 2009 { 0x19c1, 0x19c7, 1}, 2010 { 0x1a00, 0x1a16, 1}, 2011 { 0x1b05, 0x1b33, 1}, 2012 { 0x1b45, 0x1b4b, 1}, 2013 { 0x1b83, 0x1ba0, 1}, 2014 { 0x1bae, 0x1baf, 1}, 2015 { 0x1c00, 0x1c23, 1}, 2016 { 0x1c4d, 0x1c4f, 1}, 2017 { 0x1c5a, 0x1c7d, 1}, 2018 { 0x1d00, 0x1dbf, 1}, 2019 { 0x1e00, 0x1f15, 1}, 2020 { 0x1f18, 0x1f1d, 1}, 2021 { 0x1f20, 0x1f45, 1}, 2022 { 0x1f48, 0x1f4d, 1}, 2023 { 0x1f50, 0x1f57, 1}, 2024 { 0x1f59, 0x1f5d, 2}, 2025 { 0x1f5f, 0x1f7d, 1}, 2026 { 0x1f80, 0x1fb4, 1}, 2027 { 0x1fb6, 0x1fbc, 1}, 2028 { 0x1fbe, 0x1fbe, 1}, 2029 { 0x1fc2, 0x1fc4, 1}, 2030 { 0x1fc6, 0x1fcc, 1}, 2031 { 0x1fd0, 0x1fd3, 1}, 2032 { 0x1fd6, 0x1fdb, 1}, 2033 { 0x1fe0, 0x1fec, 1}, 2034 { 0x1ff2, 0x1ff4, 1}, 2035 { 0x1ff6, 0x1ffc, 1}, 2036 { 0x2071, 0x207f, 14}, 2037 { 0x2090, 0x2094, 1}, 2038 { 0x2102, 0x2107, 5}, 2039 { 0x210a, 0x2113, 1}, 2040 { 0x2115, 0x2115, 1}, 2041 { 0x2119, 0x211d, 1}, 2042 { 0x2124, 0x2128, 2}, 2043 { 0x212a, 0x212d, 1}, 2044 { 0x212f, 0x2139, 1}, 2045 { 0x213c, 0x213f, 1}, 2046 { 0x2145, 0x2149, 1}, 2047 { 0x214e, 0x214e, 1}, 2048 { 0x2183, 0x2184, 1}, 2049 { 0x2c00, 0x2c2e, 1}, 2050 { 0x2c30, 0x2c5e, 1}, 2051 { 0x2c60, 0x2c6f, 1}, 2052 { 0x2c71, 0x2c7d, 1}, 2053 { 0x2c80, 0x2ce4, 1}, 2054 { 0x2d00, 0x2d25, 1}, 2055 { 0x2d30, 0x2d65, 1}, 2056 { 0x2d6f, 0x2d6f, 1}, 2057 { 0x2d80, 0x2d96, 1}, 2058 { 0x2da0, 0x2da6, 1}, 2059 { 0x2da8, 0x2dae, 1}, 2060 { 0x2db0, 0x2db6, 1}, 2061 { 0x2db8, 0x2dbe, 1}, 2062 { 0x2dc0, 0x2dc6, 1}, 2063 { 0x2dc8, 0x2dce, 1}, 2064 { 0x2dd0, 0x2dd6, 1}, 2065 { 0x2dd8, 0x2dde, 1}, 2066 { 0x2e2f, 0x2e2f, 1}, 2067 { 0x3005, 0x3006, 1}, 2068 { 0x3031, 0x3035, 1}, 2069 { 0x303b, 0x303c, 1}, 2070 { 0x3041, 0x3096, 1}, 2071 { 0x309d, 0x309f, 1}, 2072 { 0x30a1, 0x30fa, 1}, 2073 { 0x30fc, 0x30ff, 1}, 2074 { 0x3105, 0x312d, 1}, 2075 { 0x3131, 0x318e, 1}, 2076 { 0x31a0, 0x31b7, 1}, 2077 { 0x31f0, 0x31ff, 1}, 2078 { 0x3400, 0x4db5, 1}, 2079 { 0x4e00, 0x9fc3, 1}, 2080 { 0xa000, 0xa48c, 1}, 2081 { 0xa500, 0xa60c, 1}, 2082 { 0xa610, 0xa61f, 1}, 2083 { 0xa62a, 0xa62b, 1}, 2084 { 0xa640, 0xa65f, 1}, 2085 { 0xa662, 0xa66e, 1}, 2086 { 0xa67f, 0xa697, 1}, 2087 { 0xa717, 0xa71f, 1}, 2088 { 0xa722, 0xa788, 1}, 2089 { 0xa78b, 0xa78c, 1}, 2090 { 0xa7fb, 0xa801, 1}, 2091 { 0xa803, 0xa805, 1}, 2092 { 0xa807, 0xa80a, 1}, 2093 { 0xa80c, 0xa822, 1}, 2094 { 0xa840, 0xa873, 1}, 2095 { 0xa882, 0xa8b3, 1}, 2096 { 0xa90a, 0xa925, 1}, 2097 { 0xa930, 0xa946, 1}, 2098 { 0xaa00, 0xaa28, 1}, 2099 { 0xaa40, 0xaa42, 1}, 2100 { 0xaa44, 0xaa4b, 1}, 2101 { 0xac00, 0xd7a3, 1}, 2102 { 0xf900, 0xfa2d, 1}, 2103 { 0xfa30, 0xfa6a, 1}, 2104 { 0xfa70, 0xfad9, 1}, 2105 { 0xfb00, 0xfb06, 1}, 2106 { 0xfb13, 0xfb17, 1}, 2107 { 0xfb1d, 0xfb1d, 1}, 2108 { 0xfb1f, 0xfb28, 1}, 2109 { 0xfb2a, 0xfb36, 1}, 2110 { 0xfb38, 0xfb3c, 1}, 2111 { 0xfb3e, 0xfb3e, 1}, 2112 { 0xfb40, 0xfb41, 1}, 2113 { 0xfb43, 0xfb44, 1}, 2114 { 0xfb46, 0xfbb1, 1}, 2115 { 0xfbd3, 0xfd3d, 1}, 2116 { 0xfd50, 0xfd8f, 1}, 2117 { 0xfd92, 0xfdc7, 1}, 2118 { 0xfdf0, 0xfdfb, 1}, 2119 { 0xfe70, 0xfe74, 1}, 2120 { 0xfe76, 0xfefc, 1}, 2121 { 0xff21, 0xff3a, 1}, 2122 { 0xff41, 0xff5a, 1}, 2123 { 0xff66, 0xffbe, 1}, 2124 { 0xffc2, 0xffc7, 1}, 2125 { 0xffca, 0xffcf, 1}, 2126 { 0xffd2, 0xffd7, 1}, 2127 { 0xffda, 0xffdc, 1}, 2128 { 0x10000, 0x1000b, 1}, 2129 { 0x1000d, 0x10026, 1}, 2130 { 0x10028, 0x1003a, 1}, 2131 { 0x1003c, 0x1003d, 1}, 2132 { 0x1003f, 0x1004d, 1}, 2133 { 0x10050, 0x1005d, 1}, 2134 { 0x10080, 0x100fa, 1}, 2135 { 0x10280, 0x1029c, 1}, 2136 { 0x102a0, 0x102d0, 1}, 2137 { 0x10300, 0x1031e, 1}, 2138 { 0x10330, 0x10340, 1}, 2139 { 0x10342, 0x10349, 1}, 2140 { 0x10380, 0x1039d, 1}, 2141 { 0x103a0, 0x103c3, 1}, 2142 { 0x103c8, 0x103cf, 1}, 2143 { 0x10400, 0x1049d, 1}, 2144 { 0x10800, 0x10805, 1}, 2145 { 0x10808, 0x10808, 1}, 2146 { 0x1080a, 0x10835, 1}, 2147 { 0x10837, 0x10838, 1}, 2148 { 0x1083c, 0x1083f, 3}, 2149 { 0x10900, 0x10915, 1}, 2150 { 0x10920, 0x10939, 1}, 2151 { 0x10a00, 0x10a00, 1}, 2152 { 0x10a10, 0x10a13, 1}, 2153 { 0x10a15, 0x10a17, 1}, 2154 { 0x10a19, 0x10a33, 1}, 2155 { 0x12000, 0x1236e, 1}, 2156 { 0x1d400, 0x1d454, 1}, 2157 { 0x1d456, 0x1d49c, 1}, 2158 { 0x1d49e, 0x1d49f, 1}, 2159 { 0x1d4a2, 0x1d4a2, 1}, 2160 { 0x1d4a5, 0x1d4a6, 1}, 2161 { 0x1d4a9, 0x1d4ac, 1}, 2162 { 0x1d4ae, 0x1d4b9, 1}, 2163 { 0x1d4bb, 0x1d4bb, 1}, 2164 { 0x1d4bd, 0x1d4c3, 1}, 2165 { 0x1d4c5, 0x1d505, 1}, 2166 { 0x1d507, 0x1d50a, 1}, 2167 { 0x1d50d, 0x1d514, 1}, 2168 { 0x1d516, 0x1d51c, 1}, 2169 { 0x1d51e, 0x1d539, 1}, 2170 { 0x1d53b, 0x1d53e, 1}, 2171 { 0x1d540, 0x1d544, 1}, 2172 { 0x1d546, 0x1d546, 1}, 2173 { 0x1d54a, 0x1d550, 1}, 2174 { 0x1d552, 0x1d6a5, 1}, 2175 { 0x1d6a8, 0x1d6c0, 1}, 2176 { 0x1d6c2, 0x1d6da, 1}, 2177 { 0x1d6dc, 0x1d6fa, 1}, 2178 { 0x1d6fc, 0x1d714, 1}, 2179 { 0x1d716, 0x1d734, 1}, 2180 { 0x1d736, 0x1d74e, 1}, 2181 { 0x1d750, 0x1d76e, 1}, 2182 { 0x1d770, 0x1d788, 1}, 2183 { 0x1d78a, 0x1d7a8, 1}, 2184 { 0x1d7aa, 0x1d7c2, 1}, 2185 { 0x1d7c4, 0x1d7cb, 1}, 2186 { 0x20000, 0x2a6d6, 1}, 2187 { 0x2f800, 0x2fa1d, 1}, 2188}; 2189 2190// A table of Unicode uppercase letters--Unicode code points 2191// classified as "Letter, uppercase". 2192 2193static const Unicode_range unicode_uppercase_letters[] = 2194{ 2195 { 0x0041, 0x005a, 1}, 2196 { 0x00c0, 0x00d6, 1}, 2197 { 0x00d8, 0x00de, 1}, 2198 { 0x0100, 0x0136, 2}, 2199 { 0x0139, 0x0147, 2}, 2200 { 0x014a, 0x0176, 2}, 2201 { 0x0178, 0x0179, 1}, 2202 { 0x017b, 0x017d, 2}, 2203 { 0x0181, 0x0182, 1}, 2204 { 0x0184, 0x0184, 1}, 2205 { 0x0186, 0x0187, 1}, 2206 { 0x0189, 0x018b, 1}, 2207 { 0x018e, 0x0191, 1}, 2208 { 0x0193, 0x0194, 1}, 2209 { 0x0196, 0x0198, 1}, 2210 { 0x019c, 0x019d, 1}, 2211 { 0x019f, 0x01a0, 1}, 2212 { 0x01a2, 0x01a4, 2}, 2213 { 0x01a6, 0x01a7, 1}, 2214 { 0x01a9, 0x01ac, 3}, 2215 { 0x01ae, 0x01af, 1}, 2216 { 0x01b1, 0x01b3, 1}, 2217 { 0x01b5, 0x01b5, 1}, 2218 { 0x01b7, 0x01b8, 1}, 2219 { 0x01bc, 0x01c4, 8}, 2220 { 0x01c7, 0x01cd, 3}, 2221 { 0x01cf, 0x01db, 2}, 2222 { 0x01de, 0x01ee, 2}, 2223 { 0x01f1, 0x01f4, 3}, 2224 { 0x01f6, 0x01f8, 1}, 2225 { 0x01fa, 0x0232, 2}, 2226 { 0x023a, 0x023b, 1}, 2227 { 0x023d, 0x023e, 1}, 2228 { 0x0241, 0x0241, 1}, 2229 { 0x0243, 0x0246, 1}, 2230 { 0x0248, 0x024e, 2}, 2231 { 0x0370, 0x0372, 2}, 2232 { 0x0376, 0x0386, 16}, 2233 { 0x0388, 0x038a, 1}, 2234 { 0x038c, 0x038c, 1}, 2235 { 0x038e, 0x038f, 1}, 2236 { 0x0391, 0x03a1, 1}, 2237 { 0x03a3, 0x03ab, 1}, 2238 { 0x03cf, 0x03cf, 1}, 2239 { 0x03d2, 0x03d4, 1}, 2240 { 0x03d8, 0x03ee, 2}, 2241 { 0x03f4, 0x03f7, 3}, 2242 { 0x03f9, 0x03fa, 1}, 2243 { 0x03fd, 0x042f, 1}, 2244 { 0x0460, 0x0480, 2}, 2245 { 0x048a, 0x04be, 2}, 2246 { 0x04c0, 0x04c1, 1}, 2247 { 0x04c3, 0x04cd, 2}, 2248 { 0x04d0, 0x0522, 2}, 2249 { 0x0531, 0x0556, 1}, 2250 { 0x10a0, 0x10c5, 1}, 2251 { 0x1e00, 0x1e94, 2}, 2252 { 0x1e9e, 0x1efe, 2}, 2253 { 0x1f08, 0x1f0f, 1}, 2254 { 0x1f18, 0x1f1d, 1}, 2255 { 0x1f28, 0x1f2f, 1}, 2256 { 0x1f38, 0x1f3f, 1}, 2257 { 0x1f48, 0x1f4d, 1}, 2258 { 0x1f59, 0x1f5f, 2}, 2259 { 0x1f68, 0x1f6f, 1}, 2260 { 0x1fb8, 0x1fbb, 1}, 2261 { 0x1fc8, 0x1fcb, 1}, 2262 { 0x1fd8, 0x1fdb, 1}, 2263 { 0x1fe8, 0x1fec, 1}, 2264 { 0x1ff8, 0x1ffb, 1}, 2265 { 0x2102, 0x2107, 5}, 2266 { 0x210b, 0x210d, 1}, 2267 { 0x2110, 0x2112, 1}, 2268 { 0x2115, 0x2115, 1}, 2269 { 0x2119, 0x211d, 1}, 2270 { 0x2124, 0x2128, 2}, 2271 { 0x212a, 0x212d, 1}, 2272 { 0x2130, 0x2133, 1}, 2273 { 0x213e, 0x213f, 1}, 2274 { 0x2145, 0x2183, 62}, 2275 { 0x2c00, 0x2c2e, 1}, 2276 { 0x2c60, 0x2c60, 1}, 2277 { 0x2c62, 0x2c64, 1}, 2278 { 0x2c67, 0x2c6b, 2}, 2279 { 0x2c6d, 0x2c6f, 1}, 2280 { 0x2c72, 0x2c75, 3}, 2281 { 0x2c80, 0x2ce2, 2}, 2282 { 0xa640, 0xa65e, 2}, 2283 { 0xa662, 0xa66c, 2}, 2284 { 0xa680, 0xa696, 2}, 2285 { 0xa722, 0xa72e, 2}, 2286 { 0xa732, 0xa76e, 2}, 2287 { 0xa779, 0xa77b, 2}, 2288 { 0xa77d, 0xa77e, 1}, 2289 { 0xa780, 0xa786, 2}, 2290 { 0xa78b, 0xa78b, 1}, 2291 { 0xff21, 0xff3a, 1}, 2292 { 0x10400, 0x10427, 1}, 2293 { 0x1d400, 0x1d419, 1}, 2294 { 0x1d434, 0x1d44d, 1}, 2295 { 0x1d468, 0x1d481, 1}, 2296 { 0x1d49c, 0x1d49c, 1}, 2297 { 0x1d49e, 0x1d49f, 1}, 2298 { 0x1d4a2, 0x1d4a2, 1}, 2299 { 0x1d4a5, 0x1d4a6, 1}, 2300 { 0x1d4a9, 0x1d4ac, 1}, 2301 { 0x1d4ae, 0x1d4b5, 1}, 2302 { 0x1d4d0, 0x1d4e9, 1}, 2303 { 0x1d504, 0x1d505, 1}, 2304 { 0x1d507, 0x1d50a, 1}, 2305 { 0x1d50d, 0x1d514, 1}, 2306 { 0x1d516, 0x1d51c, 1}, 2307 { 0x1d538, 0x1d539, 1}, 2308 { 0x1d53b, 0x1d53e, 1}, 2309 { 0x1d540, 0x1d544, 1}, 2310 { 0x1d546, 0x1d546, 1}, 2311 { 0x1d54a, 0x1d550, 1}, 2312 { 0x1d56c, 0x1d585, 1}, 2313 { 0x1d5a0, 0x1d5b9, 1}, 2314 { 0x1d5d4, 0x1d5ed, 1}, 2315 { 0x1d608, 0x1d621, 1}, 2316 { 0x1d63c, 0x1d655, 1}, 2317 { 0x1d670, 0x1d689, 1}, 2318 { 0x1d6a8, 0x1d6c0, 1}, 2319 { 0x1d6e2, 0x1d6fa, 1}, 2320 { 0x1d71c, 0x1d734, 1}, 2321 { 0x1d756, 0x1d76e, 1}, 2322 { 0x1d790, 0x1d7a8, 1}, 2323 { 0x1d7ca, 0x1d7ca, 1}, 2324}; 2325 2326// Return true if C is in RANGES. 2327 2328bool 2329Lex::is_in_unicode_range(unsigned int c, const Unicode_range* ranges, 2330 size_t range_size) 2331{ 2332 if (c < 0x100) 2333 { 2334 // The common case is a small value, and we know that it will be 2335 // in the first few entries of the table. Do a linear scan 2336 // rather than a binary search. 2337 for (size_t i = 0; i < range_size; ++i) 2338 { 2339 const Unicode_range* p = &ranges[i]; 2340 if (c <= p->high) 2341 { 2342 if (c < p->low) 2343 return false; 2344 return (c - p->low) % p->stride == 0; 2345 } 2346 } 2347 return false; 2348 } 2349 else 2350 { 2351 size_t lo = 0; 2352 size_t hi = range_size; 2353 while (lo < hi) 2354 { 2355 size_t mid = lo + (hi - lo) / 2; 2356 const Unicode_range* p = &ranges[mid]; 2357 if (c < p->low) 2358 hi = mid; 2359 else if (c > p->high) 2360 lo = mid + 1; 2361 else 2362 return (c - p->low) % p->stride == 0; 2363 } 2364 return false; 2365 } 2366} 2367 2368// Return whether C is a space character. 2369 2370bool 2371Lex::is_unicode_space(unsigned int c) 2372{ 2373 return Lex::is_in_unicode_range(c, unicode_space, 2374 ARRAY_SIZE(unicode_space)); 2375} 2376 2377// Return whether C is a Unicode digit--a Unicode code point 2378// classified as "Digit". 2379 2380bool 2381Lex::is_unicode_digit(unsigned int c) 2382{ 2383 return Lex::is_in_unicode_range(c, unicode_digits, 2384 ARRAY_SIZE(unicode_digits)); 2385} 2386 2387// Return whether C is a Unicode letter--a Unicode code point 2388// classified as "Letter". 2389 2390bool 2391Lex::is_unicode_letter(unsigned int c) 2392{ 2393 return Lex::is_in_unicode_range(c, unicode_letters, 2394 ARRAY_SIZE(unicode_letters)); 2395} 2396 2397// Return whether C is a Unicode uppercase letter. a Unicode code 2398// point classified as "Letter, uppercase". 2399 2400bool 2401Lex::is_unicode_uppercase(unsigned int c) 2402{ 2403 return Lex::is_in_unicode_range(c, unicode_uppercase_letters, 2404 ARRAY_SIZE(unicode_uppercase_letters)); 2405} 2406 2407// Return whether the identifier NAME should be exported. NAME is a 2408// mangled name which includes only ASCII characters. 2409 2410bool 2411Lex::is_exported_name(const std::string& name) 2412{ 2413 unsigned char c = name[0]; 2414 if (c != '$') 2415 return c >= 'A' && c <= 'Z'; 2416 else 2417 { 2418 const char* p = name.data(); 2419 size_t len = name.length(); 2420 if (len < 2 || p[1] != 'U') 2421 return false; 2422 unsigned int ci = 0; 2423 for (size_t i = 2; i < len && p[i] != '$'; ++i) 2424 { 2425 c = p[i]; 2426 if (!hex_p(c)) 2427 return false; 2428 ci <<= 4; 2429 ci |= hex_value(c); 2430 } 2431 return Lex::is_unicode_uppercase(ci); 2432 } 2433} 2434 2435// Return whether the identifier NAME contains an invalid character. 2436// This is based on how we handle invalid characters in 2437// gather_identifier. 2438 2439bool 2440Lex::is_invalid_identifier(const std::string& name) 2441{ 2442 return name.find("$INVALID$") != std::string::npos; 2443} 2444