1// lex.cc -- Go frontend lexer.
2
3// Copyright 2009 The Go Authors. All rights reserved.
4// Use of this source code is governed by a BSD-style
5// license that can be found in the LICENSE file.
6
7#include "go-system.h"
8
9#include "lex.h"
10
11// Manage mapping from keywords to the Keyword codes.
12
13class Keywords
14{
15 public:
16  // The structure which maps keywords to codes.
17  struct Mapping
18  {
19    // Keyword string.
20    const char* keystring;
21    // Keyword code.
22    Keyword keycode;
23  };
24
25  // Return the parsecode corresponding to KEYSTRING, or
26  // KEYWORD_INVALID if it is not a keyword.
27  Keyword
28  keyword_to_code(const char* keyword, size_t len) const;
29
30  // Return the string for a keyword.
31  const char*
32  keyword_to_string(Keyword) const;
33
34 private:
35  static const Mapping mapping_[];
36  static const int count_;
37};
38
39// Mapping from keyword string to keyword code.  This array must be
40// kept in sorted order, and the order must match the Keyword enum.
41// Strings are looked up using bsearch.
42
43const Keywords::Mapping
44Keywords::mapping_[] =
45{
46  { NULL,	   KEYWORD_INVALID },
47  { "__asm__",	   KEYWORD_ASM },
48  { "break",	   KEYWORD_BREAK },
49  { "case",	   KEYWORD_CASE },
50  { "chan",	   KEYWORD_CHAN },
51  { "const",	   KEYWORD_CONST },
52  { "continue",	   KEYWORD_CONTINUE },
53  { "default",	   KEYWORD_DEFAULT },
54  { "defer",	   KEYWORD_DEFER },
55  { "else",	   KEYWORD_ELSE },
56  { "fallthrough", KEYWORD_FALLTHROUGH },
57  { "for",	   KEYWORD_FOR },
58  { "func",	   KEYWORD_FUNC },
59  { "go",	   KEYWORD_GO },
60  { "goto",	   KEYWORD_GOTO },
61  { "if",	   KEYWORD_IF },
62  { "import",	   KEYWORD_IMPORT },
63  { "interface",   KEYWORD_INTERFACE },
64  { "map",	   KEYWORD_MAP },
65  { "package",	   KEYWORD_PACKAGE },
66  { "range",	   KEYWORD_RANGE },
67  { "return",	   KEYWORD_RETURN },
68  { "select",	   KEYWORD_SELECT },
69  { "struct",	   KEYWORD_STRUCT },
70  { "switch",	   KEYWORD_SWITCH },
71  { "type",	   KEYWORD_TYPE },
72  { "var",	   KEYWORD_VAR }
73};
74
75// Number of entries in the map.
76
77const int Keywords::count_ =
78  sizeof(Keywords::mapping_) / sizeof(Keywords::mapping_[0]);
79
80// Comparison function passed to bsearch.
81
82extern "C"
83{
84
85struct Keywords_search_key
86{
87  const char* str;
88  size_t len;
89};
90
91static int
92keyword_compare(const void* keyv, const void* mapv)
93{
94  const Keywords_search_key* key =
95    static_cast<const Keywords_search_key*>(keyv);
96  const Keywords::Mapping* map =
97    static_cast<const Keywords::Mapping*>(mapv);
98  if (map->keystring == NULL)
99    return 1;
100  int i = strncmp(key->str, map->keystring, key->len);
101  if (i != 0)
102    return i;
103  if (map->keystring[key->len] != '\0')
104    return -1;
105  return 0;
106}
107
108} // End extern "C".
109
110// Convert a string to a keyword code.  Return KEYWORD_INVALID if the
111// string is not a keyword.
112
113Keyword
114Keywords::keyword_to_code(const char* keyword, size_t len) const
115{
116  Keywords_search_key key;
117  key.str = keyword;
118  key.len = len;
119  void* mapv = bsearch(&key,
120                       this->mapping_,
121                       this->count_,
122                       sizeof(this->mapping_[0]),
123                       keyword_compare);
124  if (mapv == NULL)
125    return KEYWORD_INVALID;
126  Mapping* map = static_cast<Mapping*>(mapv);
127  return map->keycode;
128}
129
130// Convert a keyword code to a string.
131
132const char*
133Keywords::keyword_to_string(Keyword code) const
134{
135  go_assert(code > KEYWORD_INVALID && code < this->count_);
136  const Mapping* map = &this->mapping_[code];
137  go_assert(map->keycode == code);
138  return map->keystring;
139}
140
141// There is one instance of the Keywords class.
142
143static Keywords keywords;
144
145// Class Token.
146
147// Make a general token.
148
149Token::Token(Classification classification, Location location)
150  : classification_(classification), location_(location)
151{
152}
153
154// Destroy a token.
155
156Token::~Token()
157{
158  this->clear();
159}
160
161// Clear a token--release memory.
162
163void
164Token::clear()
165{
166  if (this->classification_ == TOKEN_INTEGER
167      || this->classification_ == TOKEN_CHARACTER)
168    mpz_clear(this->u_.integer_value);
169  else if (this->classification_ == TOKEN_FLOAT
170	   || this->classification_ == TOKEN_IMAGINARY)
171    mpfr_clear(this->u_.float_value);
172}
173
174// Construct a token.
175
176Token::Token(const Token& tok)
177  : classification_(tok.classification_), location_(tok.location_)
178{
179  switch (this->classification_)
180    {
181    case TOKEN_INVALID:
182    case TOKEN_EOF:
183      break;
184    case TOKEN_KEYWORD:
185      this->u_.keyword = tok.u_.keyword;
186      break;
187    case TOKEN_IDENTIFIER:
188    case TOKEN_STRING:
189      this->u_.string_value = tok.u_.string_value;
190      break;
191    case TOKEN_OPERATOR:
192      this->u_.op = tok.u_.op;
193      break;
194    case TOKEN_CHARACTER:
195    case TOKEN_INTEGER:
196      mpz_init_set(this->u_.integer_value, tok.u_.integer_value);
197      break;
198    case TOKEN_FLOAT:
199    case TOKEN_IMAGINARY:
200      mpfr_init_set(this->u_.float_value, tok.u_.float_value, GMP_RNDN);
201      break;
202    default:
203      go_unreachable();
204    }
205}
206
207// Assign to a token.
208
209Token&
210Token::operator=(const Token& tok)
211{
212  this->clear();
213  this->classification_ = tok.classification_;
214  this->location_ = tok.location_;
215  switch (tok.classification_)
216    {
217    case TOKEN_INVALID:
218    case TOKEN_EOF:
219      break;
220    case TOKEN_KEYWORD:
221      this->u_.keyword = tok.u_.keyword;
222      break;
223    case TOKEN_IDENTIFIER:
224      this->u_.identifier_value.name = tok.u_.identifier_value.name;
225      this->u_.identifier_value.is_exported =
226	tok.u_.identifier_value.is_exported;
227      break;
228    case TOKEN_STRING:
229      this->u_.string_value = tok.u_.string_value;
230      break;
231    case TOKEN_OPERATOR:
232      this->u_.op = tok.u_.op;
233      break;
234    case TOKEN_CHARACTER:
235    case TOKEN_INTEGER:
236      mpz_init_set(this->u_.integer_value, tok.u_.integer_value);
237      break;
238    case TOKEN_FLOAT:
239    case TOKEN_IMAGINARY:
240      mpfr_init_set(this->u_.float_value, tok.u_.float_value, GMP_RNDN);
241      break;
242    default:
243      go_unreachable();
244    }
245  return *this;
246}
247
248// Print the token for debugging.
249
250void
251Token::print(FILE* file) const
252{
253  switch (this->classification_)
254    {
255    case TOKEN_INVALID:
256      fprintf(file, "invalid");
257      break;
258    case TOKEN_EOF:
259      fprintf(file, "EOF");
260      break;
261    case TOKEN_KEYWORD:
262      fprintf(file, "keyword %s", keywords.keyword_to_string(this->u_.keyword));
263      break;
264    case TOKEN_IDENTIFIER:
265      fprintf(file, "identifier \"%s\"", this->u_.string_value->c_str());
266      break;
267    case TOKEN_STRING:
268      fprintf(file, "quoted string \"%s\"", this->u_.string_value->c_str());
269      break;
270    case TOKEN_CHARACTER:
271      fprintf(file, "character ");
272      mpz_out_str(file, 10, this->u_.integer_value);
273      break;
274    case TOKEN_INTEGER:
275      fprintf(file, "integer ");
276      mpz_out_str(file, 10, this->u_.integer_value);
277      break;
278    case TOKEN_FLOAT:
279      fprintf(file, "float ");
280      mpfr_out_str(file, 10, 0, this->u_.float_value, GMP_RNDN);
281      break;
282    case TOKEN_IMAGINARY:
283      fprintf(file, "imaginary ");
284      mpfr_out_str(file, 10, 0, this->u_.float_value, GMP_RNDN);
285      break;
286    case TOKEN_OPERATOR:
287      fprintf(file, "operator ");
288      switch (this->u_.op)
289	{
290	case OPERATOR_INVALID:
291	  fprintf(file, "invalid");
292	  break;
293	case OPERATOR_OROR:
294	  fprintf(file, "||");
295	  break;
296	case OPERATOR_ANDAND:
297	  fprintf(file, "&&");
298	  break;
299	case OPERATOR_EQEQ:
300	  fprintf(file, "==");
301	  break;
302	case OPERATOR_NOTEQ:
303	  fprintf(file, "!=");
304	  break;
305	case OPERATOR_LT:
306	  fprintf(file, "<");
307	  break;
308	case OPERATOR_LE:
309	  fprintf(file, "<=");
310	  break;
311	case OPERATOR_GT:
312	  fprintf(file, ">");
313	  break;
314	case OPERATOR_GE:
315	  fprintf(file, ">=");
316	  break;
317	case OPERATOR_PLUS:
318	  fprintf(file, "+");
319	  break;
320	case OPERATOR_MINUS:
321	  fprintf(file, "-");
322	  break;
323	case OPERATOR_OR:
324	  fprintf(file, "|");
325	  break;
326	case OPERATOR_XOR:
327	  fprintf(file, "^");
328	  break;
329	case OPERATOR_MULT:
330	  fprintf(file, "*");
331	  break;
332	case OPERATOR_DIV:
333	  fprintf(file, "/");
334	  break;
335	case OPERATOR_MOD:
336	  fprintf(file, "%%");
337	  break;
338	case OPERATOR_LSHIFT:
339	  fprintf(file, "<<");
340	  break;
341	case OPERATOR_RSHIFT:
342	  fprintf(file, ">>");
343	  break;
344	case OPERATOR_AND:
345	  fprintf(file, "&");
346	  break;
347	case OPERATOR_BITCLEAR:
348	  fprintf(file, "&^");
349	  break;
350	case OPERATOR_NOT:
351	  fprintf(file, "!");
352	  break;
353	case OPERATOR_CHANOP:
354	  fprintf(file, "<-");
355	  break;
356	case OPERATOR_EQ:
357	  fprintf(file, "=");
358	  break;
359	case OPERATOR_PLUSEQ:
360	  fprintf(file, "+=");
361	  break;
362	case OPERATOR_MINUSEQ:
363	  fprintf(file, "-=");
364	  break;
365	case OPERATOR_OREQ:
366	  fprintf(file, "|=");
367	  break;
368	case OPERATOR_XOREQ:
369	  fprintf(file, "^=");
370	  break;
371	case OPERATOR_MULTEQ:
372	  fprintf(file, "*=");
373	  break;
374	case OPERATOR_DIVEQ:
375	  fprintf(file, "/=");
376	  break;
377	case OPERATOR_MODEQ:
378	  fprintf(file, "%%=");
379	  break;
380	case OPERATOR_LSHIFTEQ:
381	  fprintf(file, "<<=");
382	  break;
383	case OPERATOR_RSHIFTEQ:
384	  fprintf(file, ">>=");
385	  break;
386	case OPERATOR_ANDEQ:
387	  fprintf(file, "&=");
388	  break;
389	case OPERATOR_BITCLEAREQ:
390	  fprintf(file, "&^=");
391	  break;
392	case OPERATOR_PLUSPLUS:
393	  fprintf(file, "++");
394	  break;
395	case OPERATOR_MINUSMINUS:
396	  fprintf(file, "--");
397	  break;
398	case OPERATOR_COLON:
399	  fprintf(file, ":");
400	  break;
401	case OPERATOR_COLONEQ:
402	  fprintf(file, ":=");
403	  break;
404	case OPERATOR_SEMICOLON:
405	  fprintf(file, ";");
406	  break;
407	case OPERATOR_DOT:
408	  fprintf(file, ".");
409	  break;
410	case OPERATOR_COMMA:
411	  fprintf(file, ",");
412	  break;
413	case OPERATOR_LPAREN:
414	  fprintf(file, "(");
415	  break;
416	case OPERATOR_RPAREN:
417	  fprintf(file, ")");
418	  break;
419	case OPERATOR_LCURLY:
420	  fprintf(file, "{");
421	  break;
422	case OPERATOR_RCURLY:
423	  fprintf(file, "}");
424	  break;
425	case OPERATOR_LSQUARE:
426	  fprintf(file, "[");
427	  break;
428	case OPERATOR_RSQUARE:
429	  fprintf(file, "]");
430	  break;
431	default:
432	  go_unreachable();
433	}
434      break;
435    default:
436      go_unreachable();
437    }
438}
439
440// Class Lex.
441
442Lex::Lex(const char* input_file_name, FILE* input_file, Linemap* linemap)
443  : input_file_name_(input_file_name), input_file_(input_file),
444    linemap_(linemap), linebuf_(NULL), linebufsize_(120), linesize_(0),
445    lineoff_(0), lineno_(0), add_semi_at_eol_(false), saw_nointerface_(false),
446    extern_()
447{
448  this->linebuf_ = new char[this->linebufsize_];
449  this->linemap_->start_file(input_file_name, 0);
450}
451
452Lex::~Lex()
453{
454  delete[] this->linebuf_;
455}
456
457// Read a new line from the file.
458
459ssize_t
460Lex::get_line()
461{
462  char* buf = this->linebuf_;
463  size_t size = this->linebufsize_;
464
465  FILE* file = this->input_file_;
466  size_t cur = 0;
467  while (true)
468    {
469      int c = getc(file);
470      if (c == EOF)
471	{
472	  if (cur == 0)
473	    return -1;
474	  break;
475	}
476      if (cur + 1 >= size)
477	{
478	  size_t ns = 2 * size + 1;
479	  if (ns < size || static_cast<ssize_t>(ns) < 0)
480	    error_at(this->location(), "out of memory");
481	  char* nb = new char[ns];
482	  memcpy(nb, buf, cur);
483	  delete[] buf;
484	  buf = nb;
485	  size = ns;
486	}
487      buf[cur] = c;
488      ++cur;
489
490      if (c == '\n')
491	break;
492    }
493
494  buf[cur] = '\0';
495
496  this->linebuf_ = buf;
497  this->linebufsize_ = size;
498
499  return cur;
500}
501
502// See if we need to read a new line.  Return true if there is a new
503// line, false if we are at EOF.
504
505bool
506Lex::require_line()
507{
508  if (this->lineoff_ < this->linesize_)
509    return true;
510
511  ssize_t got = this->get_line();
512  if (got < 0)
513    return false;
514  ++this->lineno_;
515  this->linesize_= got;
516  this->lineoff_ = 0;
517
518  this->linemap_->start_line(this->lineno_, this->linesize_);
519
520  return true;
521}
522
523// Get the current location.
524
525Location
526Lex::location() const
527{
528  return this->linemap_->get_location(this->lineoff_ + 1);
529}
530
531// Get a location slightly before the current one.  This is used for
532// slightly more efficient handling of operator tokens.
533
534Location
535Lex::earlier_location(int chars) const
536{
537  return this->linemap_->get_location(this->lineoff_ + 1 - chars);
538}
539
540// Get the next token.
541
542Token
543Lex::next_token()
544{
545  bool saw_cpp_comment = false;
546  while (true)
547    {
548      if (!this->require_line())
549	{
550	  bool add_semi_at_eol = this->add_semi_at_eol_;
551	  this->add_semi_at_eol_ = false;
552	  if (add_semi_at_eol)
553	    return this->make_operator(OPERATOR_SEMICOLON, 1);
554	  return this->make_eof_token();
555	}
556
557      if (!saw_cpp_comment)
558	this->extern_.clear();
559      saw_cpp_comment = false;
560
561      const char* p = this->linebuf_ + this->lineoff_;
562      const char* pend = this->linebuf_ + this->linesize_;
563
564      while (p < pend)
565	{
566	  unsigned char cc = *p;
567	  switch (cc)
568	    {
569	    case ' ': case '\t': case '\r':
570	      ++p;
571	      // Skip whitespace quickly.
572	      while (*p == ' ' || *p == '\t' || *p == '\r')
573		++p;
574	      break;
575
576	    case '\n':
577	      {
578		++p;
579		bool add_semi_at_eol = this->add_semi_at_eol_;
580		this->add_semi_at_eol_ = false;
581		if (add_semi_at_eol)
582		  {
583		    this->lineoff_ = p - this->linebuf_;
584		    return this->make_operator(OPERATOR_SEMICOLON, 1);
585		  }
586	      }
587	      break;
588
589	    case '/':
590	      if (p[1] == '/')
591		{
592		  this->lineoff_ = p + 2 - this->linebuf_;
593		  this->skip_cpp_comment();
594		  p = pend;
595		  if (p[-1] == '\n' && this->add_semi_at_eol_)
596		    --p;
597		  saw_cpp_comment = true;
598		}
599	      else if (p[1] == '*')
600		{
601		  this->lineoff_ = p + 2 - this->linebuf_;
602		  Location location = this->location();
603		  if (!this->skip_c_comment())
604		    return Token::make_invalid_token(location);
605		  p = this->linebuf_ + this->lineoff_;
606		  pend = this->linebuf_ + this->linesize_;
607		}
608	      else if (p[1] == '=')
609		{
610		  this->add_semi_at_eol_ = false;
611		  this->lineoff_ = p + 2 - this->linebuf_;
612		  return this->make_operator(OPERATOR_DIVEQ, 2);
613		}
614	      else
615		{
616		  this->add_semi_at_eol_ = false;
617		  this->lineoff_ = p + 1 - this->linebuf_;
618		  return this->make_operator(OPERATOR_DIV, 1);
619		}
620	      break;
621
622	    case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
623	    case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
624	    case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
625	    case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
626	    case 'Y': case 'Z':
627	    case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
628	    case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
629	    case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
630	    case 's': case 't': case 'u': case 'v': case 'w': case 'x':
631	    case 'y': case 'z':
632	    case '_':
633	      this->lineoff_ = p - this->linebuf_;
634	      return this->gather_identifier();
635
636	    case '0': case '1': case '2': case '3': case '4':
637	    case '5': case '6': case '7': case '8': case '9':
638	      this->add_semi_at_eol_ = true;
639	      this->lineoff_ = p - this->linebuf_;
640	      return this->gather_number();
641
642	    case '\'':
643	      this->add_semi_at_eol_ = true;
644	      this->lineoff_ = p - this->linebuf_;
645	      return this->gather_character();
646
647	    case '"':
648	      this->add_semi_at_eol_ = true;
649	      this->lineoff_ = p - this->linebuf_;
650	      return this->gather_string();
651
652	    case '`':
653	      this->add_semi_at_eol_ = true;
654	      this->lineoff_ = p - this->linebuf_;
655	      return this->gather_raw_string();
656
657	    case '<':
658	    case '>':
659	    case '&':
660	      if (p + 2 < pend)
661		{
662		  this->add_semi_at_eol_ = false;
663		  Operator op = this->three_character_operator(cc, p[1], p[2]);
664		  if (op != OPERATOR_INVALID)
665		    {
666		      this->lineoff_ = p + 3 - this->linebuf_;
667		      return this->make_operator(op, 3);
668		    }
669		}
670	      // Fall through.
671	    case '|':
672	    case '=':
673	    case '!':
674	    case '+':
675	    case '-':
676	    case '^':
677	    case '*':
678	      // '/' handled above.
679	    case '%':
680	    case ':':
681	    case ';':
682	    case ',':
683	    case '(': case ')':
684	    case '{': case '}':
685	    case '[': case ']':
686	      {
687		this->add_semi_at_eol_ = false;
688		Operator op = this->two_character_operator(cc, p[1]);
689		int chars;
690		if (op != OPERATOR_INVALID)
691		  {
692		    ++p;
693		    chars = 2;
694		  }
695		else
696		  {
697		    op = this->one_character_operator(cc);
698		    chars = 1;
699		  }
700		this->lineoff_ = p + 1 - this->linebuf_;
701		return this->make_operator(op, chars);
702	      }
703
704	    case '.':
705	      if (p[1] >= '0' && p[1] <= '9')
706		{
707		  this->add_semi_at_eol_ = true;
708		  this->lineoff_ = p - this->linebuf_;
709		  return this->gather_number();
710		}
711	      if (p[1] == '.' && p[2] == '.')
712		{
713		  this->add_semi_at_eol_ = false;
714		  this->lineoff_ = p + 3 - this->linebuf_;
715		  return this->make_operator(OPERATOR_ELLIPSIS, 3);
716		}
717	      this->add_semi_at_eol_ = false;
718	      this->lineoff_ = p + 1 - this->linebuf_;
719	      return this->make_operator(OPERATOR_DOT, 1);
720
721	    default:
722	      {
723		unsigned int ci;
724		bool issued_error;
725		this->lineoff_ = p - this->linebuf_;
726		const char *pnext = this->advance_one_utf8_char(p, &ci,
727								&issued_error);
728
729		// Ignore byte order mark at start of file.
730		if (ci == 0xfeff)
731		  {
732		    p = pnext;
733		    break;
734		  }
735
736		if (Lex::is_unicode_letter(ci))
737		  return this->gather_identifier();
738
739		if (!issued_error)
740		  error_at(this->location(),
741			   "invalid character 0x%x in input file",
742			   ci);
743
744		p = pend;
745
746		break;
747	      }
748	    }
749	}
750
751      this->lineoff_ = p - this->linebuf_;
752    }
753}
754
755// Fetch one UTF-8 character from a string.  Set *VALUE to the value.
756// Return the number of bytes read from the string.  Returns 0 if the
757// string does not point to a valid UTF-8 character.
758
759int
760Lex::fetch_char(const char* p, unsigned int* value)
761{
762  unsigned char c = *p;
763  if (c <= 0x7f)
764    {
765      *value = c;
766      return 1;
767    }
768  else if ((c & 0xe0) == 0xc0
769	   && (p[1] & 0xc0) == 0x80)
770    {
771      *value = (((c & 0x1f) << 6)
772		+ (p[1] & 0x3f));
773      if (*value <= 0x7f)
774	{
775	  *value = 0xfffd;
776	  return 0;
777	}
778      return 2;
779    }
780  else if ((c & 0xf0) == 0xe0
781	   && (p[1] & 0xc0) == 0x80
782	   && (p[2] & 0xc0) == 0x80)
783    {
784      *value = (((c & 0xf) << 12)
785		+ ((p[1] & 0x3f) << 6)
786		+ (p[2] & 0x3f));
787      if (*value <= 0x7ff)
788	{
789	  *value = 0xfffd;
790	  return 0;
791	}
792      return 3;
793    }
794  else if ((c & 0xf8) == 0xf0
795	   && (p[1] & 0xc0) == 0x80
796	   && (p[2] & 0xc0) == 0x80
797	   && (p[3] & 0xc0) == 0x80)
798    {
799      *value = (((c & 0x7) << 18)
800		+ ((p[1] & 0x3f) << 12)
801		+ ((p[2] & 0x3f) << 6)
802		+ (p[3] & 0x3f));
803      if (*value <= 0xffff)
804	{
805	  *value = 0xfffd;
806	  return 0;
807	}
808      return 4;
809    }
810  else
811    {
812      /* Invalid encoding. Return the Unicode replacement
813	 character.  */
814      *value = 0xfffd;
815      return 0;
816    }
817}
818
819// Advance one UTF-8 character.  Return the pointer beyond the
820// character.  Set *VALUE to the value.  Set *ISSUED_ERROR if an error
821// was issued.
822
823const char*
824Lex::advance_one_utf8_char(const char* p, unsigned int* value,
825			   bool* issued_error)
826{
827  *issued_error = false;
828
829  if (*p == '\0')
830    {
831      error_at(this->location(), "invalid NUL byte");
832      *issued_error = true;
833      *value = 0;
834      return p + 1;
835    }
836
837  int adv = Lex::fetch_char(p, value);
838  if (adv == 0)
839    {
840      error_at(this->location(), "invalid UTF-8 encoding");
841      *issued_error = true;
842      return p + 1;
843    }
844
845  // Warn about byte order mark, except at start of file.
846  if (*value == 0xfeff && (this->lineno_ != 1 || this->lineoff_ != 0))
847    {
848      error_at(this->location(), "Unicode (UTF-8) BOM in middle of file");
849      *issued_error = true;
850    }
851
852  return p + adv;
853}
854
855// Pick up an identifier.
856
857Token
858Lex::gather_identifier()
859{
860  const char* pstart = this->linebuf_ + this->lineoff_;
861  const char* p = pstart;
862  const char* pend = this->linebuf_ + this->linesize_;
863  bool is_first = true;
864  bool is_exported = false;
865  bool has_non_ascii_char = false;
866  std::string buf;
867  while (p < pend)
868    {
869      unsigned char cc = *p;
870      if (cc <= 0x7f)
871	{
872	  if ((cc < 'A' || cc > 'Z')
873	      && (cc < 'a' || cc > 'z')
874	      && cc != '_'
875	      && (cc < '0' || cc > '9'))
876	    {
877	      // Check for an invalid character here, as we get better
878	      // error behaviour if we swallow them as part of the
879	      // identifier we are building.
880	      if ((cc >= ' ' && cc < 0x7f)
881		  || cc == '\t'
882		  || cc == '\r'
883		  || cc == '\n')
884		break;
885
886	      this->lineoff_ = p - this->linebuf_;
887	      error_at(this->location(),
888		       "invalid character 0x%x in identifier",
889		       cc);
890	      if (!has_non_ascii_char)
891		{
892		  buf.assign(pstart, p - pstart);
893		  has_non_ascii_char = true;
894		}
895	      if (!Lex::is_invalid_identifier(buf))
896		buf.append("$INVALID$");
897	    }
898	  ++p;
899	  if (is_first)
900	    {
901	      is_exported = cc >= 'A' && cc <= 'Z';
902	      is_first = false;
903	    }
904	  if (has_non_ascii_char)
905	    buf.push_back(cc);
906	}
907      else
908	{
909	  unsigned int ci;
910	  bool issued_error;
911	  this->lineoff_ = p - this->linebuf_;
912	  const char* pnext = this->advance_one_utf8_char(p, &ci,
913							  &issued_error);
914	  bool is_invalid = false;
915	  if (!Lex::is_unicode_letter(ci) && !Lex::is_unicode_digit(ci))
916	    {
917	      // There is no valid place for a non-ASCII character
918	      // other than an identifier, so we get better error
919	      // handling behaviour if we swallow this character after
920	      // giving an error.
921	      if (!issued_error)
922		error_at(this->location(),
923			 "invalid character 0x%x in identifier",
924			 ci);
925	      is_invalid = true;
926	    }
927	  if (is_first)
928	    {
929	      is_exported = Lex::is_unicode_uppercase(ci);
930	      is_first = false;
931	    }
932	  if (!has_non_ascii_char)
933	    {
934	      buf.assign(pstart, p - pstart);
935	      has_non_ascii_char = true;
936	    }
937	  if (is_invalid && !Lex::is_invalid_identifier(buf))
938	    buf.append("$INVALID$");
939	  buf.append(p, pnext - p);
940	  p = pnext;
941	}
942    }
943  Location location = this->location();
944  this->add_semi_at_eol_ = true;
945  this->lineoff_ = p - this->linebuf_;
946  if (has_non_ascii_char)
947    return Token::make_identifier_token(buf, is_exported, location);
948  else
949    {
950      Keyword code = keywords.keyword_to_code(pstart, p - pstart);
951      if (code == KEYWORD_INVALID)
952	return Token::make_identifier_token(std::string(pstart, p - pstart),
953					    is_exported, location);
954      else
955	{
956	  switch (code)
957	    {
958	    case KEYWORD_BREAK:
959	    case KEYWORD_CONTINUE:
960	    case KEYWORD_FALLTHROUGH:
961	    case KEYWORD_RETURN:
962	      break;
963	    default:
964	      this->add_semi_at_eol_ = false;
965	      break;
966	    }
967	  return Token::make_keyword_token(code, location);
968	}
969    }
970}
971
972// Return whether C is a hex digit.
973
974bool
975Lex::is_hex_digit(char c)
976{
977  return ((c >= '0' && c <= '9')
978	  || (c >= 'A' && c <= 'F')
979	  || (c >= 'a' && c <= 'f'));
980}
981
982// Return whether an exponent could start at P.
983
984bool
985Lex::could_be_exponent(const char* p, const char* pend)
986{
987  if (*p != 'e' && *p != 'E')
988    return false;
989  ++p;
990  if (p >= pend)
991    return false;
992  if (*p == '+' || *p == '-')
993    {
994      ++p;
995      if (p >= pend)
996	return false;
997    }
998  return *p >= '0' && *p <= '9';
999}
1000
1001// Pick up a number.
1002
1003Token
1004Lex::gather_number()
1005{
1006  const char* pstart = this->linebuf_ + this->lineoff_;
1007  const char* p = pstart;
1008  const char* pend = this->linebuf_ + this->linesize_;
1009
1010  Location location = this->location();
1011
1012  bool neg = false;
1013  if (*p == '+')
1014    ++p;
1015  else if (*p == '-')
1016    {
1017      ++p;
1018      neg = true;
1019    }
1020
1021  const char* pnum = p;
1022  if (*p == '0')
1023    {
1024      int base;
1025      if ((p[1] == 'x' || p[1] == 'X')
1026	  && Lex::is_hex_digit(p[2]))
1027	{
1028	  base = 16;
1029	  p += 2;
1030	  pnum = p;
1031	  while (p < pend)
1032	    {
1033	      if (!Lex::is_hex_digit(*p))
1034		break;
1035	      ++p;
1036	    }
1037	}
1038      else
1039	{
1040	  base = 8;
1041	  pnum = p;
1042	  while (p < pend)
1043	    {
1044	      if (*p < '0' || *p > '7')
1045		break;
1046	      ++p;
1047	    }
1048	}
1049
1050      // A partial token that looks like an octal literal might actually be the
1051      // beginning of a floating-point or imaginary literal.
1052      if (base == 16 || (*p != '.' && *p != 'i' && !Lex::could_be_exponent(p, pend)))
1053	{
1054	  std::string s(pnum, p - pnum);
1055	  mpz_t val;
1056	  int r = mpz_init_set_str(val, s.c_str(), base);
1057	  go_assert(r == 0);
1058
1059	  if (neg)
1060	    mpz_neg(val, val);
1061
1062	  this->lineoff_ = p - this->linebuf_;
1063	  Token ret = Token::make_integer_token(val, location);
1064	  mpz_clear(val);
1065	  return ret;
1066	}
1067    }
1068
1069  while (p < pend)
1070    {
1071      if (*p < '0' || *p > '9')
1072	break;
1073      ++p;
1074    }
1075
1076  if (*p != '.' && *p != 'i' && !Lex::could_be_exponent(p, pend))
1077    {
1078      std::string s(pnum, p - pnum);
1079      mpz_t val;
1080      int r = mpz_init_set_str(val, s.c_str(), 10);
1081      go_assert(r == 0);
1082
1083      if (neg)
1084	mpz_neg(val, val);
1085
1086      this->lineoff_ = p - this->linebuf_;
1087      Token ret = Token::make_integer_token(val, location);
1088      mpz_clear(val);
1089      return ret;
1090    }
1091
1092  if (*p != 'i')
1093    {
1094      bool dot = *p == '.';
1095
1096      ++p;
1097
1098      if (!dot)
1099	{
1100	  if (*p == '+' || *p == '-')
1101	    ++p;
1102	}
1103
1104      while (p < pend)
1105	{
1106	  if (*p < '0' || *p > '9')
1107	    break;
1108	  ++p;
1109	}
1110
1111      if (dot && Lex::could_be_exponent(p, pend))
1112	{
1113	  ++p;
1114	  if (*p == '+' || *p == '-')
1115	    ++p;
1116	  while (p < pend)
1117	    {
1118	      if (*p < '0' || *p > '9')
1119		break;
1120	      ++p;
1121	    }
1122	}
1123    }
1124
1125  std::string s(pnum, p - pnum);
1126  mpfr_t val;
1127  int r = mpfr_init_set_str(val, s.c_str(), 10, GMP_RNDN);
1128  go_assert(r == 0);
1129
1130  if (neg)
1131    mpfr_neg(val, val, GMP_RNDN);
1132
1133  bool is_imaginary = *p == 'i';
1134  if (is_imaginary)
1135    ++p;
1136
1137  this->lineoff_ = p - this->linebuf_;
1138  if (is_imaginary)
1139    {
1140      Token ret = Token::make_imaginary_token(val, location);
1141      mpfr_clear(val);
1142      return ret;
1143    }
1144  else
1145    {
1146      Token ret = Token::make_float_token(val, location);
1147      mpfr_clear(val);
1148      return ret;
1149    }
1150}
1151
1152// Advance one character, possibly escaped.  Return the pointer beyond
1153// the character.  Set *VALUE to the character.  Set *IS_CHARACTER if
1154// this is a character (e.g., 'a' or '\u1234') rather than a byte
1155// value (e.g., '\001').
1156
1157const char*
1158Lex::advance_one_char(const char* p, bool is_single_quote, unsigned int* value,
1159		      bool* is_character)
1160{
1161  *value = 0;
1162  *is_character = true;
1163  if (*p != '\\')
1164    {
1165      bool issued_error;
1166      const char* ret = this->advance_one_utf8_char(p, value, &issued_error);
1167      if (is_single_quote
1168	  && (*value == '\'' || *value == '\n')
1169	  && !issued_error)
1170	error_at(this->location(), "invalid character literal");
1171      return ret;
1172    }
1173  else
1174    {
1175      ++p;
1176      switch (*p)
1177	{
1178	case '0': case '1': case '2': case '3':
1179	case '4': case '5': case '6': case '7':
1180	  *is_character = false;
1181	  if (p[1] >= '0' && p[1] <= '7'
1182	      && p[2] >= '0' && p[2] <= '7')
1183	    {
1184	      *value = ((Lex::octal_value(p[0]) << 6)
1185			+ (Lex::octal_value(p[1]) << 3)
1186			+ Lex::octal_value(p[2]));
1187	      if (*value > 255)
1188		{
1189		  error_at(this->location(), "invalid octal constant");
1190		  *value = 255;
1191		}
1192	      return p + 3;
1193	    }
1194	      error_at(this->location(), "invalid octal character");
1195	  return (p[1] >= '0' && p[1] <= '7'
1196		  ? p + 2
1197		  : p + 1);
1198
1199	case 'x':
1200	case 'X':
1201	  *is_character = false;
1202	  if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]))
1203	    {
1204	      *value = (hex_value(p[1]) << 4) + hex_value(p[2]);
1205	      return p + 3;
1206	    }
1207	  error_at(this->location(), "invalid hex character");
1208	  return (Lex::is_hex_digit(p[1])
1209		  ? p + 2
1210		  : p + 1);
1211
1212	case 'a':
1213	  *value = '\a';
1214	  return p + 1;
1215	case 'b':
1216	  *value = '\b';
1217	  return p + 1;
1218	case 'f':
1219	  *value = '\f';
1220	  return p + 1;
1221	case 'n':
1222	  *value = '\n';
1223	  return p + 1;
1224	case 'r':
1225	  *value = '\r';
1226	  return p + 1;
1227	case 't':
1228	  *value = '\t';
1229	  return p + 1;
1230	case 'v':
1231	  *value = '\v';
1232	  return p + 1;
1233	case '\\':
1234	  *value = '\\';
1235	  return p + 1;
1236	case '\'':
1237	  if (!is_single_quote)
1238	    error_at(this->location(), "invalid quoted character");
1239	  *value = '\'';
1240	  return p + 1;
1241	case '"':
1242	  if (is_single_quote)
1243	    error_at(this->location(), "invalid quoted character");
1244	  *value = '"';
1245	  return p + 1;
1246
1247	case 'u':
1248	  if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])
1249	      && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4]))
1250	    {
1251	      *value = ((hex_value(p[1]) << 12)
1252			+ (hex_value(p[2]) << 8)
1253			+ (hex_value(p[3]) << 4)
1254			+ hex_value(p[4]));
1255	      if (*value >= 0xd800 && *value < 0xe000)
1256		{
1257		  error_at(this->location(),
1258			   "invalid unicode code point 0x%x",
1259			   *value);
1260		  // Use the replacement character.
1261		  *value = 0xfffd;
1262		}
1263	      return p + 5;
1264	    }
1265	  error_at(this->location(), "invalid little unicode code point");
1266	  return p + 1;
1267
1268	case 'U':
1269	  if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])
1270	      && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4])
1271	      && Lex::is_hex_digit(p[5]) && Lex::is_hex_digit(p[6])
1272	      && Lex::is_hex_digit(p[7]) && Lex::is_hex_digit(p[8]))
1273	    {
1274	      *value = ((hex_value(p[1]) << 28)
1275			+ (hex_value(p[2]) << 24)
1276			+ (hex_value(p[3]) << 20)
1277			+ (hex_value(p[4]) << 16)
1278			+ (hex_value(p[5]) << 12)
1279			+ (hex_value(p[6]) << 8)
1280			+ (hex_value(p[7]) << 4)
1281			+ hex_value(p[8]));
1282	      if (*value > 0x10ffff
1283		  || (*value >= 0xd800 && *value < 0xe000))
1284		{
1285		  error_at(this->location(), "invalid unicode code point 0x%x",
1286			   *value);
1287		  // Use the replacement character.
1288		  *value = 0xfffd;
1289		}
1290	      return p + 9;
1291	    }
1292	  error_at(this->location(), "invalid big unicode code point");
1293	  return p + 1;
1294
1295	default:
1296	  error_at(this->location(), "invalid character after %<\\%>");
1297	  *value = *p;
1298	  return p + 1;
1299	}
1300    }
1301}
1302
1303// Append V to STR.  IS_CHARACTER is true for a character which should
1304// be stored in UTF-8, false for a general byte value which should be
1305// stored directly.
1306
1307void
1308Lex::append_char(unsigned int v, bool is_character, std::string* str,
1309		 Location location)
1310{
1311  char buf[4];
1312  size_t len;
1313  if (v <= 0x7f || !is_character)
1314    {
1315      buf[0] = v;
1316      len = 1;
1317    }
1318  else if (v <= 0x7ff)
1319    {
1320      buf[0] = 0xc0 + (v >> 6);
1321      buf[1] = 0x80 + (v & 0x3f);
1322      len = 2;
1323    }
1324  else
1325    {
1326      if (v > 0x10ffff)
1327	{
1328	  warning_at(location, 0,
1329		     "unicode code point 0x%x out of range in string", v);
1330	  // Turn it into the "replacement character".
1331	  v = 0xfffd;
1332	}
1333      if (v >= 0xd800 && v < 0xe000)
1334	{
1335	  warning_at(location, 0,
1336		     "unicode code point 0x%x is invalid surrogate pair", v);
1337	  v = 0xfffd;
1338	}
1339      if (v <= 0xffff)
1340	{
1341	  buf[0] = 0xe0 + (v >> 12);
1342	  buf[1] = 0x80 + ((v >> 6) & 0x3f);
1343	  buf[2] = 0x80 + (v & 0x3f);
1344	  len = 3;
1345	}
1346      else
1347	{
1348	  buf[0] = 0xf0 + (v >> 18);
1349	  buf[1] = 0x80 + ((v >> 12) & 0x3f);
1350	  buf[2] = 0x80 + ((v >> 6) & 0x3f);
1351	  buf[3] = 0x80 + (v & 0x3f);
1352	  len = 4;
1353	}
1354    }
1355  str->append(buf, len);
1356}
1357
1358// Pick up a character literal.
1359
1360Token
1361Lex::gather_character()
1362{
1363  ++this->lineoff_;
1364  const char* pstart = this->linebuf_ + this->lineoff_;
1365  const char* p = pstart;
1366
1367  unsigned int value;
1368  bool is_character;
1369  p = this->advance_one_char(p, true, &value, &is_character);
1370
1371  if (*p != '\'')
1372    {
1373      error_at(this->location(), "unterminated character constant");
1374      this->lineoff_ = p - this->linebuf_;
1375      return this->make_invalid_token();
1376    }
1377
1378  mpz_t val;
1379  mpz_init_set_ui(val, value);
1380
1381  Location location = this->location();
1382  this->lineoff_ = p + 1 - this->linebuf_;
1383  Token ret = Token::make_character_token(val, location);
1384  mpz_clear(val);
1385  return ret;
1386}
1387
1388// Pick up a quoted string.
1389
1390Token
1391Lex::gather_string()
1392{
1393  const char* pstart = this->linebuf_ + this->lineoff_ + 1;
1394  const char* p = pstart;
1395  const char* pend = this->linebuf_ + this->linesize_;
1396
1397  std::string value;
1398  while (*p != '"')
1399    {
1400      Location loc = this->location();
1401      unsigned int c;
1402      bool is_character;
1403      this->lineoff_ = p - this->linebuf_;
1404      p = this->advance_one_char(p, false, &c, &is_character);
1405      if (p >= pend)
1406	{
1407	  error_at(this->location(), "unterminated string");
1408	  --p;
1409	  break;
1410	}
1411      Lex::append_char(c, is_character, &value, loc);
1412    }
1413
1414  Location location = this->location();
1415  this->lineoff_ = p + 1 - this->linebuf_;
1416  return Token::make_string_token(value, location);
1417}
1418
1419// Pick up a raw string.
1420
1421Token
1422Lex::gather_raw_string()
1423{
1424  const char* p = this->linebuf_ + this->lineoff_ + 1;
1425  const char* pend = this->linebuf_ + this->linesize_;
1426  Location location = this->location();
1427
1428  std::string value;
1429  while (true)
1430    {
1431      while (p < pend)
1432	{
1433	  if (*p == '`')
1434	    {
1435	      this->lineoff_ = p + 1 - this->linebuf_;
1436	      return Token::make_string_token(value, location);
1437	    }
1438	  Location loc = this->location();
1439	  unsigned int c;
1440	  bool issued_error;
1441	  this->lineoff_ = p - this->linebuf_;
1442	  p = this->advance_one_utf8_char(p, &c, &issued_error);
1443	  // "Carriage return characters ('\r') inside raw string literals
1444	  // are discarded from the raw string value."
1445	  if (c != '\r')
1446	      Lex::append_char(c, true, &value, loc);
1447	}
1448      this->lineoff_ = p - this->linebuf_;
1449      if (!this->require_line())
1450	{
1451	  error_at(location, "unterminated raw string");
1452	  return Token::make_string_token(value, location);
1453	}
1454      p = this->linebuf_ + this->lineoff_;
1455      pend = this->linebuf_ + this->linesize_;
1456    }
1457}
1458
1459// If C1 C2 C3 are a three character operator, return the code.
1460
1461Operator
1462Lex::three_character_operator(char c1, char c2, char c3)
1463{
1464  if (c3 == '=')
1465    {
1466      if (c1 == '<' && c2 == '<')
1467	return OPERATOR_LSHIFTEQ;
1468      else if (c1 == '>' && c2 == '>')
1469	return OPERATOR_RSHIFTEQ;
1470      else if (c1 == '&' && c2 == '^')
1471	return OPERATOR_BITCLEAREQ;
1472    }
1473  return OPERATOR_INVALID;
1474}
1475
1476// If C1 C2 are a two character operator, return the code.
1477
1478Operator
1479Lex::two_character_operator(char c1, char c2)
1480{
1481  switch (c1)
1482    {
1483    case '|':
1484      if (c2 == '|')
1485	return OPERATOR_OROR;
1486      else if (c2 == '=')
1487	return OPERATOR_OREQ;
1488      break;
1489    case '&':
1490      if (c2 == '&')
1491	return OPERATOR_ANDAND;
1492      else if (c2 == '^')
1493	return OPERATOR_BITCLEAR;
1494      else if (c2 == '=')
1495	return OPERATOR_ANDEQ;
1496      break;
1497    case '^':
1498      if (c2 == '=')
1499	return OPERATOR_XOREQ;
1500      break;
1501    case '=':
1502      if (c2 == '=')
1503	return OPERATOR_EQEQ;
1504      break;
1505    case '!':
1506      if (c2 == '=')
1507	return OPERATOR_NOTEQ;
1508      break;
1509    case '<':
1510      if (c2 == '=')
1511	return OPERATOR_LE;
1512      else if (c2 == '<')
1513	return OPERATOR_LSHIFT;
1514      else if (c2 == '-')
1515	return OPERATOR_CHANOP;
1516      break;
1517    case '>':
1518      if (c2 == '=')
1519	return OPERATOR_GE;
1520      else if (c2 == '>')
1521	return OPERATOR_RSHIFT;
1522      break;
1523    case '*':
1524      if (c2 == '=')
1525	return OPERATOR_MULTEQ;
1526      break;
1527    case '/':
1528      if (c2 == '=')
1529	return OPERATOR_DIVEQ;
1530      break;
1531    case '%':
1532      if (c2 == '=')
1533	return OPERATOR_MODEQ;
1534      break;
1535    case '+':
1536      if (c2 == '+')
1537	{
1538	  this->add_semi_at_eol_ = true;
1539	  return OPERATOR_PLUSPLUS;
1540	}
1541      else if (c2 == '=')
1542	return OPERATOR_PLUSEQ;
1543      break;
1544    case '-':
1545      if (c2 == '-')
1546	{
1547	  this->add_semi_at_eol_ = true;
1548	  return OPERATOR_MINUSMINUS;
1549	}
1550      else if (c2 == '=')
1551	return OPERATOR_MINUSEQ;
1552      break;
1553    case ':':
1554      if (c2 == '=')
1555	return OPERATOR_COLONEQ;
1556      break;
1557    default:
1558      break;
1559    }
1560  return OPERATOR_INVALID;
1561}
1562
1563// If character C is an operator, return the code.
1564
1565Operator
1566Lex::one_character_operator(char c)
1567{
1568  switch (c)
1569    {
1570    case '<':
1571      return OPERATOR_LT;
1572    case '>':
1573      return OPERATOR_GT;
1574    case '+':
1575      return OPERATOR_PLUS;
1576    case '-':
1577      return OPERATOR_MINUS;
1578    case '|':
1579      return OPERATOR_OR;
1580    case '^':
1581      return OPERATOR_XOR;
1582    case '*':
1583      return OPERATOR_MULT;
1584    case '/':
1585      return OPERATOR_DIV;
1586    case '%':
1587      return OPERATOR_MOD;
1588    case '&':
1589      return OPERATOR_AND;
1590    case '!':
1591      return OPERATOR_NOT;
1592    case '=':
1593      return OPERATOR_EQ;
1594    case ':':
1595      return OPERATOR_COLON;
1596    case ';':
1597      return OPERATOR_SEMICOLON;
1598    case '.':
1599      return OPERATOR_DOT;
1600    case ',':
1601      return OPERATOR_COMMA;
1602    case '(':
1603      return OPERATOR_LPAREN;
1604    case ')':
1605      this->add_semi_at_eol_ = true;
1606      return OPERATOR_RPAREN;
1607    case '{':
1608      return OPERATOR_LCURLY;
1609    case '}':
1610      this->add_semi_at_eol_ = true;
1611      return OPERATOR_RCURLY;
1612    case '[':
1613      return OPERATOR_LSQUARE;
1614    case ']':
1615      this->add_semi_at_eol_ = true;
1616      return OPERATOR_RSQUARE;
1617    default:
1618      return OPERATOR_INVALID;
1619    }
1620}
1621
1622// Skip a C-style comment.
1623
1624bool
1625Lex::skip_c_comment()
1626{
1627  while (true)
1628    {
1629      if (!this->require_line())
1630	{
1631	  error_at(this->location(), "unterminated comment");
1632	  return false;
1633	}
1634
1635      const char* p = this->linebuf_ + this->lineoff_;
1636      const char* pend = this->linebuf_ + this->linesize_;
1637
1638      while (p < pend)
1639	{
1640	  if (p[0] == '*' && p + 1 < pend && p[1] == '/')
1641	    {
1642	      this->lineoff_ = p + 2 - this->linebuf_;
1643	      return true;
1644	    }
1645
1646	  this->lineoff_ = p - this->linebuf_;
1647	  unsigned int c;
1648	  bool issued_error;
1649	  p = this->advance_one_utf8_char(p, &c, &issued_error);
1650	}
1651
1652      this->lineoff_ = p - this->linebuf_;
1653    }
1654}
1655
1656// Skip a C++-style comment.
1657
1658void
1659Lex::skip_cpp_comment()
1660{
1661  // Ensure that if EXTERN_ is set, it means that we just saw a
1662  // //extern comment.
1663  this->extern_.clear();
1664
1665  const char* p = this->linebuf_ + this->lineoff_;
1666  const char* pend = this->linebuf_ + this->linesize_;
1667
1668  // By convention, a C++ comment at the start of the line of the form
1669  //   //line FILE:LINENO
1670  // is interpreted as setting the file name and line number of the
1671  // next source line.
1672
1673  if (this->lineoff_ == 2
1674      && pend - p > 5
1675      && memcmp(p, "line ", 5) == 0)
1676    {
1677      p += 5;
1678      while (p < pend && *p == ' ')
1679	++p;
1680      const char* pcolon = static_cast<const char*>(memchr(p, ':', pend - p));
1681      if (pcolon != NULL
1682	  && pcolon[1] >= '0'
1683	  && pcolon[1] <= '9')
1684	{
1685	  char* plend;
1686	  long lineno = strtol(pcolon + 1, &plend, 10);
1687	  if (plend > pcolon + 1
1688	      && (plend == pend
1689		  || *plend < '0'
1690		  || *plend > '9')
1691	      && lineno > 0
1692	      && lineno < 0x7fffffff)
1693	    {
1694	      unsigned int filelen = pcolon - p;
1695	      char* file = new char[filelen + 1];
1696	      memcpy(file, p, filelen);
1697	      file[filelen] = '\0';
1698
1699              this->linemap_->start_file(file, lineno);
1700	      this->lineno_ = lineno - 1;
1701
1702	      p = plend;
1703	    }
1704	}
1705    }
1706
1707  // As a special gccgo extension, a C++ comment at the start of the
1708  // line of the form
1709  //   //extern NAME
1710  // which immediately precedes a function declaration means that the
1711  // external name of the function declaration is NAME.  This is
1712  // normally used to permit Go code to call a C function.
1713  if (this->lineoff_ == 2
1714      && pend - p > 7
1715      && memcmp(p, "extern ", 7) == 0)
1716    {
1717      p += 7;
1718      while (p < pend && (*p == ' ' || *p == '\t'))
1719	++p;
1720      const char* plend = pend;
1721      while (plend > p
1722	     && (plend[-1] == ' ' || plend[-1] == '\t' || plend[-1] == '\n'))
1723	--plend;
1724      if (plend > p)
1725	this->extern_ = std::string(p, plend - p);
1726    }
1727
1728  // For field tracking analysis: a //go:nointerface comment means
1729  // that the next interface method should not be stored in the type
1730  // descriptor.  This permits it to be discarded if it is not needed.
1731  if (this->lineoff_ == 2 && memcmp(p, "go:nointerface", 14) == 0)
1732    this->saw_nointerface_ = true;
1733
1734  while (p < pend)
1735    {
1736      this->lineoff_ = p - this->linebuf_;
1737      unsigned int c;
1738      bool issued_error;
1739      p = this->advance_one_utf8_char(p, &c, &issued_error);
1740      if (issued_error)
1741	this->extern_.clear();
1742    }
1743}
1744
1745// The Unicode tables use this struct.
1746
1747struct Unicode_range
1748{
1749  // The low end of the range.
1750  unsigned int low;
1751  // The high end of the range.
1752  unsigned int high;
1753  // The stride.  This entries represents low, low + stride, low + 2 *
1754  // stride, etc., up to high.
1755  unsigned int stride;
1756};
1757
1758// A table of whitespace characters--Unicode code points classified as
1759// "Space", "C" locale whitespace characters, the "next line" control
1760// character (0085), the line separator (2028), the paragraph
1761// separator (2029), and the "zero-width non-break space" (feff).
1762
1763static const Unicode_range unicode_space[] =
1764{
1765  { 0x0009, 0x000d, 1 },
1766  { 0x0020, 0x0020, 1 },
1767  { 0x0085, 0x0085, 1 },
1768  { 0x00a0, 0x00a0, 1 },
1769  { 0x1680, 0x1680, 1 },
1770  { 0x180e, 0x180e, 1 },
1771  { 0x2000, 0x200a, 1 },
1772  { 0x2028, 0x2029, 1 },
1773  { 0x202f, 0x202f, 1 },
1774  { 0x205f, 0x205f, 1 },
1775  { 0x3000, 0x3000, 1 },
1776  { 0xfeff, 0xfeff, 1 },
1777};
1778
1779// A table of Unicode digits--Unicode code points classified as
1780// "Digit".
1781
1782static const Unicode_range unicode_digits[] =
1783{
1784  { 0x0030, 0x0039, 1},
1785  { 0x0660, 0x0669, 1},
1786  { 0x06f0, 0x06f9, 1},
1787  { 0x07c0, 0x07c9, 1},
1788  { 0x0966, 0x096f, 1},
1789  { 0x09e6, 0x09ef, 1},
1790  { 0x0a66, 0x0a6f, 1},
1791  { 0x0ae6, 0x0aef, 1},
1792  { 0x0b66, 0x0b6f, 1},
1793  { 0x0be6, 0x0bef, 1},
1794  { 0x0c66, 0x0c6f, 1},
1795  { 0x0ce6, 0x0cef, 1},
1796  { 0x0d66, 0x0d6f, 1},
1797  { 0x0e50, 0x0e59, 1},
1798  { 0x0ed0, 0x0ed9, 1},
1799  { 0x0f20, 0x0f29, 1},
1800  { 0x1040, 0x1049, 1},
1801  { 0x17e0, 0x17e9, 1},
1802  { 0x1810, 0x1819, 1},
1803  { 0x1946, 0x194f, 1},
1804  { 0x19d0, 0x19d9, 1},
1805  { 0x1b50, 0x1b59, 1},
1806  { 0xff10, 0xff19, 1},
1807  { 0x104a0, 0x104a9, 1},
1808  { 0x1d7ce, 0x1d7ff, 1},
1809};
1810
1811// A table of Unicode letters--Unicode code points classified as
1812// "Letter".
1813
1814static const Unicode_range unicode_letters[] =
1815{
1816  { 0x0041, 0x005a, 1},
1817  { 0x0061, 0x007a, 1},
1818  { 0x00aa, 0x00b5, 11},
1819  { 0x00ba, 0x00ba, 1},
1820  { 0x00c0, 0x00d6, 1},
1821  { 0x00d8, 0x00f6, 1},
1822  { 0x00f8, 0x02c1, 1},
1823  { 0x02c6, 0x02d1, 1},
1824  { 0x02e0, 0x02e4, 1},
1825  { 0x02ec, 0x02ee, 2},
1826  { 0x0370, 0x0374, 1},
1827  { 0x0376, 0x0377, 1},
1828  { 0x037a, 0x037d, 1},
1829  { 0x0386, 0x0386, 1},
1830  { 0x0388, 0x038a, 1},
1831  { 0x038c, 0x038c, 1},
1832  { 0x038e, 0x03a1, 1},
1833  { 0x03a3, 0x03f5, 1},
1834  { 0x03f7, 0x0481, 1},
1835  { 0x048a, 0x0523, 1},
1836  { 0x0531, 0x0556, 1},
1837  { 0x0559, 0x0559, 1},
1838  { 0x0561, 0x0587, 1},
1839  { 0x05d0, 0x05ea, 1},
1840  { 0x05f0, 0x05f2, 1},
1841  { 0x0621, 0x064a, 1},
1842  { 0x066e, 0x066f, 1},
1843  { 0x0671, 0x06d3, 1},
1844  { 0x06d5, 0x06d5, 1},
1845  { 0x06e5, 0x06e6, 1},
1846  { 0x06ee, 0x06ef, 1},
1847  { 0x06fa, 0x06fc, 1},
1848  { 0x06ff, 0x0710, 17},
1849  { 0x0712, 0x072f, 1},
1850  { 0x074d, 0x07a5, 1},
1851  { 0x07b1, 0x07b1, 1},
1852  { 0x07ca, 0x07ea, 1},
1853  { 0x07f4, 0x07f5, 1},
1854  { 0x07fa, 0x07fa, 1},
1855  { 0x0904, 0x0939, 1},
1856  { 0x093d, 0x0950, 19},
1857  { 0x0958, 0x0961, 1},
1858  { 0x0971, 0x0972, 1},
1859  { 0x097b, 0x097f, 1},
1860  { 0x0985, 0x098c, 1},
1861  { 0x098f, 0x0990, 1},
1862  { 0x0993, 0x09a8, 1},
1863  { 0x09aa, 0x09b0, 1},
1864  { 0x09b2, 0x09b2, 1},
1865  { 0x09b6, 0x09b9, 1},
1866  { 0x09bd, 0x09ce, 17},
1867  { 0x09dc, 0x09dd, 1},
1868  { 0x09df, 0x09e1, 1},
1869  { 0x09f0, 0x09f1, 1},
1870  { 0x0a05, 0x0a0a, 1},
1871  { 0x0a0f, 0x0a10, 1},
1872  { 0x0a13, 0x0a28, 1},
1873  { 0x0a2a, 0x0a30, 1},
1874  { 0x0a32, 0x0a33, 1},
1875  { 0x0a35, 0x0a36, 1},
1876  { 0x0a38, 0x0a39, 1},
1877  { 0x0a59, 0x0a5c, 1},
1878  { 0x0a5e, 0x0a5e, 1},
1879  { 0x0a72, 0x0a74, 1},
1880  { 0x0a85, 0x0a8d, 1},
1881  { 0x0a8f, 0x0a91, 1},
1882  { 0x0a93, 0x0aa8, 1},
1883  { 0x0aaa, 0x0ab0, 1},
1884  { 0x0ab2, 0x0ab3, 1},
1885  { 0x0ab5, 0x0ab9, 1},
1886  { 0x0abd, 0x0ad0, 19},
1887  { 0x0ae0, 0x0ae1, 1},
1888  { 0x0b05, 0x0b0c, 1},
1889  { 0x0b0f, 0x0b10, 1},
1890  { 0x0b13, 0x0b28, 1},
1891  { 0x0b2a, 0x0b30, 1},
1892  { 0x0b32, 0x0b33, 1},
1893  { 0x0b35, 0x0b39, 1},
1894  { 0x0b3d, 0x0b3d, 1},
1895  { 0x0b5c, 0x0b5d, 1},
1896  { 0x0b5f, 0x0b61, 1},
1897  { 0x0b71, 0x0b83, 18},
1898  { 0x0b85, 0x0b8a, 1},
1899  { 0x0b8e, 0x0b90, 1},
1900  { 0x0b92, 0x0b95, 1},
1901  { 0x0b99, 0x0b9a, 1},
1902  { 0x0b9c, 0x0b9c, 1},
1903  { 0x0b9e, 0x0b9f, 1},
1904  { 0x0ba3, 0x0ba4, 1},
1905  { 0x0ba8, 0x0baa, 1},
1906  { 0x0bae, 0x0bb9, 1},
1907  { 0x0bd0, 0x0bd0, 1},
1908  { 0x0c05, 0x0c0c, 1},
1909  { 0x0c0e, 0x0c10, 1},
1910  { 0x0c12, 0x0c28, 1},
1911  { 0x0c2a, 0x0c33, 1},
1912  { 0x0c35, 0x0c39, 1},
1913  { 0x0c3d, 0x0c3d, 1},
1914  { 0x0c58, 0x0c59, 1},
1915  { 0x0c60, 0x0c61, 1},
1916  { 0x0c85, 0x0c8c, 1},
1917  { 0x0c8e, 0x0c90, 1},
1918  { 0x0c92, 0x0ca8, 1},
1919  { 0x0caa, 0x0cb3, 1},
1920  { 0x0cb5, 0x0cb9, 1},
1921  { 0x0cbd, 0x0cde, 33},
1922  { 0x0ce0, 0x0ce1, 1},
1923  { 0x0d05, 0x0d0c, 1},
1924  { 0x0d0e, 0x0d10, 1},
1925  { 0x0d12, 0x0d28, 1},
1926  { 0x0d2a, 0x0d39, 1},
1927  { 0x0d3d, 0x0d3d, 1},
1928  { 0x0d60, 0x0d61, 1},
1929  { 0x0d7a, 0x0d7f, 1},
1930  { 0x0d85, 0x0d96, 1},
1931  { 0x0d9a, 0x0db1, 1},
1932  { 0x0db3, 0x0dbb, 1},
1933  { 0x0dbd, 0x0dbd, 1},
1934  { 0x0dc0, 0x0dc6, 1},
1935  { 0x0e01, 0x0e30, 1},
1936  { 0x0e32, 0x0e33, 1},
1937  { 0x0e40, 0x0e46, 1},
1938  { 0x0e81, 0x0e82, 1},
1939  { 0x0e84, 0x0e84, 1},
1940  { 0x0e87, 0x0e88, 1},
1941  { 0x0e8a, 0x0e8d, 3},
1942  { 0x0e94, 0x0e97, 1},
1943  { 0x0e99, 0x0e9f, 1},
1944  { 0x0ea1, 0x0ea3, 1},
1945  { 0x0ea5, 0x0ea7, 2},
1946  { 0x0eaa, 0x0eab, 1},
1947  { 0x0ead, 0x0eb0, 1},
1948  { 0x0eb2, 0x0eb3, 1},
1949  { 0x0ebd, 0x0ebd, 1},
1950  { 0x0ec0, 0x0ec4, 1},
1951  { 0x0ec6, 0x0ec6, 1},
1952  { 0x0edc, 0x0edd, 1},
1953  { 0x0f00, 0x0f00, 1},
1954  { 0x0f40, 0x0f47, 1},
1955  { 0x0f49, 0x0f6c, 1},
1956  { 0x0f88, 0x0f8b, 1},
1957  { 0x1000, 0x102a, 1},
1958  { 0x103f, 0x103f, 1},
1959  { 0x1050, 0x1055, 1},
1960  { 0x105a, 0x105d, 1},
1961  { 0x1061, 0x1061, 1},
1962  { 0x1065, 0x1066, 1},
1963  { 0x106e, 0x1070, 1},
1964  { 0x1075, 0x1081, 1},
1965  { 0x108e, 0x108e, 1},
1966  { 0x10a0, 0x10c5, 1},
1967  { 0x10d0, 0x10fa, 1},
1968  { 0x10fc, 0x10fc, 1},
1969  { 0x1100, 0x1159, 1},
1970  { 0x115f, 0x11a2, 1},
1971  { 0x11a8, 0x11f9, 1},
1972  { 0x1200, 0x1248, 1},
1973  { 0x124a, 0x124d, 1},
1974  { 0x1250, 0x1256, 1},
1975  { 0x1258, 0x1258, 1},
1976  { 0x125a, 0x125d, 1},
1977  { 0x1260, 0x1288, 1},
1978  { 0x128a, 0x128d, 1},
1979  { 0x1290, 0x12b0, 1},
1980  { 0x12b2, 0x12b5, 1},
1981  { 0x12b8, 0x12be, 1},
1982  { 0x12c0, 0x12c0, 1},
1983  { 0x12c2, 0x12c5, 1},
1984  { 0x12c8, 0x12d6, 1},
1985  { 0x12d8, 0x1310, 1},
1986  { 0x1312, 0x1315, 1},
1987  { 0x1318, 0x135a, 1},
1988  { 0x1380, 0x138f, 1},
1989  { 0x13a0, 0x13f4, 1},
1990  { 0x1401, 0x166c, 1},
1991  { 0x166f, 0x1676, 1},
1992  { 0x1681, 0x169a, 1},
1993  { 0x16a0, 0x16ea, 1},
1994  { 0x1700, 0x170c, 1},
1995  { 0x170e, 0x1711, 1},
1996  { 0x1720, 0x1731, 1},
1997  { 0x1740, 0x1751, 1},
1998  { 0x1760, 0x176c, 1},
1999  { 0x176e, 0x1770, 1},
2000  { 0x1780, 0x17b3, 1},
2001  { 0x17d7, 0x17dc, 5},
2002  { 0x1820, 0x1877, 1},
2003  { 0x1880, 0x18a8, 1},
2004  { 0x18aa, 0x18aa, 1},
2005  { 0x1900, 0x191c, 1},
2006  { 0x1950, 0x196d, 1},
2007  { 0x1970, 0x1974, 1},
2008  { 0x1980, 0x19a9, 1},
2009  { 0x19c1, 0x19c7, 1},
2010  { 0x1a00, 0x1a16, 1},
2011  { 0x1b05, 0x1b33, 1},
2012  { 0x1b45, 0x1b4b, 1},
2013  { 0x1b83, 0x1ba0, 1},
2014  { 0x1bae, 0x1baf, 1},
2015  { 0x1c00, 0x1c23, 1},
2016  { 0x1c4d, 0x1c4f, 1},
2017  { 0x1c5a, 0x1c7d, 1},
2018  { 0x1d00, 0x1dbf, 1},
2019  { 0x1e00, 0x1f15, 1},
2020  { 0x1f18, 0x1f1d, 1},
2021  { 0x1f20, 0x1f45, 1},
2022  { 0x1f48, 0x1f4d, 1},
2023  { 0x1f50, 0x1f57, 1},
2024  { 0x1f59, 0x1f5d, 2},
2025  { 0x1f5f, 0x1f7d, 1},
2026  { 0x1f80, 0x1fb4, 1},
2027  { 0x1fb6, 0x1fbc, 1},
2028  { 0x1fbe, 0x1fbe, 1},
2029  { 0x1fc2, 0x1fc4, 1},
2030  { 0x1fc6, 0x1fcc, 1},
2031  { 0x1fd0, 0x1fd3, 1},
2032  { 0x1fd6, 0x1fdb, 1},
2033  { 0x1fe0, 0x1fec, 1},
2034  { 0x1ff2, 0x1ff4, 1},
2035  { 0x1ff6, 0x1ffc, 1},
2036  { 0x2071, 0x207f, 14},
2037  { 0x2090, 0x2094, 1},
2038  { 0x2102, 0x2107, 5},
2039  { 0x210a, 0x2113, 1},
2040  { 0x2115, 0x2115, 1},
2041  { 0x2119, 0x211d, 1},
2042  { 0x2124, 0x2128, 2},
2043  { 0x212a, 0x212d, 1},
2044  { 0x212f, 0x2139, 1},
2045  { 0x213c, 0x213f, 1},
2046  { 0x2145, 0x2149, 1},
2047  { 0x214e, 0x214e, 1},
2048  { 0x2183, 0x2184, 1},
2049  { 0x2c00, 0x2c2e, 1},
2050  { 0x2c30, 0x2c5e, 1},
2051  { 0x2c60, 0x2c6f, 1},
2052  { 0x2c71, 0x2c7d, 1},
2053  { 0x2c80, 0x2ce4, 1},
2054  { 0x2d00, 0x2d25, 1},
2055  { 0x2d30, 0x2d65, 1},
2056  { 0x2d6f, 0x2d6f, 1},
2057  { 0x2d80, 0x2d96, 1},
2058  { 0x2da0, 0x2da6, 1},
2059  { 0x2da8, 0x2dae, 1},
2060  { 0x2db0, 0x2db6, 1},
2061  { 0x2db8, 0x2dbe, 1},
2062  { 0x2dc0, 0x2dc6, 1},
2063  { 0x2dc8, 0x2dce, 1},
2064  { 0x2dd0, 0x2dd6, 1},
2065  { 0x2dd8, 0x2dde, 1},
2066  { 0x2e2f, 0x2e2f, 1},
2067  { 0x3005, 0x3006, 1},
2068  { 0x3031, 0x3035, 1},
2069  { 0x303b, 0x303c, 1},
2070  { 0x3041, 0x3096, 1},
2071  { 0x309d, 0x309f, 1},
2072  { 0x30a1, 0x30fa, 1},
2073  { 0x30fc, 0x30ff, 1},
2074  { 0x3105, 0x312d, 1},
2075  { 0x3131, 0x318e, 1},
2076  { 0x31a0, 0x31b7, 1},
2077  { 0x31f0, 0x31ff, 1},
2078  { 0x3400, 0x4db5, 1},
2079  { 0x4e00, 0x9fc3, 1},
2080  { 0xa000, 0xa48c, 1},
2081  { 0xa500, 0xa60c, 1},
2082  { 0xa610, 0xa61f, 1},
2083  { 0xa62a, 0xa62b, 1},
2084  { 0xa640, 0xa65f, 1},
2085  { 0xa662, 0xa66e, 1},
2086  { 0xa67f, 0xa697, 1},
2087  { 0xa717, 0xa71f, 1},
2088  { 0xa722, 0xa788, 1},
2089  { 0xa78b, 0xa78c, 1},
2090  { 0xa7fb, 0xa801, 1},
2091  { 0xa803, 0xa805, 1},
2092  { 0xa807, 0xa80a, 1},
2093  { 0xa80c, 0xa822, 1},
2094  { 0xa840, 0xa873, 1},
2095  { 0xa882, 0xa8b3, 1},
2096  { 0xa90a, 0xa925, 1},
2097  { 0xa930, 0xa946, 1},
2098  { 0xaa00, 0xaa28, 1},
2099  { 0xaa40, 0xaa42, 1},
2100  { 0xaa44, 0xaa4b, 1},
2101  { 0xac00, 0xd7a3, 1},
2102  { 0xf900, 0xfa2d, 1},
2103  { 0xfa30, 0xfa6a, 1},
2104  { 0xfa70, 0xfad9, 1},
2105  { 0xfb00, 0xfb06, 1},
2106  { 0xfb13, 0xfb17, 1},
2107  { 0xfb1d, 0xfb1d, 1},
2108  { 0xfb1f, 0xfb28, 1},
2109  { 0xfb2a, 0xfb36, 1},
2110  { 0xfb38, 0xfb3c, 1},
2111  { 0xfb3e, 0xfb3e, 1},
2112  { 0xfb40, 0xfb41, 1},
2113  { 0xfb43, 0xfb44, 1},
2114  { 0xfb46, 0xfbb1, 1},
2115  { 0xfbd3, 0xfd3d, 1},
2116  { 0xfd50, 0xfd8f, 1},
2117  { 0xfd92, 0xfdc7, 1},
2118  { 0xfdf0, 0xfdfb, 1},
2119  { 0xfe70, 0xfe74, 1},
2120  { 0xfe76, 0xfefc, 1},
2121  { 0xff21, 0xff3a, 1},
2122  { 0xff41, 0xff5a, 1},
2123  { 0xff66, 0xffbe, 1},
2124  { 0xffc2, 0xffc7, 1},
2125  { 0xffca, 0xffcf, 1},
2126  { 0xffd2, 0xffd7, 1},
2127  { 0xffda, 0xffdc, 1},
2128  { 0x10000, 0x1000b, 1},
2129  { 0x1000d, 0x10026, 1},
2130  { 0x10028, 0x1003a, 1},
2131  { 0x1003c, 0x1003d, 1},
2132  { 0x1003f, 0x1004d, 1},
2133  { 0x10050, 0x1005d, 1},
2134  { 0x10080, 0x100fa, 1},
2135  { 0x10280, 0x1029c, 1},
2136  { 0x102a0, 0x102d0, 1},
2137  { 0x10300, 0x1031e, 1},
2138  { 0x10330, 0x10340, 1},
2139  { 0x10342, 0x10349, 1},
2140  { 0x10380, 0x1039d, 1},
2141  { 0x103a0, 0x103c3, 1},
2142  { 0x103c8, 0x103cf, 1},
2143  { 0x10400, 0x1049d, 1},
2144  { 0x10800, 0x10805, 1},
2145  { 0x10808, 0x10808, 1},
2146  { 0x1080a, 0x10835, 1},
2147  { 0x10837, 0x10838, 1},
2148  { 0x1083c, 0x1083f, 3},
2149  { 0x10900, 0x10915, 1},
2150  { 0x10920, 0x10939, 1},
2151  { 0x10a00, 0x10a00, 1},
2152  { 0x10a10, 0x10a13, 1},
2153  { 0x10a15, 0x10a17, 1},
2154  { 0x10a19, 0x10a33, 1},
2155  { 0x12000, 0x1236e, 1},
2156  { 0x1d400, 0x1d454, 1},
2157  { 0x1d456, 0x1d49c, 1},
2158  { 0x1d49e, 0x1d49f, 1},
2159  { 0x1d4a2, 0x1d4a2, 1},
2160  { 0x1d4a5, 0x1d4a6, 1},
2161  { 0x1d4a9, 0x1d4ac, 1},
2162  { 0x1d4ae, 0x1d4b9, 1},
2163  { 0x1d4bb, 0x1d4bb, 1},
2164  { 0x1d4bd, 0x1d4c3, 1},
2165  { 0x1d4c5, 0x1d505, 1},
2166  { 0x1d507, 0x1d50a, 1},
2167  { 0x1d50d, 0x1d514, 1},
2168  { 0x1d516, 0x1d51c, 1},
2169  { 0x1d51e, 0x1d539, 1},
2170  { 0x1d53b, 0x1d53e, 1},
2171  { 0x1d540, 0x1d544, 1},
2172  { 0x1d546, 0x1d546, 1},
2173  { 0x1d54a, 0x1d550, 1},
2174  { 0x1d552, 0x1d6a5, 1},
2175  { 0x1d6a8, 0x1d6c0, 1},
2176  { 0x1d6c2, 0x1d6da, 1},
2177  { 0x1d6dc, 0x1d6fa, 1},
2178  { 0x1d6fc, 0x1d714, 1},
2179  { 0x1d716, 0x1d734, 1},
2180  { 0x1d736, 0x1d74e, 1},
2181  { 0x1d750, 0x1d76e, 1},
2182  { 0x1d770, 0x1d788, 1},
2183  { 0x1d78a, 0x1d7a8, 1},
2184  { 0x1d7aa, 0x1d7c2, 1},
2185  { 0x1d7c4, 0x1d7cb, 1},
2186  { 0x20000, 0x2a6d6, 1},
2187  { 0x2f800, 0x2fa1d, 1},
2188};
2189
2190// A table of Unicode uppercase letters--Unicode code points
2191// classified as "Letter, uppercase".
2192
2193static const Unicode_range unicode_uppercase_letters[] =
2194{
2195  { 0x0041, 0x005a, 1},
2196  { 0x00c0, 0x00d6, 1},
2197  { 0x00d8, 0x00de, 1},
2198  { 0x0100, 0x0136, 2},
2199  { 0x0139, 0x0147, 2},
2200  { 0x014a, 0x0176, 2},
2201  { 0x0178, 0x0179, 1},
2202  { 0x017b, 0x017d, 2},
2203  { 0x0181, 0x0182, 1},
2204  { 0x0184, 0x0184, 1},
2205  { 0x0186, 0x0187, 1},
2206  { 0x0189, 0x018b, 1},
2207  { 0x018e, 0x0191, 1},
2208  { 0x0193, 0x0194, 1},
2209  { 0x0196, 0x0198, 1},
2210  { 0x019c, 0x019d, 1},
2211  { 0x019f, 0x01a0, 1},
2212  { 0x01a2, 0x01a4, 2},
2213  { 0x01a6, 0x01a7, 1},
2214  { 0x01a9, 0x01ac, 3},
2215  { 0x01ae, 0x01af, 1},
2216  { 0x01b1, 0x01b3, 1},
2217  { 0x01b5, 0x01b5, 1},
2218  { 0x01b7, 0x01b8, 1},
2219  { 0x01bc, 0x01c4, 8},
2220  { 0x01c7, 0x01cd, 3},
2221  { 0x01cf, 0x01db, 2},
2222  { 0x01de, 0x01ee, 2},
2223  { 0x01f1, 0x01f4, 3},
2224  { 0x01f6, 0x01f8, 1},
2225  { 0x01fa, 0x0232, 2},
2226  { 0x023a, 0x023b, 1},
2227  { 0x023d, 0x023e, 1},
2228  { 0x0241, 0x0241, 1},
2229  { 0x0243, 0x0246, 1},
2230  { 0x0248, 0x024e, 2},
2231  { 0x0370, 0x0372, 2},
2232  { 0x0376, 0x0386, 16},
2233  { 0x0388, 0x038a, 1},
2234  { 0x038c, 0x038c, 1},
2235  { 0x038e, 0x038f, 1},
2236  { 0x0391, 0x03a1, 1},
2237  { 0x03a3, 0x03ab, 1},
2238  { 0x03cf, 0x03cf, 1},
2239  { 0x03d2, 0x03d4, 1},
2240  { 0x03d8, 0x03ee, 2},
2241  { 0x03f4, 0x03f7, 3},
2242  { 0x03f9, 0x03fa, 1},
2243  { 0x03fd, 0x042f, 1},
2244  { 0x0460, 0x0480, 2},
2245  { 0x048a, 0x04be, 2},
2246  { 0x04c0, 0x04c1, 1},
2247  { 0x04c3, 0x04cd, 2},
2248  { 0x04d0, 0x0522, 2},
2249  { 0x0531, 0x0556, 1},
2250  { 0x10a0, 0x10c5, 1},
2251  { 0x1e00, 0x1e94, 2},
2252  { 0x1e9e, 0x1efe, 2},
2253  { 0x1f08, 0x1f0f, 1},
2254  { 0x1f18, 0x1f1d, 1},
2255  { 0x1f28, 0x1f2f, 1},
2256  { 0x1f38, 0x1f3f, 1},
2257  { 0x1f48, 0x1f4d, 1},
2258  { 0x1f59, 0x1f5f, 2},
2259  { 0x1f68, 0x1f6f, 1},
2260  { 0x1fb8, 0x1fbb, 1},
2261  { 0x1fc8, 0x1fcb, 1},
2262  { 0x1fd8, 0x1fdb, 1},
2263  { 0x1fe8, 0x1fec, 1},
2264  { 0x1ff8, 0x1ffb, 1},
2265  { 0x2102, 0x2107, 5},
2266  { 0x210b, 0x210d, 1},
2267  { 0x2110, 0x2112, 1},
2268  { 0x2115, 0x2115, 1},
2269  { 0x2119, 0x211d, 1},
2270  { 0x2124, 0x2128, 2},
2271  { 0x212a, 0x212d, 1},
2272  { 0x2130, 0x2133, 1},
2273  { 0x213e, 0x213f, 1},
2274  { 0x2145, 0x2183, 62},
2275  { 0x2c00, 0x2c2e, 1},
2276  { 0x2c60, 0x2c60, 1},
2277  { 0x2c62, 0x2c64, 1},
2278  { 0x2c67, 0x2c6b, 2},
2279  { 0x2c6d, 0x2c6f, 1},
2280  { 0x2c72, 0x2c75, 3},
2281  { 0x2c80, 0x2ce2, 2},
2282  { 0xa640, 0xa65e, 2},
2283  { 0xa662, 0xa66c, 2},
2284  { 0xa680, 0xa696, 2},
2285  { 0xa722, 0xa72e, 2},
2286  { 0xa732, 0xa76e, 2},
2287  { 0xa779, 0xa77b, 2},
2288  { 0xa77d, 0xa77e, 1},
2289  { 0xa780, 0xa786, 2},
2290  { 0xa78b, 0xa78b, 1},
2291  { 0xff21, 0xff3a, 1},
2292  { 0x10400, 0x10427, 1},
2293  { 0x1d400, 0x1d419, 1},
2294  { 0x1d434, 0x1d44d, 1},
2295  { 0x1d468, 0x1d481, 1},
2296  { 0x1d49c, 0x1d49c, 1},
2297  { 0x1d49e, 0x1d49f, 1},
2298  { 0x1d4a2, 0x1d4a2, 1},
2299  { 0x1d4a5, 0x1d4a6, 1},
2300  { 0x1d4a9, 0x1d4ac, 1},
2301  { 0x1d4ae, 0x1d4b5, 1},
2302  { 0x1d4d0, 0x1d4e9, 1},
2303  { 0x1d504, 0x1d505, 1},
2304  { 0x1d507, 0x1d50a, 1},
2305  { 0x1d50d, 0x1d514, 1},
2306  { 0x1d516, 0x1d51c, 1},
2307  { 0x1d538, 0x1d539, 1},
2308  { 0x1d53b, 0x1d53e, 1},
2309  { 0x1d540, 0x1d544, 1},
2310  { 0x1d546, 0x1d546, 1},
2311  { 0x1d54a, 0x1d550, 1},
2312  { 0x1d56c, 0x1d585, 1},
2313  { 0x1d5a0, 0x1d5b9, 1},
2314  { 0x1d5d4, 0x1d5ed, 1},
2315  { 0x1d608, 0x1d621, 1},
2316  { 0x1d63c, 0x1d655, 1},
2317  { 0x1d670, 0x1d689, 1},
2318  { 0x1d6a8, 0x1d6c0, 1},
2319  { 0x1d6e2, 0x1d6fa, 1},
2320  { 0x1d71c, 0x1d734, 1},
2321  { 0x1d756, 0x1d76e, 1},
2322  { 0x1d790, 0x1d7a8, 1},
2323  { 0x1d7ca, 0x1d7ca, 1},
2324};
2325
2326// Return true if C is in RANGES.
2327
2328bool
2329Lex::is_in_unicode_range(unsigned int c, const Unicode_range* ranges,
2330			 size_t range_size)
2331{
2332  if (c < 0x100)
2333    {
2334      // The common case is a small value, and we know that it will be
2335      // in the first few entries of the table.  Do a linear scan
2336      // rather than a binary search.
2337      for (size_t i = 0; i < range_size; ++i)
2338	{
2339	  const Unicode_range* p = &ranges[i];
2340	  if (c <= p->high)
2341	    {
2342	      if (c < p->low)
2343		return false;
2344	      return (c - p->low) % p->stride == 0;
2345	    }
2346	}
2347      return false;
2348    }
2349  else
2350    {
2351      size_t lo = 0;
2352      size_t hi = range_size;
2353      while (lo < hi)
2354	{
2355	  size_t mid = lo + (hi - lo) / 2;
2356	  const Unicode_range* p = &ranges[mid];
2357	  if (c < p->low)
2358	    hi = mid;
2359	  else if (c > p->high)
2360	    lo = mid + 1;
2361	  else
2362	    return (c - p->low) % p->stride == 0;
2363	}
2364      return false;
2365    }
2366}
2367
2368// Return whether C is a space character.
2369
2370bool
2371Lex::is_unicode_space(unsigned int c)
2372{
2373  return Lex::is_in_unicode_range(c, unicode_space,
2374				  ARRAY_SIZE(unicode_space));
2375}
2376
2377// Return whether C is a Unicode digit--a Unicode code point
2378// classified as "Digit".
2379
2380bool
2381Lex::is_unicode_digit(unsigned int c)
2382{
2383  return Lex::is_in_unicode_range(c, unicode_digits,
2384				  ARRAY_SIZE(unicode_digits));
2385}
2386
2387// Return whether C is a Unicode letter--a Unicode code point
2388// classified as "Letter".
2389
2390bool
2391Lex::is_unicode_letter(unsigned int c)
2392{
2393  return Lex::is_in_unicode_range(c, unicode_letters,
2394				  ARRAY_SIZE(unicode_letters));
2395}
2396
2397// Return whether C is a Unicode uppercase letter.  a Unicode code
2398// point classified as "Letter, uppercase".
2399
2400bool
2401Lex::is_unicode_uppercase(unsigned int c)
2402{
2403  return Lex::is_in_unicode_range(c, unicode_uppercase_letters,
2404				  ARRAY_SIZE(unicode_uppercase_letters));
2405}
2406
2407// Return whether the identifier NAME should be exported.  NAME is a
2408// mangled name which includes only ASCII characters.
2409
2410bool
2411Lex::is_exported_name(const std::string& name)
2412{
2413  unsigned char c = name[0];
2414  if (c != '$')
2415    return c >= 'A' && c <= 'Z';
2416  else
2417    {
2418      const char* p = name.data();
2419      size_t len = name.length();
2420      if (len < 2 || p[1] != 'U')
2421	return false;
2422      unsigned int ci = 0;
2423      for (size_t i = 2; i < len && p[i] != '$'; ++i)
2424	{
2425	  c = p[i];
2426	  if (!hex_p(c))
2427	    return false;
2428	  ci <<= 4;
2429	  ci |= hex_value(c);
2430	}
2431      return Lex::is_unicode_uppercase(ci);
2432    }
2433}
2434
2435// Return whether the identifier NAME contains an invalid character.
2436// This is based on how we handle invalid characters in
2437// gather_identifier.
2438
2439bool
2440Lex::is_invalid_identifier(const std::string& name)
2441{
2442  return name.find("$INVALID$") != std::string::npos;
2443}
2444