compiler: Give error for byte-order-mark in middle of file.
[gcc.git] / gcc / go / gofrontend / lex.cc
1 // lex.cc -- Go frontend lexer.
2
3 // Copyright 2009 The Go Authors. All rights reserved.
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file.
6
7 #include "go-system.h"
8
9 #include "lex.h"
10
11 // Manage mapping from keywords to the Keyword codes.
12
13 class Keywords
14 {
15 public:
16 // The structure which maps keywords to codes.
17 struct Mapping
18 {
19 // Keyword string.
20 const char* keystring;
21 // Keyword code.
22 Keyword keycode;
23 };
24
25 // Return the parsecode corresponding to KEYSTRING, or
26 // KEYWORD_INVALID if it is not a keyword.
27 Keyword
28 keyword_to_code(const char* keyword, size_t len) const;
29
30 // Return the string for a keyword.
31 const char*
32 keyword_to_string(Keyword) const;
33
34 private:
35 static const Mapping mapping_[];
36 static const int count_;
37 };
38
39 // Mapping from keyword string to keyword code. This array must be
40 // kept in sorted order, and the order must match the Keyword enum.
41 // Strings are looked up using bsearch.
42
43 const Keywords::Mapping
44 Keywords::mapping_[] =
45 {
46 { NULL, KEYWORD_INVALID },
47 { "__asm__", KEYWORD_ASM },
48 { "break", KEYWORD_BREAK },
49 { "case", KEYWORD_CASE },
50 { "chan", KEYWORD_CHAN },
51 { "const", KEYWORD_CONST },
52 { "continue", KEYWORD_CONTINUE },
53 { "default", KEYWORD_DEFAULT },
54 { "defer", KEYWORD_DEFER },
55 { "else", KEYWORD_ELSE },
56 { "fallthrough", KEYWORD_FALLTHROUGH },
57 { "for", KEYWORD_FOR },
58 { "func", KEYWORD_FUNC },
59 { "go", KEYWORD_GO },
60 { "goto", KEYWORD_GOTO },
61 { "if", KEYWORD_IF },
62 { "import", KEYWORD_IMPORT },
63 { "interface", KEYWORD_INTERFACE },
64 { "map", KEYWORD_MAP },
65 { "package", KEYWORD_PACKAGE },
66 { "range", KEYWORD_RANGE },
67 { "return", KEYWORD_RETURN },
68 { "select", KEYWORD_SELECT },
69 { "struct", KEYWORD_STRUCT },
70 { "switch", KEYWORD_SWITCH },
71 { "type", KEYWORD_TYPE },
72 { "var", KEYWORD_VAR }
73 };
74
75 // Number of entries in the map.
76
77 const int Keywords::count_ =
78 sizeof(Keywords::mapping_) / sizeof(Keywords::mapping_[0]);
79
80 // Comparison function passed to bsearch.
81
82 extern "C"
83 {
84
85 struct Keywords_search_key
86 {
87 const char* str;
88 size_t len;
89 };
90
91 static int
92 keyword_compare(const void* keyv, const void* mapv)
93 {
94 const Keywords_search_key* key =
95 static_cast<const Keywords_search_key*>(keyv);
96 const Keywords::Mapping* map =
97 static_cast<const Keywords::Mapping*>(mapv);
98 if (map->keystring == NULL)
99 return 1;
100 int i = strncmp(key->str, map->keystring, key->len);
101 if (i != 0)
102 return i;
103 if (map->keystring[key->len] != '\0')
104 return -1;
105 return 0;
106 }
107
108 } // End extern "C".
109
110 // Convert a string to a keyword code. Return KEYWORD_INVALID if the
111 // string is not a keyword.
112
113 Keyword
114 Keywords::keyword_to_code(const char* keyword, size_t len) const
115 {
116 Keywords_search_key key;
117 key.str = keyword;
118 key.len = len;
119 void* mapv = bsearch(&key,
120 this->mapping_,
121 this->count_,
122 sizeof(this->mapping_[0]),
123 keyword_compare);
124 if (mapv == NULL)
125 return KEYWORD_INVALID;
126 Mapping* map = static_cast<Mapping*>(mapv);
127 return map->keycode;
128 }
129
130 // Convert a keyword code to a string.
131
132 const char*
133 Keywords::keyword_to_string(Keyword code) const
134 {
135 go_assert(code > KEYWORD_INVALID && code < this->count_);
136 const Mapping* map = &this->mapping_[code];
137 go_assert(map->keycode == code);
138 return map->keystring;
139 }
140
141 // There is one instance of the Keywords class.
142
143 static Keywords keywords;
144
145 // Class Token.
146
147 // Make a general token.
148
149 Token::Token(Classification classification, Location location)
150 : classification_(classification), location_(location)
151 {
152 }
153
154 // Destroy a token.
155
156 Token::~Token()
157 {
158 this->clear();
159 }
160
161 // Clear a token--release memory.
162
163 void
164 Token::clear()
165 {
166 if (this->classification_ == TOKEN_INTEGER
167 || this->classification_ == TOKEN_CHARACTER)
168 mpz_clear(this->u_.integer_value);
169 else if (this->classification_ == TOKEN_FLOAT
170 || this->classification_ == TOKEN_IMAGINARY)
171 mpfr_clear(this->u_.float_value);
172 }
173
174 // Construct a token.
175
176 Token::Token(const Token& tok)
177 : classification_(tok.classification_), location_(tok.location_)
178 {
179 switch (this->classification_)
180 {
181 case TOKEN_INVALID:
182 case TOKEN_EOF:
183 break;
184 case TOKEN_KEYWORD:
185 this->u_.keyword = tok.u_.keyword;
186 break;
187 case TOKEN_IDENTIFIER:
188 case TOKEN_STRING:
189 this->u_.string_value = tok.u_.string_value;
190 break;
191 case TOKEN_OPERATOR:
192 this->u_.op = tok.u_.op;
193 break;
194 case TOKEN_CHARACTER:
195 case TOKEN_INTEGER:
196 mpz_init_set(this->u_.integer_value, tok.u_.integer_value);
197 break;
198 case TOKEN_FLOAT:
199 case TOKEN_IMAGINARY:
200 mpfr_init_set(this->u_.float_value, tok.u_.float_value, GMP_RNDN);
201 break;
202 default:
203 go_unreachable();
204 }
205 }
206
207 // Assign to a token.
208
209 Token&
210 Token::operator=(const Token& tok)
211 {
212 this->clear();
213 this->classification_ = tok.classification_;
214 this->location_ = tok.location_;
215 switch (tok.classification_)
216 {
217 case TOKEN_INVALID:
218 case TOKEN_EOF:
219 break;
220 case TOKEN_KEYWORD:
221 this->u_.keyword = tok.u_.keyword;
222 break;
223 case TOKEN_IDENTIFIER:
224 this->u_.identifier_value.name = tok.u_.identifier_value.name;
225 this->u_.identifier_value.is_exported =
226 tok.u_.identifier_value.is_exported;
227 break;
228 case TOKEN_STRING:
229 this->u_.string_value = tok.u_.string_value;
230 break;
231 case TOKEN_OPERATOR:
232 this->u_.op = tok.u_.op;
233 break;
234 case TOKEN_CHARACTER:
235 case TOKEN_INTEGER:
236 mpz_init_set(this->u_.integer_value, tok.u_.integer_value);
237 break;
238 case TOKEN_FLOAT:
239 case TOKEN_IMAGINARY:
240 mpfr_init_set(this->u_.float_value, tok.u_.float_value, GMP_RNDN);
241 break;
242 default:
243 go_unreachable();
244 }
245 return *this;
246 }
247
248 // Print the token for debugging.
249
250 void
251 Token::print(FILE* file) const
252 {
253 switch (this->classification_)
254 {
255 case TOKEN_INVALID:
256 fprintf(file, "invalid");
257 break;
258 case TOKEN_EOF:
259 fprintf(file, "EOF");
260 break;
261 case TOKEN_KEYWORD:
262 fprintf(file, "keyword %s", keywords.keyword_to_string(this->u_.keyword));
263 break;
264 case TOKEN_IDENTIFIER:
265 fprintf(file, "identifier \"%s\"", this->u_.string_value->c_str());
266 break;
267 case TOKEN_STRING:
268 fprintf(file, "quoted string \"%s\"", this->u_.string_value->c_str());
269 break;
270 case TOKEN_CHARACTER:
271 fprintf(file, "character ");
272 mpz_out_str(file, 10, this->u_.integer_value);
273 break;
274 case TOKEN_INTEGER:
275 fprintf(file, "integer ");
276 mpz_out_str(file, 10, this->u_.integer_value);
277 break;
278 case TOKEN_FLOAT:
279 fprintf(file, "float ");
280 mpfr_out_str(file, 10, 0, this->u_.float_value, GMP_RNDN);
281 break;
282 case TOKEN_IMAGINARY:
283 fprintf(file, "imaginary ");
284 mpfr_out_str(file, 10, 0, this->u_.float_value, GMP_RNDN);
285 break;
286 case TOKEN_OPERATOR:
287 fprintf(file, "operator ");
288 switch (this->u_.op)
289 {
290 case OPERATOR_INVALID:
291 fprintf(file, "invalid");
292 break;
293 case OPERATOR_OROR:
294 fprintf(file, "||");
295 break;
296 case OPERATOR_ANDAND:
297 fprintf(file, "&&");
298 break;
299 case OPERATOR_EQEQ:
300 fprintf(file, "==");
301 break;
302 case OPERATOR_NOTEQ:
303 fprintf(file, "!=");
304 break;
305 case OPERATOR_LT:
306 fprintf(file, "<");
307 break;
308 case OPERATOR_LE:
309 fprintf(file, "<=");
310 break;
311 case OPERATOR_GT:
312 fprintf(file, ">");
313 break;
314 case OPERATOR_GE:
315 fprintf(file, ">=");
316 break;
317 case OPERATOR_PLUS:
318 fprintf(file, "+");
319 break;
320 case OPERATOR_MINUS:
321 fprintf(file, "-");
322 break;
323 case OPERATOR_OR:
324 fprintf(file, "|");
325 break;
326 case OPERATOR_XOR:
327 fprintf(file, "^");
328 break;
329 case OPERATOR_MULT:
330 fprintf(file, "*");
331 break;
332 case OPERATOR_DIV:
333 fprintf(file, "/");
334 break;
335 case OPERATOR_MOD:
336 fprintf(file, "%%");
337 break;
338 case OPERATOR_LSHIFT:
339 fprintf(file, "<<");
340 break;
341 case OPERATOR_RSHIFT:
342 fprintf(file, ">>");
343 break;
344 case OPERATOR_AND:
345 fprintf(file, "&");
346 break;
347 case OPERATOR_BITCLEAR:
348 fprintf(file, "&^");
349 break;
350 case OPERATOR_NOT:
351 fprintf(file, "!");
352 break;
353 case OPERATOR_CHANOP:
354 fprintf(file, "<-");
355 break;
356 case OPERATOR_EQ:
357 fprintf(file, "=");
358 break;
359 case OPERATOR_PLUSEQ:
360 fprintf(file, "+=");
361 break;
362 case OPERATOR_MINUSEQ:
363 fprintf(file, "-=");
364 break;
365 case OPERATOR_OREQ:
366 fprintf(file, "|=");
367 break;
368 case OPERATOR_XOREQ:
369 fprintf(file, "^=");
370 break;
371 case OPERATOR_MULTEQ:
372 fprintf(file, "*=");
373 break;
374 case OPERATOR_DIVEQ:
375 fprintf(file, "/=");
376 break;
377 case OPERATOR_MODEQ:
378 fprintf(file, "%%=");
379 break;
380 case OPERATOR_LSHIFTEQ:
381 fprintf(file, "<<=");
382 break;
383 case OPERATOR_RSHIFTEQ:
384 fprintf(file, ">>=");
385 break;
386 case OPERATOR_ANDEQ:
387 fprintf(file, "&=");
388 break;
389 case OPERATOR_BITCLEAREQ:
390 fprintf(file, "&^=");
391 break;
392 case OPERATOR_PLUSPLUS:
393 fprintf(file, "++");
394 break;
395 case OPERATOR_MINUSMINUS:
396 fprintf(file, "--");
397 break;
398 case OPERATOR_COLON:
399 fprintf(file, ":");
400 break;
401 case OPERATOR_COLONEQ:
402 fprintf(file, ":=");
403 break;
404 case OPERATOR_SEMICOLON:
405 fprintf(file, ";");
406 break;
407 case OPERATOR_DOT:
408 fprintf(file, ".");
409 break;
410 case OPERATOR_COMMA:
411 fprintf(file, ",");
412 break;
413 case OPERATOR_LPAREN:
414 fprintf(file, "(");
415 break;
416 case OPERATOR_RPAREN:
417 fprintf(file, ")");
418 break;
419 case OPERATOR_LCURLY:
420 fprintf(file, "{");
421 break;
422 case OPERATOR_RCURLY:
423 fprintf(file, "}");
424 break;
425 case OPERATOR_LSQUARE:
426 fprintf(file, "[");
427 break;
428 case OPERATOR_RSQUARE:
429 fprintf(file, "]");
430 break;
431 default:
432 go_unreachable();
433 }
434 break;
435 default:
436 go_unreachable();
437 }
438 }
439
440 // Class Lex.
441
442 Lex::Lex(const char* input_file_name, FILE* input_file, Linemap* linemap)
443 : input_file_name_(input_file_name), input_file_(input_file),
444 linemap_(linemap), linebuf_(NULL), linebufsize_(120), linesize_(0),
445 lineoff_(0), lineno_(0), add_semi_at_eol_(false), extern_()
446 {
447 this->linebuf_ = new char[this->linebufsize_];
448 this->linemap_->start_file(input_file_name, 0);
449 }
450
451 Lex::~Lex()
452 {
453 delete[] this->linebuf_;
454 }
455
456 // Read a new line from the file.
457
458 ssize_t
459 Lex::get_line()
460 {
461 char* buf = this->linebuf_;
462 size_t size = this->linebufsize_;
463
464 FILE* file = this->input_file_;
465 size_t cur = 0;
466 while (true)
467 {
468 int c = getc(file);
469 if (c == EOF)
470 {
471 if (cur == 0)
472 return -1;
473 break;
474 }
475 if (cur + 1 >= size)
476 {
477 size_t ns = 2 * size + 1;
478 if (ns < size || static_cast<ssize_t>(ns) < 0)
479 error_at(this->location(), "out of memory");
480 char* nb = new char[ns];
481 memcpy(nb, buf, cur);
482 delete[] buf;
483 buf = nb;
484 size = ns;
485 }
486 buf[cur] = c;
487 ++cur;
488
489 if (c == '\n')
490 break;
491 }
492
493 buf[cur] = '\0';
494
495 this->linebuf_ = buf;
496 this->linebufsize_ = size;
497
498 return cur;
499 }
500
501 // See if we need to read a new line. Return true if there is a new
502 // line, false if we are at EOF.
503
504 bool
505 Lex::require_line()
506 {
507 if (this->lineoff_ < this->linesize_)
508 return true;
509
510 ssize_t got = this->get_line();
511 if (got < 0)
512 return false;
513 ++this->lineno_;
514 this->linesize_= got;
515 this->lineoff_ = 0;
516
517 this->linemap_->start_line(this->lineno_, this->linesize_);
518
519 return true;
520 }
521
522 // Get the current location.
523
524 Location
525 Lex::location() const
526 {
527 return this->linemap_->get_location(this->lineoff_ + 1);
528 }
529
530 // Get a location slightly before the current one. This is used for
531 // slightly more efficient handling of operator tokens.
532
533 Location
534 Lex::earlier_location(int chars) const
535 {
536 return this->linemap_->get_location(this->lineoff_ + 1 - chars);
537 }
538
539 // Get the next token.
540
541 Token
542 Lex::next_token()
543 {
544 bool saw_cpp_comment = false;
545 while (true)
546 {
547 if (!this->require_line())
548 {
549 bool add_semi_at_eol = this->add_semi_at_eol_;
550 this->add_semi_at_eol_ = false;
551 if (add_semi_at_eol)
552 return this->make_operator(OPERATOR_SEMICOLON, 1);
553 return this->make_eof_token();
554 }
555
556 if (!saw_cpp_comment)
557 this->extern_.clear();
558 saw_cpp_comment = false;
559
560 const char* p = this->linebuf_ + this->lineoff_;
561 const char* pend = this->linebuf_ + this->linesize_;
562
563 while (p < pend)
564 {
565 unsigned char cc = *p;
566 switch (cc)
567 {
568 case ' ': case '\t': case '\r':
569 ++p;
570 // Skip whitespace quickly.
571 while (*p == ' ' || *p == '\t' || *p == '\r')
572 ++p;
573 break;
574
575 case '\n':
576 {
577 ++p;
578 bool add_semi_at_eol = this->add_semi_at_eol_;
579 this->add_semi_at_eol_ = false;
580 if (add_semi_at_eol)
581 {
582 this->lineoff_ = p - this->linebuf_;
583 return this->make_operator(OPERATOR_SEMICOLON, 1);
584 }
585 }
586 break;
587
588 case '/':
589 if (p[1] == '/')
590 {
591 this->lineoff_ = p + 2 - this->linebuf_;
592 this->skip_cpp_comment();
593 p = pend;
594 if (p[-1] == '\n' && this->add_semi_at_eol_)
595 --p;
596 saw_cpp_comment = true;
597 }
598 else if (p[1] == '*')
599 {
600 this->lineoff_ = p - this->linebuf_;
601 Location location = this->location();
602 if (!this->skip_c_comment())
603 return Token::make_invalid_token(location);
604 p = this->linebuf_ + this->lineoff_;
605 pend = this->linebuf_ + this->linesize_;
606 }
607 else if (p[1] == '=')
608 {
609 this->add_semi_at_eol_ = false;
610 this->lineoff_ = p + 2 - this->linebuf_;
611 return this->make_operator(OPERATOR_DIVEQ, 2);
612 }
613 else
614 {
615 this->add_semi_at_eol_ = false;
616 this->lineoff_ = p + 1 - this->linebuf_;
617 return this->make_operator(OPERATOR_DIV, 1);
618 }
619 break;
620
621 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
622 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
623 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
624 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
625 case 'Y': case 'Z':
626 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
627 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
628 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
629 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
630 case 'y': case 'z':
631 case '_':
632 this->lineoff_ = p - this->linebuf_;
633 return this->gather_identifier();
634
635 case '0': case '1': case '2': case '3': case '4':
636 case '5': case '6': case '7': case '8': case '9':
637 this->add_semi_at_eol_ = true;
638 this->lineoff_ = p - this->linebuf_;
639 return this->gather_number();
640
641 case '\'':
642 this->add_semi_at_eol_ = true;
643 this->lineoff_ = p - this->linebuf_;
644 return this->gather_character();
645
646 case '"':
647 this->add_semi_at_eol_ = true;
648 this->lineoff_ = p - this->linebuf_;
649 return this->gather_string();
650
651 case '`':
652 this->add_semi_at_eol_ = true;
653 this->lineoff_ = p - this->linebuf_;
654 return this->gather_raw_string();
655
656 case '<':
657 case '>':
658 case '&':
659 if (p + 2 < pend)
660 {
661 this->add_semi_at_eol_ = false;
662 Operator op = this->three_character_operator(cc, p[1], p[2]);
663 if (op != OPERATOR_INVALID)
664 {
665 this->lineoff_ = p + 3 - this->linebuf_;
666 return this->make_operator(op, 3);
667 }
668 }
669 // Fall through.
670 case '|':
671 case '=':
672 case '!':
673 case '+':
674 case '-':
675 case '^':
676 case '*':
677 // '/' handled above.
678 case '%':
679 case ':':
680 case ';':
681 case ',':
682 case '(': case ')':
683 case '{': case '}':
684 case '[': case ']':
685 {
686 this->add_semi_at_eol_ = false;
687 Operator op = this->two_character_operator(cc, p[1]);
688 int chars;
689 if (op != OPERATOR_INVALID)
690 {
691 ++p;
692 chars = 2;
693 }
694 else
695 {
696 op = this->one_character_operator(cc);
697 chars = 1;
698 }
699 this->lineoff_ = p + 1 - this->linebuf_;
700 return this->make_operator(op, chars);
701 }
702
703 case '.':
704 if (p[1] >= '0' && p[1] <= '9')
705 {
706 this->add_semi_at_eol_ = true;
707 this->lineoff_ = p - this->linebuf_;
708 return this->gather_number();
709 }
710 if (p[1] == '.' && p[2] == '.')
711 {
712 this->add_semi_at_eol_ = false;
713 this->lineoff_ = p + 3 - this->linebuf_;
714 return this->make_operator(OPERATOR_ELLIPSIS, 3);
715 }
716 this->add_semi_at_eol_ = false;
717 this->lineoff_ = p + 1 - this->linebuf_;
718 return this->make_operator(OPERATOR_DOT, 1);
719
720 default:
721 {
722 unsigned int ci;
723 bool issued_error;
724 this->lineoff_ = p - this->linebuf_;
725 const char *pnext = this->advance_one_utf8_char(p, &ci,
726 &issued_error);
727
728 // Ignore byte order mark at start of file.
729 if (ci == 0xfeff)
730 {
731 p = pnext;
732 break;
733 }
734
735 if (Lex::is_unicode_letter(ci))
736 return this->gather_identifier();
737
738 if (!issued_error)
739 error_at(this->location(),
740 "invalid character 0x%x in input file",
741 ci);
742
743 p = pend;
744
745 break;
746 }
747 }
748 }
749
750 this->lineoff_ = p - this->linebuf_;
751 }
752 }
753
754 // Fetch one UTF-8 character from a string. Set *VALUE to the value.
755 // Return the number of bytes read from the string. Returns 0 if the
756 // string does not point to a valid UTF-8 character.
757
758 int
759 Lex::fetch_char(const char* p, unsigned int* value)
760 {
761 unsigned char c = *p;
762 if (c <= 0x7f)
763 {
764 *value = c;
765 return 1;
766 }
767 else if ((c & 0xe0) == 0xc0
768 && (p[1] & 0xc0) == 0x80)
769 {
770 *value = (((c & 0x1f) << 6)
771 + (p[1] & 0x3f));
772 if (*value <= 0x7f)
773 {
774 *value = 0xfffd;
775 return 0;
776 }
777 return 2;
778 }
779 else if ((c & 0xf0) == 0xe0
780 && (p[1] & 0xc0) == 0x80
781 && (p[2] & 0xc0) == 0x80)
782 {
783 *value = (((c & 0xf) << 12)
784 + ((p[1] & 0x3f) << 6)
785 + (p[2] & 0x3f));
786 if (*value <= 0x7ff)
787 {
788 *value = 0xfffd;
789 return 0;
790 }
791 return 3;
792 }
793 else if ((c & 0xf8) == 0xf0
794 && (p[1] & 0xc0) == 0x80
795 && (p[2] & 0xc0) == 0x80
796 && (p[3] & 0xc0) == 0x80)
797 {
798 *value = (((c & 0x7) << 18)
799 + ((p[1] & 0x3f) << 12)
800 + ((p[2] & 0x3f) << 6)
801 + (p[3] & 0x3f));
802 if (*value <= 0xffff)
803 {
804 *value = 0xfffd;
805 return 0;
806 }
807 return 4;
808 }
809 else
810 {
811 /* Invalid encoding. Return the Unicode replacement
812 character. */
813 *value = 0xfffd;
814 return 0;
815 }
816 }
817
818 // Advance one UTF-8 character. Return the pointer beyond the
819 // character. Set *VALUE to the value. Set *ISSUED_ERROR if an error
820 // was issued.
821
822 const char*
823 Lex::advance_one_utf8_char(const char* p, unsigned int* value,
824 bool* issued_error)
825 {
826 *issued_error = false;
827
828 if (*p == '\0')
829 {
830 error_at(this->location(), "invalid NUL byte");
831 *issued_error = true;
832 *value = 0;
833 return p + 1;
834 }
835
836 int adv = Lex::fetch_char(p, value);
837 if (adv == 0)
838 {
839 error_at(this->location(), "invalid UTF-8 encoding");
840 *issued_error = true;
841 return p + 1;
842 }
843
844 // Warn about byte order mark, except at start of file.
845 if (*value == 0xfeff && (this->lineno_ != 1 || this->lineoff_ != 0))
846 {
847 error_at(this->location(), "Unicode (UTF-8) BOM in middle of file");
848 *issued_error = true;
849 }
850
851 return p + adv;
852 }
853
854 // Pick up an identifier.
855
856 Token
857 Lex::gather_identifier()
858 {
859 const char* pstart = this->linebuf_ + this->lineoff_;
860 const char* p = pstart;
861 const char* pend = this->linebuf_ + this->linesize_;
862 bool is_first = true;
863 bool is_exported = false;
864 bool has_non_ascii_char = false;
865 std::string buf;
866 while (p < pend)
867 {
868 unsigned char cc = *p;
869 if (cc <= 0x7f)
870 {
871 if ((cc < 'A' || cc > 'Z')
872 && (cc < 'a' || cc > 'z')
873 && cc != '_'
874 && (cc < '0' || cc > '9'))
875 break;
876 ++p;
877 if (is_first)
878 {
879 is_exported = cc >= 'A' && cc <= 'Z';
880 is_first = false;
881 }
882 if (has_non_ascii_char)
883 buf.push_back(cc);
884 }
885 else
886 {
887 unsigned int ci;
888 bool issued_error;
889 this->lineoff_ = p - this->linebuf_;
890 const char* pnext = this->advance_one_utf8_char(p, &ci,
891 &issued_error);
892 bool is_invalid = false;
893 if (!Lex::is_unicode_letter(ci) && !Lex::is_unicode_digit(ci))
894 {
895 // There is no valid place for a non-ASCII character
896 // other than an identifier, so we get better error
897 // handling behaviour if we swallow this character after
898 // giving an error.
899 if (!issued_error)
900 error_at(this->location(),
901 "invalid character 0x%x in identifier",
902 ci);
903 is_invalid = true;
904 }
905 if (is_first)
906 {
907 is_exported = Lex::is_unicode_uppercase(ci);
908 is_first = false;
909 }
910 if (!has_non_ascii_char)
911 {
912 buf.assign(pstart, p - pstart);
913 has_non_ascii_char = true;
914 }
915 if (is_invalid && !Lex::is_invalid_identifier(buf))
916 buf.append("$INVALID$");
917 p = pnext;
918 char ubuf[50];
919 // This assumes that all assemblers can handle an identifier
920 // with a '$' character.
921 snprintf(ubuf, sizeof ubuf, "$U%x$", ci);
922 buf.append(ubuf);
923 }
924 }
925 Location location = this->location();
926 this->add_semi_at_eol_ = true;
927 this->lineoff_ = p - this->linebuf_;
928 if (has_non_ascii_char)
929 return Token::make_identifier_token(buf, is_exported, location);
930 else
931 {
932 Keyword code = keywords.keyword_to_code(pstart, p - pstart);
933 if (code == KEYWORD_INVALID)
934 return Token::make_identifier_token(std::string(pstart, p - pstart),
935 is_exported, location);
936 else
937 {
938 switch (code)
939 {
940 case KEYWORD_BREAK:
941 case KEYWORD_CONTINUE:
942 case KEYWORD_FALLTHROUGH:
943 case KEYWORD_RETURN:
944 break;
945 default:
946 this->add_semi_at_eol_ = false;
947 break;
948 }
949 return Token::make_keyword_token(code, location);
950 }
951 }
952 }
953
954 // Return whether C is a hex digit.
955
956 bool
957 Lex::is_hex_digit(char c)
958 {
959 return ((c >= '0' && c <= '9')
960 || (c >= 'A' && c <= 'F')
961 || (c >= 'a' && c <= 'f'));
962 }
963
964 // Return whether an exponent could start at P.
965
966 bool
967 Lex::could_be_exponent(const char* p, const char* pend)
968 {
969 if (*p != 'e' && *p != 'E')
970 return false;
971 ++p;
972 if (p >= pend)
973 return false;
974 if (*p == '+' || *p == '-')
975 {
976 ++p;
977 if (p >= pend)
978 return false;
979 }
980 return *p >= '0' && *p <= '9';
981 }
982
983 // Pick up a number.
984
985 Token
986 Lex::gather_number()
987 {
988 const char* pstart = this->linebuf_ + this->lineoff_;
989 const char* p = pstart;
990 const char* pend = this->linebuf_ + this->linesize_;
991
992 Location location = this->location();
993
994 bool neg = false;
995 if (*p == '+')
996 ++p;
997 else if (*p == '-')
998 {
999 ++p;
1000 neg = true;
1001 }
1002
1003 const char* pnum = p;
1004 if (*p == '0')
1005 {
1006 int base;
1007 if ((p[1] == 'x' || p[1] == 'X')
1008 && Lex::is_hex_digit(p[2]))
1009 {
1010 base = 16;
1011 p += 2;
1012 pnum = p;
1013 while (p < pend)
1014 {
1015 if (!Lex::is_hex_digit(*p))
1016 break;
1017 ++p;
1018 }
1019 }
1020 else
1021 {
1022 base = 8;
1023 pnum = p;
1024 while (p < pend)
1025 {
1026 if (*p < '0' || *p > '7')
1027 break;
1028 ++p;
1029 }
1030 }
1031
1032 // A partial token that looks like an octal literal might actually be the
1033 // beginning of a floating-point or imaginary literal.
1034 if (base == 16 || (*p != '.' && *p != 'i' && !Lex::could_be_exponent(p, pend)))
1035 {
1036 std::string s(pnum, p - pnum);
1037 mpz_t val;
1038 int r = mpz_init_set_str(val, s.c_str(), base);
1039 go_assert(r == 0);
1040
1041 if (neg)
1042 mpz_neg(val, val);
1043
1044 this->lineoff_ = p - this->linebuf_;
1045 Token ret = Token::make_integer_token(val, location);
1046 mpz_clear(val);
1047 return ret;
1048 }
1049 }
1050
1051 while (p < pend)
1052 {
1053 if (*p < '0' || *p > '9')
1054 break;
1055 ++p;
1056 }
1057
1058 if (*p != '.' && *p != 'i' && !Lex::could_be_exponent(p, pend))
1059 {
1060 std::string s(pnum, p - pnum);
1061 mpz_t val;
1062 int r = mpz_init_set_str(val, s.c_str(), 10);
1063 go_assert(r == 0);
1064
1065 if (neg)
1066 mpz_neg(val, val);
1067
1068 this->lineoff_ = p - this->linebuf_;
1069 Token ret = Token::make_integer_token(val, location);
1070 mpz_clear(val);
1071 return ret;
1072 }
1073
1074 if (*p != 'i')
1075 {
1076 bool dot = *p == '.';
1077
1078 ++p;
1079
1080 if (!dot)
1081 {
1082 if (*p == '+' || *p == '-')
1083 ++p;
1084 }
1085
1086 while (p < pend)
1087 {
1088 if (*p < '0' || *p > '9')
1089 break;
1090 ++p;
1091 }
1092
1093 if (dot && Lex::could_be_exponent(p, pend))
1094 {
1095 ++p;
1096 if (*p == '+' || *p == '-')
1097 ++p;
1098 while (p < pend)
1099 {
1100 if (*p < '0' || *p > '9')
1101 break;
1102 ++p;
1103 }
1104 }
1105 }
1106
1107 std::string s(pnum, p - pnum);
1108 mpfr_t val;
1109 int r = mpfr_init_set_str(val, s.c_str(), 10, GMP_RNDN);
1110 go_assert(r == 0);
1111
1112 if (neg)
1113 mpfr_neg(val, val, GMP_RNDN);
1114
1115 bool is_imaginary = *p == 'i';
1116 if (is_imaginary)
1117 ++p;
1118
1119 this->lineoff_ = p - this->linebuf_;
1120 if (is_imaginary)
1121 {
1122 Token ret = Token::make_imaginary_token(val, location);
1123 mpfr_clear(val);
1124 return ret;
1125 }
1126 else
1127 {
1128 Token ret = Token::make_float_token(val, location);
1129 mpfr_clear(val);
1130 return ret;
1131 }
1132 }
1133
1134 // Advance one character, possibly escaped. Return the pointer beyond
1135 // the character. Set *VALUE to the character. Set *IS_CHARACTER if
1136 // this is a character (e.g., 'a' or '\u1234') rather than a byte
1137 // value (e.g., '\001').
1138
1139 const char*
1140 Lex::advance_one_char(const char* p, bool is_single_quote, unsigned int* value,
1141 bool* is_character)
1142 {
1143 *value = 0;
1144 *is_character = true;
1145 if (*p != '\\')
1146 {
1147 bool issued_error;
1148 const char* ret = this->advance_one_utf8_char(p, value, &issued_error);
1149 if (is_single_quote
1150 && (*value == '\'' || *value == '\n')
1151 && !issued_error)
1152 error_at(this->location(), "invalid character literal");
1153 return ret;
1154 }
1155 else
1156 {
1157 ++p;
1158 switch (*p)
1159 {
1160 case '0': case '1': case '2': case '3':
1161 case '4': case '5': case '6': case '7':
1162 *is_character = false;
1163 if (p[1] >= '0' && p[1] <= '7'
1164 && p[2] >= '0' && p[2] <= '7')
1165 {
1166 *value = ((Lex::octal_value(p[0]) << 6)
1167 + (Lex::octal_value(p[1]) << 3)
1168 + Lex::octal_value(p[2]));
1169 if (*value > 255)
1170 {
1171 error_at(this->location(), "invalid octal constant");
1172 *value = 255;
1173 }
1174 return p + 3;
1175 }
1176 error_at(this->location(), "invalid octal character");
1177 return (p[1] >= '0' && p[1] <= '7'
1178 ? p + 2
1179 : p + 1);
1180
1181 case 'x':
1182 case 'X':
1183 *is_character = false;
1184 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]))
1185 {
1186 *value = (hex_value(p[1]) << 4) + hex_value(p[2]);
1187 return p + 3;
1188 }
1189 error_at(this->location(), "invalid hex character");
1190 return (Lex::is_hex_digit(p[1])
1191 ? p + 2
1192 : p + 1);
1193
1194 case 'a':
1195 *value = '\a';
1196 return p + 1;
1197 case 'b':
1198 *value = '\b';
1199 return p + 1;
1200 case 'f':
1201 *value = '\f';
1202 return p + 1;
1203 case 'n':
1204 *value = '\n';
1205 return p + 1;
1206 case 'r':
1207 *value = '\r';
1208 return p + 1;
1209 case 't':
1210 *value = '\t';
1211 return p + 1;
1212 case 'v':
1213 *value = '\v';
1214 return p + 1;
1215 case '\\':
1216 *value = '\\';
1217 return p + 1;
1218 case '\'':
1219 if (!is_single_quote)
1220 error_at(this->location(), "invalid quoted character");
1221 *value = '\'';
1222 return p + 1;
1223 case '"':
1224 if (is_single_quote)
1225 error_at(this->location(), "invalid quoted character");
1226 *value = '"';
1227 return p + 1;
1228
1229 case 'u':
1230 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])
1231 && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4]))
1232 {
1233 *value = ((hex_value(p[1]) << 12)
1234 + (hex_value(p[2]) << 8)
1235 + (hex_value(p[3]) << 4)
1236 + hex_value(p[4]));
1237 if (*value >= 0xd800 && *value < 0xe000)
1238 {
1239 error_at(this->location(),
1240 "invalid unicode code point 0x%x",
1241 *value);
1242 // Use the replacement character.
1243 *value = 0xfffd;
1244 }
1245 return p + 5;
1246 }
1247 error_at(this->location(), "invalid little unicode code point");
1248 return p + 1;
1249
1250 case 'U':
1251 if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])
1252 && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4])
1253 && Lex::is_hex_digit(p[5]) && Lex::is_hex_digit(p[6])
1254 && Lex::is_hex_digit(p[7]) && Lex::is_hex_digit(p[8]))
1255 {
1256 *value = ((hex_value(p[1]) << 28)
1257 + (hex_value(p[2]) << 24)
1258 + (hex_value(p[3]) << 20)
1259 + (hex_value(p[4]) << 16)
1260 + (hex_value(p[5]) << 12)
1261 + (hex_value(p[6]) << 8)
1262 + (hex_value(p[7]) << 4)
1263 + hex_value(p[8]));
1264 if (*value > 0x10ffff
1265 || (*value >= 0xd800 && *value < 0xe000))
1266 {
1267 error_at(this->location(), "invalid unicode code point 0x%x",
1268 *value);
1269 // Use the replacement character.
1270 *value = 0xfffd;
1271 }
1272 return p + 9;
1273 }
1274 error_at(this->location(), "invalid big unicode code point");
1275 return p + 1;
1276
1277 default:
1278 error_at(this->location(), "invalid character after %<\\%>");
1279 *value = *p;
1280 return p + 1;
1281 }
1282 }
1283 }
1284
1285 // Append V to STR. IS_CHARACTER is true for a character which should
1286 // be stored in UTF-8, false for a general byte value which should be
1287 // stored directly.
1288
1289 void
1290 Lex::append_char(unsigned int v, bool is_character, std::string* str,
1291 Location location)
1292 {
1293 char buf[4];
1294 size_t len;
1295 if (v <= 0x7f || !is_character)
1296 {
1297 buf[0] = v;
1298 len = 1;
1299 }
1300 else if (v <= 0x7ff)
1301 {
1302 buf[0] = 0xc0 + (v >> 6);
1303 buf[1] = 0x80 + (v & 0x3f);
1304 len = 2;
1305 }
1306 else
1307 {
1308 if (v > 0x10ffff)
1309 {
1310 warning_at(location, 0,
1311 "unicode code point 0x%x out of range in string", v);
1312 // Turn it into the "replacement character".
1313 v = 0xfffd;
1314 }
1315 if (v <= 0xffff)
1316 {
1317 buf[0] = 0xe0 + (v >> 12);
1318 buf[1] = 0x80 + ((v >> 6) & 0x3f);
1319 buf[2] = 0x80 + (v & 0x3f);
1320 len = 3;
1321 }
1322 else
1323 {
1324 buf[0] = 0xf0 + (v >> 18);
1325 buf[1] = 0x80 + ((v >> 12) & 0x3f);
1326 buf[2] = 0x80 + ((v >> 6) & 0x3f);
1327 buf[3] = 0x80 + (v & 0x3f);
1328 len = 4;
1329 }
1330 }
1331 str->append(buf, len);
1332 }
1333
1334 // Pick up a character literal.
1335
1336 Token
1337 Lex::gather_character()
1338 {
1339 ++this->lineoff_;
1340 const char* pstart = this->linebuf_ + this->lineoff_;
1341 const char* p = pstart;
1342
1343 unsigned int value;
1344 bool is_character;
1345 p = this->advance_one_char(p, true, &value, &is_character);
1346
1347 if (*p != '\'')
1348 {
1349 error_at(this->location(), "unterminated character constant");
1350 this->lineoff_ = p - this->linebuf_;
1351 return this->make_invalid_token();
1352 }
1353
1354 mpz_t val;
1355 mpz_init_set_ui(val, value);
1356
1357 Location location = this->location();
1358 this->lineoff_ = p + 1 - this->linebuf_;
1359 Token ret = Token::make_character_token(val, location);
1360 mpz_clear(val);
1361 return ret;
1362 }
1363
1364 // Pick up a quoted string.
1365
1366 Token
1367 Lex::gather_string()
1368 {
1369 const char* pstart = this->linebuf_ + this->lineoff_ + 1;
1370 const char* p = pstart;
1371 const char* pend = this->linebuf_ + this->linesize_;
1372
1373 std::string value;
1374 while (*p != '"')
1375 {
1376 Location loc = this->location();
1377 unsigned int c;
1378 bool is_character;
1379 this->lineoff_ = p - this->linebuf_;
1380 p = this->advance_one_char(p, false, &c, &is_character);
1381 if (p >= pend)
1382 {
1383 error_at(this->location(), "unterminated string");
1384 --p;
1385 break;
1386 }
1387 Lex::append_char(c, is_character, &value, loc);
1388 }
1389
1390 Location location = this->location();
1391 this->lineoff_ = p + 1 - this->linebuf_;
1392 return Token::make_string_token(value, location);
1393 }
1394
1395 // Pick up a raw string.
1396
1397 Token
1398 Lex::gather_raw_string()
1399 {
1400 const char* p = this->linebuf_ + this->lineoff_ + 1;
1401 const char* pend = this->linebuf_ + this->linesize_;
1402 Location location = this->location();
1403
1404 std::string value;
1405 while (true)
1406 {
1407 while (p < pend)
1408 {
1409 if (*p == '`')
1410 {
1411 this->lineoff_ = p + 1 - this->linebuf_;
1412 return Token::make_string_token(value, location);
1413 }
1414 Location loc = this->location();
1415 unsigned int c;
1416 bool issued_error;
1417 this->lineoff_ = p - this->linebuf_;
1418 p = this->advance_one_utf8_char(p, &c, &issued_error);
1419 Lex::append_char(c, true, &value, loc);
1420 }
1421 this->lineoff_ = p - this->linebuf_;
1422 if (!this->require_line())
1423 {
1424 error_at(location, "unterminated raw string");
1425 return Token::make_string_token(value, location);
1426 }
1427 p = this->linebuf_ + this->lineoff_;
1428 pend = this->linebuf_ + this->linesize_;
1429 }
1430 }
1431
1432 // If C1 C2 C3 are a three character operator, return the code.
1433
1434 Operator
1435 Lex::three_character_operator(char c1, char c2, char c3)
1436 {
1437 if (c3 == '=')
1438 {
1439 if (c1 == '<' && c2 == '<')
1440 return OPERATOR_LSHIFTEQ;
1441 else if (c1 == '>' && c2 == '>')
1442 return OPERATOR_RSHIFTEQ;
1443 else if (c1 == '&' && c2 == '^')
1444 return OPERATOR_BITCLEAREQ;
1445 }
1446 return OPERATOR_INVALID;
1447 }
1448
1449 // If C1 C2 are a two character operator, return the code.
1450
1451 Operator
1452 Lex::two_character_operator(char c1, char c2)
1453 {
1454 switch (c1)
1455 {
1456 case '|':
1457 if (c2 == '|')
1458 return OPERATOR_OROR;
1459 else if (c2 == '=')
1460 return OPERATOR_OREQ;
1461 break;
1462 case '&':
1463 if (c2 == '&')
1464 return OPERATOR_ANDAND;
1465 else if (c2 == '^')
1466 return OPERATOR_BITCLEAR;
1467 else if (c2 == '=')
1468 return OPERATOR_ANDEQ;
1469 break;
1470 case '^':
1471 if (c2 == '=')
1472 return OPERATOR_XOREQ;
1473 break;
1474 case '=':
1475 if (c2 == '=')
1476 return OPERATOR_EQEQ;
1477 break;
1478 case '!':
1479 if (c2 == '=')
1480 return OPERATOR_NOTEQ;
1481 break;
1482 case '<':
1483 if (c2 == '=')
1484 return OPERATOR_LE;
1485 else if (c2 == '<')
1486 return OPERATOR_LSHIFT;
1487 else if (c2 == '-')
1488 return OPERATOR_CHANOP;
1489 break;
1490 case '>':
1491 if (c2 == '=')
1492 return OPERATOR_GE;
1493 else if (c2 == '>')
1494 return OPERATOR_RSHIFT;
1495 break;
1496 case '*':
1497 if (c2 == '=')
1498 return OPERATOR_MULTEQ;
1499 break;
1500 case '/':
1501 if (c2 == '=')
1502 return OPERATOR_DIVEQ;
1503 break;
1504 case '%':
1505 if (c2 == '=')
1506 return OPERATOR_MODEQ;
1507 break;
1508 case '+':
1509 if (c2 == '+')
1510 {
1511 this->add_semi_at_eol_ = true;
1512 return OPERATOR_PLUSPLUS;
1513 }
1514 else if (c2 == '=')
1515 return OPERATOR_PLUSEQ;
1516 break;
1517 case '-':
1518 if (c2 == '-')
1519 {
1520 this->add_semi_at_eol_ = true;
1521 return OPERATOR_MINUSMINUS;
1522 }
1523 else if (c2 == '=')
1524 return OPERATOR_MINUSEQ;
1525 break;
1526 case ':':
1527 if (c2 == '=')
1528 return OPERATOR_COLONEQ;
1529 break;
1530 default:
1531 break;
1532 }
1533 return OPERATOR_INVALID;
1534 }
1535
1536 // If character C is an operator, return the code.
1537
1538 Operator
1539 Lex::one_character_operator(char c)
1540 {
1541 switch (c)
1542 {
1543 case '<':
1544 return OPERATOR_LT;
1545 case '>':
1546 return OPERATOR_GT;
1547 case '+':
1548 return OPERATOR_PLUS;
1549 case '-':
1550 return OPERATOR_MINUS;
1551 case '|':
1552 return OPERATOR_OR;
1553 case '^':
1554 return OPERATOR_XOR;
1555 case '*':
1556 return OPERATOR_MULT;
1557 case '/':
1558 return OPERATOR_DIV;
1559 case '%':
1560 return OPERATOR_MOD;
1561 case '&':
1562 return OPERATOR_AND;
1563 case '!':
1564 return OPERATOR_NOT;
1565 case '=':
1566 return OPERATOR_EQ;
1567 case ':':
1568 return OPERATOR_COLON;
1569 case ';':
1570 return OPERATOR_SEMICOLON;
1571 case '.':
1572 return OPERATOR_DOT;
1573 case ',':
1574 return OPERATOR_COMMA;
1575 case '(':
1576 return OPERATOR_LPAREN;
1577 case ')':
1578 this->add_semi_at_eol_ = true;
1579 return OPERATOR_RPAREN;
1580 case '{':
1581 return OPERATOR_LCURLY;
1582 case '}':
1583 this->add_semi_at_eol_ = true;
1584 return OPERATOR_RCURLY;
1585 case '[':
1586 return OPERATOR_LSQUARE;
1587 case ']':
1588 this->add_semi_at_eol_ = true;
1589 return OPERATOR_RSQUARE;
1590 default:
1591 return OPERATOR_INVALID;
1592 }
1593 }
1594
1595 // Skip a C-style comment.
1596
1597 bool
1598 Lex::skip_c_comment()
1599 {
1600 while (true)
1601 {
1602 if (!this->require_line())
1603 {
1604 error_at(this->location(), "unterminated comment");
1605 return false;
1606 }
1607
1608 const char* p = this->linebuf_ + this->lineoff_;
1609 const char* pend = this->linebuf_ + this->linesize_;
1610
1611 while (p < pend)
1612 {
1613 if (p[0] == '*' && p + 1 < pend && p[1] == '/')
1614 {
1615 this->lineoff_ = p + 2 - this->linebuf_;
1616 return true;
1617 }
1618
1619 this->lineoff_ = p - this->linebuf_;
1620 unsigned int c;
1621 bool issued_error;
1622 p = this->advance_one_utf8_char(p, &c, &issued_error);
1623 }
1624
1625 this->lineoff_ = p - this->linebuf_;
1626 }
1627 }
1628
1629 // Skip a C++-style comment.
1630
1631 void
1632 Lex::skip_cpp_comment()
1633 {
1634 // Ensure that if EXTERN_ is set, it means that we just saw a
1635 // //extern comment.
1636 this->extern_.clear();
1637
1638 const char* p = this->linebuf_ + this->lineoff_;
1639 const char* pend = this->linebuf_ + this->linesize_;
1640
1641 // By convention, a C++ comment at the start of the line of the form
1642 // //line FILE:LINENO
1643 // is interpreted as setting the file name and line number of the
1644 // next source line.
1645
1646 if (this->lineoff_ == 2
1647 && pend - p > 5
1648 && memcmp(p, "line ", 5) == 0)
1649 {
1650 p += 5;
1651 while (p < pend && *p == ' ')
1652 ++p;
1653 const char* pcolon = static_cast<const char*>(memchr(p, ':', pend - p));
1654 if (pcolon != NULL
1655 && pcolon[1] >= '0'
1656 && pcolon[1] <= '9')
1657 {
1658 char* plend;
1659 long lineno = strtol(pcolon + 1, &plend, 10);
1660 if (plend > pcolon + 1
1661 && (plend == pend
1662 || *plend < '0'
1663 || *plend > '9')
1664 && lineno > 0
1665 && lineno < 0x7fffffff)
1666 {
1667 unsigned int filelen = pcolon - p;
1668 char* file = new char[filelen + 1];
1669 memcpy(file, p, filelen);
1670 file[filelen] = '\0';
1671
1672 this->linemap_->start_file(file, lineno);
1673 this->lineno_ = lineno - 1;
1674
1675 p = plend;
1676 }
1677 }
1678 }
1679
1680 // As a special gccgo extension, a C++ comment at the start of the
1681 // line of the form
1682 // //extern NAME
1683 // which immediately precedes a function declaration means that the
1684 // external name of the function declaration is NAME. This is
1685 // normally used to permit Go code to call a C function.
1686 if (this->lineoff_ == 2
1687 && pend - p > 7
1688 && memcmp(p, "extern ", 7) == 0)
1689 {
1690 p += 7;
1691 while (p < pend && (*p == ' ' || *p == '\t'))
1692 ++p;
1693 const char* plend = pend;
1694 while (plend > p
1695 && (plend[-1] == ' ' || plend[-1] == '\t' || plend[-1] == '\n'))
1696 --plend;
1697 if (plend > p)
1698 this->extern_ = std::string(p, plend - p);
1699 }
1700
1701 while (p < pend)
1702 {
1703 this->lineoff_ = p - this->linebuf_;
1704 unsigned int c;
1705 bool issued_error;
1706 p = this->advance_one_utf8_char(p, &c, &issued_error);
1707 if (issued_error)
1708 this->extern_.clear();
1709 }
1710 }
1711
1712 // The Unicode tables use this struct.
1713
1714 struct Unicode_range
1715 {
1716 // The low end of the range.
1717 unsigned int low;
1718 // The high end of the range.
1719 unsigned int high;
1720 // The stride. This entries represents low, low + stride, low + 2 *
1721 // stride, etc., up to high.
1722 unsigned int stride;
1723 };
1724
1725 // A table of whitespace characters--Unicode code points classified as
1726 // "Space", "C" locale whitespace characters, the "next line" control
1727 // character (0085), the line separator (2028), the paragraph
1728 // separator (2029), and the "zero-width non-break space" (feff).
1729
1730 static const Unicode_range unicode_space[] =
1731 {
1732 { 0x0009, 0x000d, 1 },
1733 { 0x0020, 0x0020, 1 },
1734 { 0x0085, 0x0085, 1 },
1735 { 0x00a0, 0x00a0, 1 },
1736 { 0x1680, 0x1680, 1 },
1737 { 0x180e, 0x180e, 1 },
1738 { 0x2000, 0x200a, 1 },
1739 { 0x2028, 0x2029, 1 },
1740 { 0x202f, 0x202f, 1 },
1741 { 0x205f, 0x205f, 1 },
1742 { 0x3000, 0x3000, 1 },
1743 { 0xfeff, 0xfeff, 1 },
1744 };
1745
1746 // A table of Unicode digits--Unicode code points classified as
1747 // "Digit".
1748
1749 static const Unicode_range unicode_digits[] =
1750 {
1751 { 0x0030, 0x0039, 1},
1752 { 0x0660, 0x0669, 1},
1753 { 0x06f0, 0x06f9, 1},
1754 { 0x07c0, 0x07c9, 1},
1755 { 0x0966, 0x096f, 1},
1756 { 0x09e6, 0x09ef, 1},
1757 { 0x0a66, 0x0a6f, 1},
1758 { 0x0ae6, 0x0aef, 1},
1759 { 0x0b66, 0x0b6f, 1},
1760 { 0x0be6, 0x0bef, 1},
1761 { 0x0c66, 0x0c6f, 1},
1762 { 0x0ce6, 0x0cef, 1},
1763 { 0x0d66, 0x0d6f, 1},
1764 { 0x0e50, 0x0e59, 1},
1765 { 0x0ed0, 0x0ed9, 1},
1766 { 0x0f20, 0x0f29, 1},
1767 { 0x1040, 0x1049, 1},
1768 { 0x17e0, 0x17e9, 1},
1769 { 0x1810, 0x1819, 1},
1770 { 0x1946, 0x194f, 1},
1771 { 0x19d0, 0x19d9, 1},
1772 { 0x1b50, 0x1b59, 1},
1773 { 0xff10, 0xff19, 1},
1774 { 0x104a0, 0x104a9, 1},
1775 { 0x1d7ce, 0x1d7ff, 1},
1776 };
1777
1778 // A table of Unicode letters--Unicode code points classified as
1779 // "Letter".
1780
1781 static const Unicode_range unicode_letters[] =
1782 {
1783 { 0x0041, 0x005a, 1},
1784 { 0x0061, 0x007a, 1},
1785 { 0x00aa, 0x00b5, 11},
1786 { 0x00ba, 0x00ba, 1},
1787 { 0x00c0, 0x00d6, 1},
1788 { 0x00d8, 0x00f6, 1},
1789 { 0x00f8, 0x02c1, 1},
1790 { 0x02c6, 0x02d1, 1},
1791 { 0x02e0, 0x02e4, 1},
1792 { 0x02ec, 0x02ee, 2},
1793 { 0x0370, 0x0374, 1},
1794 { 0x0376, 0x0377, 1},
1795 { 0x037a, 0x037d, 1},
1796 { 0x0386, 0x0386, 1},
1797 { 0x0388, 0x038a, 1},
1798 { 0x038c, 0x038c, 1},
1799 { 0x038e, 0x03a1, 1},
1800 { 0x03a3, 0x03f5, 1},
1801 { 0x03f7, 0x0481, 1},
1802 { 0x048a, 0x0523, 1},
1803 { 0x0531, 0x0556, 1},
1804 { 0x0559, 0x0559, 1},
1805 { 0x0561, 0x0587, 1},
1806 { 0x05d0, 0x05ea, 1},
1807 { 0x05f0, 0x05f2, 1},
1808 { 0x0621, 0x064a, 1},
1809 { 0x066e, 0x066f, 1},
1810 { 0x0671, 0x06d3, 1},
1811 { 0x06d5, 0x06d5, 1},
1812 { 0x06e5, 0x06e6, 1},
1813 { 0x06ee, 0x06ef, 1},
1814 { 0x06fa, 0x06fc, 1},
1815 { 0x06ff, 0x0710, 17},
1816 { 0x0712, 0x072f, 1},
1817 { 0x074d, 0x07a5, 1},
1818 { 0x07b1, 0x07b1, 1},
1819 { 0x07ca, 0x07ea, 1},
1820 { 0x07f4, 0x07f5, 1},
1821 { 0x07fa, 0x07fa, 1},
1822 { 0x0904, 0x0939, 1},
1823 { 0x093d, 0x0950, 19},
1824 { 0x0958, 0x0961, 1},
1825 { 0x0971, 0x0972, 1},
1826 { 0x097b, 0x097f, 1},
1827 { 0x0985, 0x098c, 1},
1828 { 0x098f, 0x0990, 1},
1829 { 0x0993, 0x09a8, 1},
1830 { 0x09aa, 0x09b0, 1},
1831 { 0x09b2, 0x09b2, 1},
1832 { 0x09b6, 0x09b9, 1},
1833 { 0x09bd, 0x09ce, 17},
1834 { 0x09dc, 0x09dd, 1},
1835 { 0x09df, 0x09e1, 1},
1836 { 0x09f0, 0x09f1, 1},
1837 { 0x0a05, 0x0a0a, 1},
1838 { 0x0a0f, 0x0a10, 1},
1839 { 0x0a13, 0x0a28, 1},
1840 { 0x0a2a, 0x0a30, 1},
1841 { 0x0a32, 0x0a33, 1},
1842 { 0x0a35, 0x0a36, 1},
1843 { 0x0a38, 0x0a39, 1},
1844 { 0x0a59, 0x0a5c, 1},
1845 { 0x0a5e, 0x0a5e, 1},
1846 { 0x0a72, 0x0a74, 1},
1847 { 0x0a85, 0x0a8d, 1},
1848 { 0x0a8f, 0x0a91, 1},
1849 { 0x0a93, 0x0aa8, 1},
1850 { 0x0aaa, 0x0ab0, 1},
1851 { 0x0ab2, 0x0ab3, 1},
1852 { 0x0ab5, 0x0ab9, 1},
1853 { 0x0abd, 0x0ad0, 19},
1854 { 0x0ae0, 0x0ae1, 1},
1855 { 0x0b05, 0x0b0c, 1},
1856 { 0x0b0f, 0x0b10, 1},
1857 { 0x0b13, 0x0b28, 1},
1858 { 0x0b2a, 0x0b30, 1},
1859 { 0x0b32, 0x0b33, 1},
1860 { 0x0b35, 0x0b39, 1},
1861 { 0x0b3d, 0x0b3d, 1},
1862 { 0x0b5c, 0x0b5d, 1},
1863 { 0x0b5f, 0x0b61, 1},
1864 { 0x0b71, 0x0b83, 18},
1865 { 0x0b85, 0x0b8a, 1},
1866 { 0x0b8e, 0x0b90, 1},
1867 { 0x0b92, 0x0b95, 1},
1868 { 0x0b99, 0x0b9a, 1},
1869 { 0x0b9c, 0x0b9c, 1},
1870 { 0x0b9e, 0x0b9f, 1},
1871 { 0x0ba3, 0x0ba4, 1},
1872 { 0x0ba8, 0x0baa, 1},
1873 { 0x0bae, 0x0bb9, 1},
1874 { 0x0bd0, 0x0bd0, 1},
1875 { 0x0c05, 0x0c0c, 1},
1876 { 0x0c0e, 0x0c10, 1},
1877 { 0x0c12, 0x0c28, 1},
1878 { 0x0c2a, 0x0c33, 1},
1879 { 0x0c35, 0x0c39, 1},
1880 { 0x0c3d, 0x0c3d, 1},
1881 { 0x0c58, 0x0c59, 1},
1882 { 0x0c60, 0x0c61, 1},
1883 { 0x0c85, 0x0c8c, 1},
1884 { 0x0c8e, 0x0c90, 1},
1885 { 0x0c92, 0x0ca8, 1},
1886 { 0x0caa, 0x0cb3, 1},
1887 { 0x0cb5, 0x0cb9, 1},
1888 { 0x0cbd, 0x0cde, 33},
1889 { 0x0ce0, 0x0ce1, 1},
1890 { 0x0d05, 0x0d0c, 1},
1891 { 0x0d0e, 0x0d10, 1},
1892 { 0x0d12, 0x0d28, 1},
1893 { 0x0d2a, 0x0d39, 1},
1894 { 0x0d3d, 0x0d3d, 1},
1895 { 0x0d60, 0x0d61, 1},
1896 { 0x0d7a, 0x0d7f, 1},
1897 { 0x0d85, 0x0d96, 1},
1898 { 0x0d9a, 0x0db1, 1},
1899 { 0x0db3, 0x0dbb, 1},
1900 { 0x0dbd, 0x0dbd, 1},
1901 { 0x0dc0, 0x0dc6, 1},
1902 { 0x0e01, 0x0e30, 1},
1903 { 0x0e32, 0x0e33, 1},
1904 { 0x0e40, 0x0e46, 1},
1905 { 0x0e81, 0x0e82, 1},
1906 { 0x0e84, 0x0e84, 1},
1907 { 0x0e87, 0x0e88, 1},
1908 { 0x0e8a, 0x0e8d, 3},
1909 { 0x0e94, 0x0e97, 1},
1910 { 0x0e99, 0x0e9f, 1},
1911 { 0x0ea1, 0x0ea3, 1},
1912 { 0x0ea5, 0x0ea7, 2},
1913 { 0x0eaa, 0x0eab, 1},
1914 { 0x0ead, 0x0eb0, 1},
1915 { 0x0eb2, 0x0eb3, 1},
1916 { 0x0ebd, 0x0ebd, 1},
1917 { 0x0ec0, 0x0ec4, 1},
1918 { 0x0ec6, 0x0ec6, 1},
1919 { 0x0edc, 0x0edd, 1},
1920 { 0x0f00, 0x0f00, 1},
1921 { 0x0f40, 0x0f47, 1},
1922 { 0x0f49, 0x0f6c, 1},
1923 { 0x0f88, 0x0f8b, 1},
1924 { 0x1000, 0x102a, 1},
1925 { 0x103f, 0x103f, 1},
1926 { 0x1050, 0x1055, 1},
1927 { 0x105a, 0x105d, 1},
1928 { 0x1061, 0x1061, 1},
1929 { 0x1065, 0x1066, 1},
1930 { 0x106e, 0x1070, 1},
1931 { 0x1075, 0x1081, 1},
1932 { 0x108e, 0x108e, 1},
1933 { 0x10a0, 0x10c5, 1},
1934 { 0x10d0, 0x10fa, 1},
1935 { 0x10fc, 0x10fc, 1},
1936 { 0x1100, 0x1159, 1},
1937 { 0x115f, 0x11a2, 1},
1938 { 0x11a8, 0x11f9, 1},
1939 { 0x1200, 0x1248, 1},
1940 { 0x124a, 0x124d, 1},
1941 { 0x1250, 0x1256, 1},
1942 { 0x1258, 0x1258, 1},
1943 { 0x125a, 0x125d, 1},
1944 { 0x1260, 0x1288, 1},
1945 { 0x128a, 0x128d, 1},
1946 { 0x1290, 0x12b0, 1},
1947 { 0x12b2, 0x12b5, 1},
1948 { 0x12b8, 0x12be, 1},
1949 { 0x12c0, 0x12c0, 1},
1950 { 0x12c2, 0x12c5, 1},
1951 { 0x12c8, 0x12d6, 1},
1952 { 0x12d8, 0x1310, 1},
1953 { 0x1312, 0x1315, 1},
1954 { 0x1318, 0x135a, 1},
1955 { 0x1380, 0x138f, 1},
1956 { 0x13a0, 0x13f4, 1},
1957 { 0x1401, 0x166c, 1},
1958 { 0x166f, 0x1676, 1},
1959 { 0x1681, 0x169a, 1},
1960 { 0x16a0, 0x16ea, 1},
1961 { 0x1700, 0x170c, 1},
1962 { 0x170e, 0x1711, 1},
1963 { 0x1720, 0x1731, 1},
1964 { 0x1740, 0x1751, 1},
1965 { 0x1760, 0x176c, 1},
1966 { 0x176e, 0x1770, 1},
1967 { 0x1780, 0x17b3, 1},
1968 { 0x17d7, 0x17dc, 5},
1969 { 0x1820, 0x1877, 1},
1970 { 0x1880, 0x18a8, 1},
1971 { 0x18aa, 0x18aa, 1},
1972 { 0x1900, 0x191c, 1},
1973 { 0x1950, 0x196d, 1},
1974 { 0x1970, 0x1974, 1},
1975 { 0x1980, 0x19a9, 1},
1976 { 0x19c1, 0x19c7, 1},
1977 { 0x1a00, 0x1a16, 1},
1978 { 0x1b05, 0x1b33, 1},
1979 { 0x1b45, 0x1b4b, 1},
1980 { 0x1b83, 0x1ba0, 1},
1981 { 0x1bae, 0x1baf, 1},
1982 { 0x1c00, 0x1c23, 1},
1983 { 0x1c4d, 0x1c4f, 1},
1984 { 0x1c5a, 0x1c7d, 1},
1985 { 0x1d00, 0x1dbf, 1},
1986 { 0x1e00, 0x1f15, 1},
1987 { 0x1f18, 0x1f1d, 1},
1988 { 0x1f20, 0x1f45, 1},
1989 { 0x1f48, 0x1f4d, 1},
1990 { 0x1f50, 0x1f57, 1},
1991 { 0x1f59, 0x1f5d, 2},
1992 { 0x1f5f, 0x1f7d, 1},
1993 { 0x1f80, 0x1fb4, 1},
1994 { 0x1fb6, 0x1fbc, 1},
1995 { 0x1fbe, 0x1fbe, 1},
1996 { 0x1fc2, 0x1fc4, 1},
1997 { 0x1fc6, 0x1fcc, 1},
1998 { 0x1fd0, 0x1fd3, 1},
1999 { 0x1fd6, 0x1fdb, 1},
2000 { 0x1fe0, 0x1fec, 1},
2001 { 0x1ff2, 0x1ff4, 1},
2002 { 0x1ff6, 0x1ffc, 1},
2003 { 0x2071, 0x207f, 14},
2004 { 0x2090, 0x2094, 1},
2005 { 0x2102, 0x2107, 5},
2006 { 0x210a, 0x2113, 1},
2007 { 0x2115, 0x2115, 1},
2008 { 0x2119, 0x211d, 1},
2009 { 0x2124, 0x2128, 2},
2010 { 0x212a, 0x212d, 1},
2011 { 0x212f, 0x2139, 1},
2012 { 0x213c, 0x213f, 1},
2013 { 0x2145, 0x2149, 1},
2014 { 0x214e, 0x214e, 1},
2015 { 0x2183, 0x2184, 1},
2016 { 0x2c00, 0x2c2e, 1},
2017 { 0x2c30, 0x2c5e, 1},
2018 { 0x2c60, 0x2c6f, 1},
2019 { 0x2c71, 0x2c7d, 1},
2020 { 0x2c80, 0x2ce4, 1},
2021 { 0x2d00, 0x2d25, 1},
2022 { 0x2d30, 0x2d65, 1},
2023 { 0x2d6f, 0x2d6f, 1},
2024 { 0x2d80, 0x2d96, 1},
2025 { 0x2da0, 0x2da6, 1},
2026 { 0x2da8, 0x2dae, 1},
2027 { 0x2db0, 0x2db6, 1},
2028 { 0x2db8, 0x2dbe, 1},
2029 { 0x2dc0, 0x2dc6, 1},
2030 { 0x2dc8, 0x2dce, 1},
2031 { 0x2dd0, 0x2dd6, 1},
2032 { 0x2dd8, 0x2dde, 1},
2033 { 0x2e2f, 0x2e2f, 1},
2034 { 0x3005, 0x3006, 1},
2035 { 0x3031, 0x3035, 1},
2036 { 0x303b, 0x303c, 1},
2037 { 0x3041, 0x3096, 1},
2038 { 0x309d, 0x309f, 1},
2039 { 0x30a1, 0x30fa, 1},
2040 { 0x30fc, 0x30ff, 1},
2041 { 0x3105, 0x312d, 1},
2042 { 0x3131, 0x318e, 1},
2043 { 0x31a0, 0x31b7, 1},
2044 { 0x31f0, 0x31ff, 1},
2045 { 0x3400, 0x4db5, 1},
2046 { 0x4e00, 0x9fc3, 1},
2047 { 0xa000, 0xa48c, 1},
2048 { 0xa500, 0xa60c, 1},
2049 { 0xa610, 0xa61f, 1},
2050 { 0xa62a, 0xa62b, 1},
2051 { 0xa640, 0xa65f, 1},
2052 { 0xa662, 0xa66e, 1},
2053 { 0xa67f, 0xa697, 1},
2054 { 0xa717, 0xa71f, 1},
2055 { 0xa722, 0xa788, 1},
2056 { 0xa78b, 0xa78c, 1},
2057 { 0xa7fb, 0xa801, 1},
2058 { 0xa803, 0xa805, 1},
2059 { 0xa807, 0xa80a, 1},
2060 { 0xa80c, 0xa822, 1},
2061 { 0xa840, 0xa873, 1},
2062 { 0xa882, 0xa8b3, 1},
2063 { 0xa90a, 0xa925, 1},
2064 { 0xa930, 0xa946, 1},
2065 { 0xaa00, 0xaa28, 1},
2066 { 0xaa40, 0xaa42, 1},
2067 { 0xaa44, 0xaa4b, 1},
2068 { 0xac00, 0xd7a3, 1},
2069 { 0xf900, 0xfa2d, 1},
2070 { 0xfa30, 0xfa6a, 1},
2071 { 0xfa70, 0xfad9, 1},
2072 { 0xfb00, 0xfb06, 1},
2073 { 0xfb13, 0xfb17, 1},
2074 { 0xfb1d, 0xfb1d, 1},
2075 { 0xfb1f, 0xfb28, 1},
2076 { 0xfb2a, 0xfb36, 1},
2077 { 0xfb38, 0xfb3c, 1},
2078 { 0xfb3e, 0xfb3e, 1},
2079 { 0xfb40, 0xfb41, 1},
2080 { 0xfb43, 0xfb44, 1},
2081 { 0xfb46, 0xfbb1, 1},
2082 { 0xfbd3, 0xfd3d, 1},
2083 { 0xfd50, 0xfd8f, 1},
2084 { 0xfd92, 0xfdc7, 1},
2085 { 0xfdf0, 0xfdfb, 1},
2086 { 0xfe70, 0xfe74, 1},
2087 { 0xfe76, 0xfefc, 1},
2088 { 0xff21, 0xff3a, 1},
2089 { 0xff41, 0xff5a, 1},
2090 { 0xff66, 0xffbe, 1},
2091 { 0xffc2, 0xffc7, 1},
2092 { 0xffca, 0xffcf, 1},
2093 { 0xffd2, 0xffd7, 1},
2094 { 0xffda, 0xffdc, 1},
2095 { 0x10000, 0x1000b, 1},
2096 { 0x1000d, 0x10026, 1},
2097 { 0x10028, 0x1003a, 1},
2098 { 0x1003c, 0x1003d, 1},
2099 { 0x1003f, 0x1004d, 1},
2100 { 0x10050, 0x1005d, 1},
2101 { 0x10080, 0x100fa, 1},
2102 { 0x10280, 0x1029c, 1},
2103 { 0x102a0, 0x102d0, 1},
2104 { 0x10300, 0x1031e, 1},
2105 { 0x10330, 0x10340, 1},
2106 { 0x10342, 0x10349, 1},
2107 { 0x10380, 0x1039d, 1},
2108 { 0x103a0, 0x103c3, 1},
2109 { 0x103c8, 0x103cf, 1},
2110 { 0x10400, 0x1049d, 1},
2111 { 0x10800, 0x10805, 1},
2112 { 0x10808, 0x10808, 1},
2113 { 0x1080a, 0x10835, 1},
2114 { 0x10837, 0x10838, 1},
2115 { 0x1083c, 0x1083f, 3},
2116 { 0x10900, 0x10915, 1},
2117 { 0x10920, 0x10939, 1},
2118 { 0x10a00, 0x10a00, 1},
2119 { 0x10a10, 0x10a13, 1},
2120 { 0x10a15, 0x10a17, 1},
2121 { 0x10a19, 0x10a33, 1},
2122 { 0x12000, 0x1236e, 1},
2123 { 0x1d400, 0x1d454, 1},
2124 { 0x1d456, 0x1d49c, 1},
2125 { 0x1d49e, 0x1d49f, 1},
2126 { 0x1d4a2, 0x1d4a2, 1},
2127 { 0x1d4a5, 0x1d4a6, 1},
2128 { 0x1d4a9, 0x1d4ac, 1},
2129 { 0x1d4ae, 0x1d4b9, 1},
2130 { 0x1d4bb, 0x1d4bb, 1},
2131 { 0x1d4bd, 0x1d4c3, 1},
2132 { 0x1d4c5, 0x1d505, 1},
2133 { 0x1d507, 0x1d50a, 1},
2134 { 0x1d50d, 0x1d514, 1},
2135 { 0x1d516, 0x1d51c, 1},
2136 { 0x1d51e, 0x1d539, 1},
2137 { 0x1d53b, 0x1d53e, 1},
2138 { 0x1d540, 0x1d544, 1},
2139 { 0x1d546, 0x1d546, 1},
2140 { 0x1d54a, 0x1d550, 1},
2141 { 0x1d552, 0x1d6a5, 1},
2142 { 0x1d6a8, 0x1d6c0, 1},
2143 { 0x1d6c2, 0x1d6da, 1},
2144 { 0x1d6dc, 0x1d6fa, 1},
2145 { 0x1d6fc, 0x1d714, 1},
2146 { 0x1d716, 0x1d734, 1},
2147 { 0x1d736, 0x1d74e, 1},
2148 { 0x1d750, 0x1d76e, 1},
2149 { 0x1d770, 0x1d788, 1},
2150 { 0x1d78a, 0x1d7a8, 1},
2151 { 0x1d7aa, 0x1d7c2, 1},
2152 { 0x1d7c4, 0x1d7cb, 1},
2153 { 0x20000, 0x2a6d6, 1},
2154 { 0x2f800, 0x2fa1d, 1},
2155 };
2156
2157 // A table of Unicode uppercase letters--Unicode code points
2158 // classified as "Letter, uppercase".
2159
2160 static const Unicode_range unicode_uppercase_letters[] =
2161 {
2162 { 0x0041, 0x005a, 1},
2163 { 0x00c0, 0x00d6, 1},
2164 { 0x00d8, 0x00de, 1},
2165 { 0x0100, 0x0136, 2},
2166 { 0x0139, 0x0147, 2},
2167 { 0x014a, 0x0176, 2},
2168 { 0x0178, 0x0179, 1},
2169 { 0x017b, 0x017d, 2},
2170 { 0x0181, 0x0182, 1},
2171 { 0x0184, 0x0184, 1},
2172 { 0x0186, 0x0187, 1},
2173 { 0x0189, 0x018b, 1},
2174 { 0x018e, 0x0191, 1},
2175 { 0x0193, 0x0194, 1},
2176 { 0x0196, 0x0198, 1},
2177 { 0x019c, 0x019d, 1},
2178 { 0x019f, 0x01a0, 1},
2179 { 0x01a2, 0x01a4, 2},
2180 { 0x01a6, 0x01a7, 1},
2181 { 0x01a9, 0x01ac, 3},
2182 { 0x01ae, 0x01af, 1},
2183 { 0x01b1, 0x01b3, 1},
2184 { 0x01b5, 0x01b5, 1},
2185 { 0x01b7, 0x01b8, 1},
2186 { 0x01bc, 0x01c4, 8},
2187 { 0x01c7, 0x01cd, 3},
2188 { 0x01cf, 0x01db, 2},
2189 { 0x01de, 0x01ee, 2},
2190 { 0x01f1, 0x01f4, 3},
2191 { 0x01f6, 0x01f8, 1},
2192 { 0x01fa, 0x0232, 2},
2193 { 0x023a, 0x023b, 1},
2194 { 0x023d, 0x023e, 1},
2195 { 0x0241, 0x0241, 1},
2196 { 0x0243, 0x0246, 1},
2197 { 0x0248, 0x024e, 2},
2198 { 0x0370, 0x0372, 2},
2199 { 0x0376, 0x0386, 16},
2200 { 0x0388, 0x038a, 1},
2201 { 0x038c, 0x038c, 1},
2202 { 0x038e, 0x038f, 1},
2203 { 0x0391, 0x03a1, 1},
2204 { 0x03a3, 0x03ab, 1},
2205 { 0x03cf, 0x03cf, 1},
2206 { 0x03d2, 0x03d4, 1},
2207 { 0x03d8, 0x03ee, 2},
2208 { 0x03f4, 0x03f7, 3},
2209 { 0x03f9, 0x03fa, 1},
2210 { 0x03fd, 0x042f, 1},
2211 { 0x0460, 0x0480, 2},
2212 { 0x048a, 0x04be, 2},
2213 { 0x04c0, 0x04c1, 1},
2214 { 0x04c3, 0x04cd, 2},
2215 { 0x04d0, 0x0522, 2},
2216 { 0x0531, 0x0556, 1},
2217 { 0x10a0, 0x10c5, 1},
2218 { 0x1e00, 0x1e94, 2},
2219 { 0x1e9e, 0x1efe, 2},
2220 { 0x1f08, 0x1f0f, 1},
2221 { 0x1f18, 0x1f1d, 1},
2222 { 0x1f28, 0x1f2f, 1},
2223 { 0x1f38, 0x1f3f, 1},
2224 { 0x1f48, 0x1f4d, 1},
2225 { 0x1f59, 0x1f5f, 2},
2226 { 0x1f68, 0x1f6f, 1},
2227 { 0x1fb8, 0x1fbb, 1},
2228 { 0x1fc8, 0x1fcb, 1},
2229 { 0x1fd8, 0x1fdb, 1},
2230 { 0x1fe8, 0x1fec, 1},
2231 { 0x1ff8, 0x1ffb, 1},
2232 { 0x2102, 0x2107, 5},
2233 { 0x210b, 0x210d, 1},
2234 { 0x2110, 0x2112, 1},
2235 { 0x2115, 0x2115, 1},
2236 { 0x2119, 0x211d, 1},
2237 { 0x2124, 0x2128, 2},
2238 { 0x212a, 0x212d, 1},
2239 { 0x2130, 0x2133, 1},
2240 { 0x213e, 0x213f, 1},
2241 { 0x2145, 0x2183, 62},
2242 { 0x2c00, 0x2c2e, 1},
2243 { 0x2c60, 0x2c60, 1},
2244 { 0x2c62, 0x2c64, 1},
2245 { 0x2c67, 0x2c6b, 2},
2246 { 0x2c6d, 0x2c6f, 1},
2247 { 0x2c72, 0x2c75, 3},
2248 { 0x2c80, 0x2ce2, 2},
2249 { 0xa640, 0xa65e, 2},
2250 { 0xa662, 0xa66c, 2},
2251 { 0xa680, 0xa696, 2},
2252 { 0xa722, 0xa72e, 2},
2253 { 0xa732, 0xa76e, 2},
2254 { 0xa779, 0xa77b, 2},
2255 { 0xa77d, 0xa77e, 1},
2256 { 0xa780, 0xa786, 2},
2257 { 0xa78b, 0xa78b, 1},
2258 { 0xff21, 0xff3a, 1},
2259 { 0x10400, 0x10427, 1},
2260 { 0x1d400, 0x1d419, 1},
2261 { 0x1d434, 0x1d44d, 1},
2262 { 0x1d468, 0x1d481, 1},
2263 { 0x1d49c, 0x1d49c, 1},
2264 { 0x1d49e, 0x1d49f, 1},
2265 { 0x1d4a2, 0x1d4a2, 1},
2266 { 0x1d4a5, 0x1d4a6, 1},
2267 { 0x1d4a9, 0x1d4ac, 1},
2268 { 0x1d4ae, 0x1d4b5, 1},
2269 { 0x1d4d0, 0x1d4e9, 1},
2270 { 0x1d504, 0x1d505, 1},
2271 { 0x1d507, 0x1d50a, 1},
2272 { 0x1d50d, 0x1d514, 1},
2273 { 0x1d516, 0x1d51c, 1},
2274 { 0x1d538, 0x1d539, 1},
2275 { 0x1d53b, 0x1d53e, 1},
2276 { 0x1d540, 0x1d544, 1},
2277 { 0x1d546, 0x1d546, 1},
2278 { 0x1d54a, 0x1d550, 1},
2279 { 0x1d56c, 0x1d585, 1},
2280 { 0x1d5a0, 0x1d5b9, 1},
2281 { 0x1d5d4, 0x1d5ed, 1},
2282 { 0x1d608, 0x1d621, 1},
2283 { 0x1d63c, 0x1d655, 1},
2284 { 0x1d670, 0x1d689, 1},
2285 { 0x1d6a8, 0x1d6c0, 1},
2286 { 0x1d6e2, 0x1d6fa, 1},
2287 { 0x1d71c, 0x1d734, 1},
2288 { 0x1d756, 0x1d76e, 1},
2289 { 0x1d790, 0x1d7a8, 1},
2290 { 0x1d7ca, 0x1d7ca, 1},
2291 };
2292
2293 // Return true if C is in RANGES.
2294
2295 bool
2296 Lex::is_in_unicode_range(unsigned int c, const Unicode_range* ranges,
2297 size_t range_size)
2298 {
2299 if (c < 0x100)
2300 {
2301 // The common case is a small value, and we know that it will be
2302 // in the first few entries of the table. Do a linear scan
2303 // rather than a binary search.
2304 for (size_t i = 0; i < range_size; ++i)
2305 {
2306 const Unicode_range* p = &ranges[i];
2307 if (c <= p->high)
2308 {
2309 if (c < p->low)
2310 return false;
2311 return (c - p->low) % p->stride == 0;
2312 }
2313 }
2314 return false;
2315 }
2316 else
2317 {
2318 size_t lo = 0;
2319 size_t hi = range_size;
2320 while (lo < hi)
2321 {
2322 size_t mid = lo + (hi - lo) / 2;
2323 const Unicode_range* p = &ranges[mid];
2324 if (c < p->low)
2325 hi = mid;
2326 else if (c > p->high)
2327 lo = mid + 1;
2328 else
2329 return (c - p->low) % p->stride == 0;
2330 }
2331 return false;
2332 }
2333 }
2334
2335 // Return whether C is a space character.
2336
2337 bool
2338 Lex::is_unicode_space(unsigned int c)
2339 {
2340 return Lex::is_in_unicode_range(c, unicode_space,
2341 ARRAY_SIZE(unicode_space));
2342 }
2343
2344 // Return whether C is a Unicode digit--a Unicode code point
2345 // classified as "Digit".
2346
2347 bool
2348 Lex::is_unicode_digit(unsigned int c)
2349 {
2350 return Lex::is_in_unicode_range(c, unicode_digits,
2351 ARRAY_SIZE(unicode_digits));
2352 }
2353
2354 // Return whether C is a Unicode letter--a Unicode code point
2355 // classified as "Letter".
2356
2357 bool
2358 Lex::is_unicode_letter(unsigned int c)
2359 {
2360 return Lex::is_in_unicode_range(c, unicode_letters,
2361 ARRAY_SIZE(unicode_letters));
2362 }
2363
2364 // Return whether C is a Unicode uppercase letter. a Unicode code
2365 // point classified as "Letter, uppercase".
2366
2367 bool
2368 Lex::is_unicode_uppercase(unsigned int c)
2369 {
2370 return Lex::is_in_unicode_range(c, unicode_uppercase_letters,
2371 ARRAY_SIZE(unicode_uppercase_letters));
2372 }
2373
2374 // Return whether the identifier NAME should be exported. NAME is a
2375 // mangled name which includes only ASCII characters.
2376
2377 bool
2378 Lex::is_exported_name(const std::string& name)
2379 {
2380 unsigned char c = name[0];
2381 if (c != '$')
2382 return c >= 'A' && c <= 'Z';
2383 else
2384 {
2385 const char* p = name.data();
2386 size_t len = name.length();
2387 if (len < 2 || p[1] != 'U')
2388 return false;
2389 unsigned int ci = 0;
2390 for (size_t i = 2; i < len && p[i] != '$'; ++i)
2391 {
2392 c = p[i];
2393 if (!hex_p(c))
2394 return false;
2395 ci <<= 4;
2396 ci |= hex_value(c);
2397 }
2398 return Lex::is_unicode_uppercase(ci);
2399 }
2400 }
2401
2402 // Return whether the identifier NAME contains an invalid character.
2403 // This is based on how we handle invalid characters in
2404 // gather_identifier.
2405
2406 bool
2407 Lex::is_invalid_identifier(const std::string& name)
2408 {
2409 return name.find("$INVALID$") != std::string::npos;
2410 }