1 // lex.cc -- Go frontend lexer.
3 // Copyright 2009 The Go Authors. All rights reserved.
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file.
11 // Manage mapping from keywords to the Keyword codes.
16 // The structure which maps keywords to codes.
20 const char* keystring
;
25 // Return the parsecode corresponding to KEYSTRING, or
26 // KEYWORD_INVALID if it is not a keyword.
28 keyword_to_code(const char* keyword
, size_t len
) const;
30 // Return the string for a keyword.
32 keyword_to_string(Keyword
) const;
35 static const Mapping mapping_
[];
36 static const int count_
;
39 // Mapping from keyword string to keyword code. This array must be
40 // kept in sorted order, and the order must match the Keyword enum.
41 // Strings are looked up using bsearch.
43 const Keywords::Mapping
44 Keywords::mapping_
[] =
46 { NULL
, KEYWORD_INVALID
},
47 { "__asm__", KEYWORD_ASM
},
48 { "break", KEYWORD_BREAK
},
49 { "case", KEYWORD_CASE
},
50 { "chan", KEYWORD_CHAN
},
51 { "const", KEYWORD_CONST
},
52 { "continue", KEYWORD_CONTINUE
},
53 { "default", KEYWORD_DEFAULT
},
54 { "defer", KEYWORD_DEFER
},
55 { "else", KEYWORD_ELSE
},
56 { "fallthrough", KEYWORD_FALLTHROUGH
},
57 { "for", KEYWORD_FOR
},
58 { "func", KEYWORD_FUNC
},
60 { "goto", KEYWORD_GOTO
},
62 { "import", KEYWORD_IMPORT
},
63 { "interface", KEYWORD_INTERFACE
},
64 { "map", KEYWORD_MAP
},
65 { "package", KEYWORD_PACKAGE
},
66 { "range", KEYWORD_RANGE
},
67 { "return", KEYWORD_RETURN
},
68 { "select", KEYWORD_SELECT
},
69 { "struct", KEYWORD_STRUCT
},
70 { "switch", KEYWORD_SWITCH
},
71 { "type", KEYWORD_TYPE
},
72 { "var", KEYWORD_VAR
}
75 // Number of entries in the map.
77 const int Keywords::count_
=
78 sizeof(Keywords::mapping_
) / sizeof(Keywords::mapping_
[0]);
80 // Comparison function passed to bsearch.
85 struct Keywords_search_key
92 keyword_compare(const void* keyv
, const void* mapv
)
94 const Keywords_search_key
* key
=
95 static_cast<const Keywords_search_key
*>(keyv
);
96 const Keywords::Mapping
* map
=
97 static_cast<const Keywords::Mapping
*>(mapv
);
98 if (map
->keystring
== NULL
)
100 int i
= strncmp(key
->str
, map
->keystring
, key
->len
);
103 if (map
->keystring
[key
->len
] != '\0')
110 // Convert a string to a keyword code. Return KEYWORD_INVALID if the
111 // string is not a keyword.
114 Keywords::keyword_to_code(const char* keyword
, size_t len
) const
116 Keywords_search_key key
;
119 void* mapv
= bsearch(&key
,
122 sizeof(this->mapping_
[0]),
125 return KEYWORD_INVALID
;
126 Mapping
* map
= static_cast<Mapping
*>(mapv
);
130 // Convert a keyword code to a string.
133 Keywords::keyword_to_string(Keyword code
) const
135 go_assert(code
> KEYWORD_INVALID
&& code
< this->count_
);
136 const Mapping
* map
= &this->mapping_
[code
];
137 go_assert(map
->keycode
== code
);
138 return map
->keystring
;
141 // There is one instance of the Keywords class.
143 static Keywords keywords
;
147 // Make a general token.
149 Token::Token(Classification classification
, source_location location
)
150 : classification_(classification
), location_(location
)
161 // Clear a token--release memory.
166 if (this->classification_
== TOKEN_INTEGER
)
167 mpz_clear(this->u_
.integer_value
);
168 else if (this->classification_
== TOKEN_FLOAT
169 || this->classification_
== TOKEN_IMAGINARY
)
170 mpfr_clear(this->u_
.float_value
);
173 // Construct a token.
175 Token::Token(const Token
& tok
)
176 : classification_(tok
.classification_
), location_(tok
.location_
)
178 switch (this->classification_
)
184 this->u_
.keyword
= tok
.u_
.keyword
;
186 case TOKEN_IDENTIFIER
:
188 this->u_
.string_value
= tok
.u_
.string_value
;
191 this->u_
.op
= tok
.u_
.op
;
194 mpz_init_set(this->u_
.integer_value
, tok
.u_
.integer_value
);
197 case TOKEN_IMAGINARY
:
198 mpfr_init_set(this->u_
.float_value
, tok
.u_
.float_value
, GMP_RNDN
);
205 // Assign to a token.
208 Token::operator=(const Token
& tok
)
211 this->classification_
= tok
.classification_
;
212 this->location_
= tok
.location_
;
213 switch (tok
.classification_
)
219 this->u_
.keyword
= tok
.u_
.keyword
;
221 case TOKEN_IDENTIFIER
:
222 this->u_
.identifier_value
.name
= tok
.u_
.identifier_value
.name
;
223 this->u_
.identifier_value
.is_exported
=
224 tok
.u_
.identifier_value
.is_exported
;
227 this->u_
.string_value
= tok
.u_
.string_value
;
230 this->u_
.op
= tok
.u_
.op
;
233 mpz_init_set(this->u_
.integer_value
, tok
.u_
.integer_value
);
236 case TOKEN_IMAGINARY
:
237 mpfr_init_set(this->u_
.float_value
, tok
.u_
.float_value
, GMP_RNDN
);
245 // Print the token for debugging.
248 Token::print(FILE* file
) const
250 switch (this->classification_
)
253 fprintf(file
, "invalid");
256 fprintf(file
, "EOF");
259 fprintf(file
, "keyword %s", keywords
.keyword_to_string(this->u_
.keyword
));
261 case TOKEN_IDENTIFIER
:
262 fprintf(file
, "identifier \"%s\"", this->u_
.string_value
->c_str());
265 fprintf(file
, "quoted string \"%s\"", this->u_
.string_value
->c_str());
268 fprintf(file
, "integer ");
269 mpz_out_str(file
, 10, this->u_
.integer_value
);
272 fprintf(file
, "float ");
273 mpfr_out_str(file
, 10, 0, this->u_
.float_value
, GMP_RNDN
);
275 case TOKEN_IMAGINARY
:
276 fprintf(file
, "imaginary ");
277 mpfr_out_str(file
, 10, 0, this->u_
.float_value
, GMP_RNDN
);
280 fprintf(file
, "operator ");
283 case OPERATOR_INVALID
:
284 fprintf(file
, "invalid");
289 case OPERATOR_ANDAND
:
331 case OPERATOR_LSHIFT
:
334 case OPERATOR_RSHIFT
:
340 case OPERATOR_BITCLEAR
:
346 case OPERATOR_CHANOP
:
352 case OPERATOR_PLUSEQ
:
355 case OPERATOR_MINUSEQ
:
364 case OPERATOR_MULTEQ
:
371 fprintf(file
, "%%=");
373 case OPERATOR_LSHIFTEQ
:
374 fprintf(file
, "<<=");
376 case OPERATOR_RSHIFTEQ
:
377 fprintf(file
, ">>=");
382 case OPERATOR_BITCLEAREQ
:
383 fprintf(file
, "&^=");
385 case OPERATOR_PLUSPLUS
:
388 case OPERATOR_MINUSMINUS
:
394 case OPERATOR_COLONEQ
:
397 case OPERATOR_SEMICOLON
:
406 case OPERATOR_LPAREN
:
409 case OPERATOR_RPAREN
:
412 case OPERATOR_LCURLY
:
415 case OPERATOR_RCURLY
:
418 case OPERATOR_LSQUARE
:
421 case OPERATOR_RSQUARE
:
435 Lex::Lex(const char* input_file_name
, FILE* input_file
)
436 : input_file_name_(input_file_name
), input_file_(input_file
),
437 linebuf_(NULL
), linebufsize_(120), linesize_(0), lineoff_(0),
438 lineno_(0), add_semi_at_eol_(false)
440 this->linebuf_
= new char[this->linebufsize_
];
441 linemap_add(line_table
, LC_ENTER
, 0, input_file_name
, 1);
446 delete[] this->linebuf_
;
447 linemap_add(line_table
, LC_LEAVE
, 0, NULL
, 0);
450 // Read a new line from the file.
455 char* buf
= this->linebuf_
;
456 size_t size
= this->linebufsize_
;
458 FILE* file
= this->input_file_
;
471 size_t ns
= 2 * size
+ 1;
472 if (ns
< size
|| static_cast<ssize_t
>(ns
) < 0)
473 error_at(this->location(), "out of memory");
474 char* nb
= new char[ns
];
475 memcpy(nb
, buf
, cur
);
489 this->linebuf_
= buf
;
490 this->linebufsize_
= size
;
495 // See if we need to read a new line. Return true if there is a new
496 // line, false if we are at EOF.
501 if (this->lineoff_
< this->linesize_
)
504 ssize_t got
= this->get_line();
508 this->linesize_
= got
;
511 linemap_line_start(line_table
, this->lineno_
, this->linesize_
);
516 // Get the current location.
519 Lex::location() const
521 source_location location
;
522 LINEMAP_POSITION_FOR_COLUMN(location
, line_table
, this->lineoff_
+ 1);
526 // Get a location slightly before the current one. This is used for
527 // slightly more efficient handling of operator tokens.
530 Lex::earlier_location(int chars
) const
532 source_location location
;
533 LINEMAP_POSITION_FOR_COLUMN(location
, line_table
, this->lineoff_
+ 1 - chars
);
537 // Get the next token.
544 if (!this->require_line())
546 bool add_semi_at_eol
= this->add_semi_at_eol_
;
547 this->add_semi_at_eol_
= false;
549 return this->make_operator(OPERATOR_SEMICOLON
, 1);
550 return this->make_eof_token();
553 const char* p
= this->linebuf_
+ this->lineoff_
;
554 const char* pend
= this->linebuf_
+ this->linesize_
;
558 unsigned char cc
= *p
;
561 case ' ': case '\t': case '\r':
563 // Skip whitespace quickly.
564 while (*p
== ' ' || *p
== '\t' || *p
== '\r')
571 bool add_semi_at_eol
= this->add_semi_at_eol_
;
572 this->add_semi_at_eol_
= false;
575 this->lineoff_
= p
- this->linebuf_
;
576 return this->make_operator(OPERATOR_SEMICOLON
, 1);
584 this->lineoff_
= p
+ 2 - this->linebuf_
;
585 this->skip_cpp_comment();
587 if (p
[-1] == '\n' && this->add_semi_at_eol_
)
590 else if (p
[1] == '*')
592 this->lineoff_
= p
- this->linebuf_
;
593 source_location location
= this->location();
594 if (!this->skip_c_comment())
595 return Token::make_invalid_token(location
);
596 p
= this->linebuf_
+ this->lineoff_
;
597 pend
= this->linebuf_
+ this->linesize_
;
599 else if (p
[1] == '=')
601 this->add_semi_at_eol_
= false;
602 this->lineoff_
= p
+ 2 - this->linebuf_
;
603 return this->make_operator(OPERATOR_DIVEQ
, 2);
607 this->add_semi_at_eol_
= false;
608 this->lineoff_
= p
+ 1 - this->linebuf_
;
609 return this->make_operator(OPERATOR_DIV
, 1);
613 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
614 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
615 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
616 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
618 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
619 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
620 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
621 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
624 this->lineoff_
= p
- this->linebuf_
;
625 return this->gather_identifier();
627 case '0': case '1': case '2': case '3': case '4':
628 case '5': case '6': case '7': case '8': case '9':
629 this->add_semi_at_eol_
= true;
630 this->lineoff_
= p
- this->linebuf_
;
631 return this->gather_number();
634 this->add_semi_at_eol_
= true;
635 this->lineoff_
= p
- this->linebuf_
;
636 return this->gather_character();
639 this->add_semi_at_eol_
= true;
640 this->lineoff_
= p
- this->linebuf_
;
641 return this->gather_string();
644 this->add_semi_at_eol_
= true;
645 this->lineoff_
= p
- this->linebuf_
;
646 return this->gather_raw_string();
653 this->add_semi_at_eol_
= false;
654 Operator op
= this->three_character_operator(cc
, p
[1], p
[2]);
655 if (op
!= OPERATOR_INVALID
)
657 this->lineoff_
= p
+ 3 - this->linebuf_
;
658 return this->make_operator(op
, 3);
669 // '/' handled above.
678 this->add_semi_at_eol_
= false;
679 Operator op
= this->two_character_operator(cc
, p
[1]);
681 if (op
!= OPERATOR_INVALID
)
688 op
= this->one_character_operator(cc
);
691 this->lineoff_
= p
+ 1 - this->linebuf_
;
692 return this->make_operator(op
, chars
);
696 if (p
[1] >= '0' && p
[1] <= '9')
698 this->add_semi_at_eol_
= true;
699 this->lineoff_
= p
- this->linebuf_
;
700 return this->gather_number();
702 if (p
[1] == '.' && p
[2] == '.')
704 this->add_semi_at_eol_
= false;
705 this->lineoff_
= p
+ 3 - this->linebuf_
;
706 return this->make_operator(OPERATOR_ELLIPSIS
, 3);
708 this->add_semi_at_eol_
= false;
709 this->lineoff_
= p
+ 1 - this->linebuf_
;
710 return this->make_operator(OPERATOR_DOT
, 1);
716 this->lineoff_
= p
- this->linebuf_
;
717 this->advance_one_utf8_char(p
, &ci
, &issued_error
);
718 if (Lex::is_unicode_letter(ci
))
719 return this->gather_identifier();
722 error_at(this->location(),
723 "invalid character 0x%x in input file",
733 this->lineoff_
= p
- this->linebuf_
;
737 // Fetch one UTF-8 character from a string. Set *VALUE to the value.
738 // Return the number of bytes read from the string. Returns 0 if the
739 // string does not point to a valid UTF-8 character.
742 Lex::fetch_char(const char* p
, unsigned int* value
)
744 unsigned char c
= *p
;
750 else if ((c
& 0xe0) == 0xc0
751 && (p
[1] & 0xc0) == 0x80)
753 *value
= (((c
& 0x1f) << 6)
762 else if ((c
& 0xf0) == 0xe0
763 && (p
[1] & 0xc0) == 0x80
764 && (p
[2] & 0xc0) == 0x80)
766 *value
= (((c
& 0xf) << 12)
767 + ((p
[1] & 0x3f) << 6)
776 else if ((c
& 0xf8) == 0xf0
777 && (p
[1] & 0xc0) == 0x80
778 && (p
[2] & 0xc0) == 0x80
779 && (p
[3] & 0xc0) == 0x80)
781 *value
= (((c
& 0x7) << 18)
782 + ((p
[1] & 0x3f) << 12)
783 + ((p
[2] & 0x3f) << 6)
785 if (*value
<= 0xffff)
794 /* Invalid encoding. Return the Unicode replacement
801 // Advance one UTF-8 character. Return the pointer beyond the
802 // character. Set *VALUE to the value. Set *ISSUED_ERROR if an error
806 Lex::advance_one_utf8_char(const char* p
, unsigned int* value
,
809 *issued_error
= false;
813 error_at(this->location(), "invalid NUL byte");
814 *issued_error
= true;
819 int adv
= Lex::fetch_char(p
, value
);
822 error_at(this->location(), "invalid UTF-8 encoding");
823 *issued_error
= true;
829 // Pick up an identifier.
832 Lex::gather_identifier()
834 const char* pstart
= this->linebuf_
+ this->lineoff_
;
835 const char* p
= pstart
;
836 const char* pend
= this->linebuf_
+ this->linesize_
;
837 bool is_first
= true;
838 bool is_exported
= false;
839 bool has_non_ascii_char
= false;
843 unsigned char cc
= *p
;
846 if ((cc
< 'A' || cc
> 'Z')
847 && (cc
< 'a' || cc
> 'z')
849 && (cc
< '0' || cc
> '9'))
854 is_exported
= cc
>= 'A' && cc
<= 'Z';
857 if (has_non_ascii_char
)
864 this->lineoff_
= p
- this->linebuf_
;
865 const char* pnext
= this->advance_one_utf8_char(p
, &ci
,
867 if (!Lex::is_unicode_letter(ci
) && !Lex::is_unicode_digit(ci
))
869 // There is no valid place for a non-ASCII character
870 // other than an identifier, so we get better error
871 // handling behaviour if we swallow this character after
874 error_at(this->location(),
875 "invalid character 0x%x in identifier",
880 is_exported
= Lex::is_unicode_uppercase(ci
);
883 if (!has_non_ascii_char
)
885 buf
.assign(pstart
, p
- pstart
);
886 has_non_ascii_char
= true;
890 // This assumes that all assemblers can handle an identifier
891 // with a '$' character.
892 snprintf(ubuf
, sizeof ubuf
, "$U%x$", ci
);
896 source_location location
= this->location();
897 this->add_semi_at_eol_
= true;
898 this->lineoff_
= p
- this->linebuf_
;
899 if (has_non_ascii_char
)
900 return Token::make_identifier_token(buf
, is_exported
, location
);
903 Keyword code
= keywords
.keyword_to_code(pstart
, p
- pstart
);
904 if (code
== KEYWORD_INVALID
)
905 return Token::make_identifier_token(std::string(pstart
, p
- pstart
),
906 is_exported
, location
);
912 case KEYWORD_CONTINUE
:
913 case KEYWORD_FALLTHROUGH
:
917 this->add_semi_at_eol_
= false;
920 return Token::make_keyword_token(code
, location
);
925 // Return whether C is a hex digit.
928 Lex::is_hex_digit(char c
)
930 return ((c
>= '0' && c
<= '9')
931 || (c
>= 'A' && c
<= 'F')
932 || (c
>= 'a' && c
<= 'f'));
935 // Return whether an exponent could start at P.
938 Lex::could_be_exponent(const char* p
, const char* pend
)
940 if (*p
!= 'e' && *p
!= 'E')
945 if (*p
== '+' || *p
== '-')
951 return *p
>= '0' && *p
<= '9';
959 const char* pstart
= this->linebuf_
+ this->lineoff_
;
960 const char* p
= pstart
;
961 const char* pend
= this->linebuf_
+ this->linesize_
;
963 source_location location
= this->location();
974 const char* pnum
= p
;
978 if ((p
[1] == 'x' || p
[1] == 'X')
979 && Lex::is_hex_digit(p
[2]))
986 if (!Lex::is_hex_digit(*p
))
997 if (*p
< '0' || *p
> '7')
1003 if (*p
!= '.' && *p
!= 'i' && !Lex::could_be_exponent(p
, pend
))
1005 std::string
s(pnum
, p
- pnum
);
1007 int r
= mpz_init_set_str(val
, s
.c_str(), base
);
1013 this->lineoff_
= p
- this->linebuf_
;
1014 Token ret
= Token::make_integer_token(val
, location
);
1022 if (*p
< '0' || *p
> '9')
1027 if (*p
!= '.' && *p
!= 'i' && !Lex::could_be_exponent(p
, pend
))
1029 std::string
s(pnum
, p
- pnum
);
1031 int r
= mpz_init_set_str(val
, s
.c_str(), 10);
1037 this->lineoff_
= p
- this->linebuf_
;
1038 Token ret
= Token::make_integer_token(val
, location
);
1045 bool dot
= *p
== '.';
1051 if (*p
== '+' || *p
== '-')
1057 if (*p
< '0' || *p
> '9')
1062 if (dot
&& Lex::could_be_exponent(p
, pend
))
1065 if (*p
== '+' || *p
== '-')
1069 if (*p
< '0' || *p
> '9')
1076 std::string
s(pnum
, p
- pnum
);
1078 int r
= mpfr_init_set_str(val
, s
.c_str(), 10, GMP_RNDN
);
1082 mpfr_neg(val
, val
, GMP_RNDN
);
1084 bool is_imaginary
= *p
== 'i';
1088 this->lineoff_
= p
- this->linebuf_
;
1091 Token ret
= Token::make_imaginary_token(val
, location
);
1097 Token ret
= Token::make_float_token(val
, location
);
1103 // Advance one character, possibly escaped. Return the pointer beyond
1104 // the character. Set *VALUE to the character. Set *IS_CHARACTER if
1105 // this is a character (e.g., 'a' or '\u1234') rather than a byte
1106 // value (e.g., '\001').
1109 Lex::advance_one_char(const char* p
, bool is_single_quote
, unsigned int* value
,
1113 *is_character
= true;
1117 const char* ret
= this->advance_one_utf8_char(p
, value
, &issued_error
);
1119 && (*value
== '\'' || *value
== '\n')
1121 error_at(this->location(), "invalid character literal");
1129 case '0': case '1': case '2': case '3':
1130 case '4': case '5': case '6': case '7':
1131 *is_character
= false;
1132 if (p
[1] >= '0' && p
[1] <= '7'
1133 && p
[2] >= '0' && p
[2] <= '7')
1135 *value
= ((Lex::octal_value(p
[0]) << 6)
1136 + (Lex::octal_value(p
[1]) << 3)
1137 + Lex::octal_value(p
[2]));
1140 error_at(this->location(), "invalid octal constant");
1145 error_at(this->location(), "invalid octal character");
1146 return (p
[1] >= '0' && p
[1] <= '7'
1152 *is_character
= false;
1153 if (Lex::is_hex_digit(p
[1]) && Lex::is_hex_digit(p
[2]))
1155 *value
= (hex_value(p
[1]) << 4) + hex_value(p
[2]);
1158 error_at(this->location(), "invalid hex character");
1159 return (Lex::is_hex_digit(p
[1])
1188 if (!is_single_quote
)
1189 error_at(this->location(), "invalid quoted character");
1193 if (is_single_quote
)
1194 error_at(this->location(), "invalid quoted character");
1199 if (Lex::is_hex_digit(p
[1]) && Lex::is_hex_digit(p
[2])
1200 && Lex::is_hex_digit(p
[3]) && Lex::is_hex_digit(p
[4]))
1202 *value
= ((hex_value(p
[1]) << 12)
1203 + (hex_value(p
[2]) << 8)
1204 + (hex_value(p
[3]) << 4)
1206 if (*value
>= 0xd800 && *value
< 0xe000)
1208 error_at(this->location(),
1209 "invalid unicode code point 0x%x",
1211 // Use the replacement character.
1216 error_at(this->location(), "invalid little unicode code point");
1220 if (Lex::is_hex_digit(p
[1]) && Lex::is_hex_digit(p
[2])
1221 && Lex::is_hex_digit(p
[3]) && Lex::is_hex_digit(p
[4])
1222 && Lex::is_hex_digit(p
[5]) && Lex::is_hex_digit(p
[6])
1223 && Lex::is_hex_digit(p
[7]) && Lex::is_hex_digit(p
[8]))
1225 *value
= ((hex_value(p
[1]) << 28)
1226 + (hex_value(p
[2]) << 24)
1227 + (hex_value(p
[3]) << 20)
1228 + (hex_value(p
[4]) << 16)
1229 + (hex_value(p
[5]) << 12)
1230 + (hex_value(p
[6]) << 8)
1231 + (hex_value(p
[7]) << 4)
1233 if (*value
> 0x10ffff
1234 || (*value
>= 0xd800 && *value
< 0xe000))
1236 error_at(this->location(), "invalid unicode code point 0x%x",
1238 // Use the replacement character.
1243 error_at(this->location(), "invalid big unicode code point");
1247 error_at(this->location(), "invalid character after %<\\%>");
1254 // Append V to STR. IS_CHARACTER is true for a character which should
1255 // be stored in UTF-8, false for a general byte value which should be
1259 Lex::append_char(unsigned int v
, bool is_character
, std::string
* str
,
1260 source_location location
)
1264 if (v
<= 0x7f || !is_character
)
1269 else if (v
<= 0x7ff)
1271 buf
[0] = 0xc0 + (v
>> 6);
1272 buf
[1] = 0x80 + (v
& 0x3f);
1279 warning_at(location
, 0,
1280 "unicode code point 0x%x out of range in string", v
);
1281 // Turn it into the "replacement character".
1286 buf
[0] = 0xe0 + (v
>> 12);
1287 buf
[1] = 0x80 + ((v
>> 6) & 0x3f);
1288 buf
[2] = 0x80 + (v
& 0x3f);
1293 buf
[0] = 0xf0 + (v
>> 18);
1294 buf
[1] = 0x80 + ((v
>> 12) & 0x3f);
1295 buf
[2] = 0x80 + ((v
>> 6) & 0x3f);
1296 buf
[3] = 0x80 + (v
& 0x3f);
1300 str
->append(buf
, len
);
1303 // Pick up a character literal.
1306 Lex::gather_character()
1309 const char* pstart
= this->linebuf_
+ this->lineoff_
;
1310 const char* p
= pstart
;
1314 p
= this->advance_one_char(p
, true, &value
, &is_character
);
1318 error_at(this->location(), "unterminated character constant");
1319 this->lineoff_
= p
- this->linebuf_
;
1320 return this->make_invalid_token();
1324 mpz_init_set_ui(val
, value
);
1326 source_location location
= this->location();
1327 this->lineoff_
= p
+ 1 - this->linebuf_
;
1328 Token ret
= Token::make_integer_token(val
, location
);
1333 // Pick up a quoted string.
1336 Lex::gather_string()
1338 const char* pstart
= this->linebuf_
+ this->lineoff_
+ 1;
1339 const char* p
= pstart
;
1340 const char* pend
= this->linebuf_
+ this->linesize_
;
1345 source_location loc
= this->location();
1348 this->lineoff_
= p
- this->linebuf_
;
1349 p
= this->advance_one_char(p
, false, &c
, &is_character
);
1352 error_at(this->location(), "unterminated string");
1356 Lex::append_char(c
, is_character
, &value
, loc
);
1359 source_location location
= this->location();
1360 this->lineoff_
= p
+ 1 - this->linebuf_
;
1361 return Token::make_string_token(value
, location
);
1364 // Pick up a raw string.
1367 Lex::gather_raw_string()
1369 const char* p
= this->linebuf_
+ this->lineoff_
+ 1;
1370 const char* pend
= this->linebuf_
+ this->linesize_
;
1371 source_location location
= this->location();
1380 this->lineoff_
= p
+ 1 - this->linebuf_
;
1381 return Token::make_string_token(value
, location
);
1383 source_location loc
= this->location();
1386 this->lineoff_
= p
- this->linebuf_
;
1387 p
= this->advance_one_utf8_char(p
, &c
, &issued_error
);
1388 Lex::append_char(c
, true, &value
, loc
);
1390 this->lineoff_
= p
- this->linebuf_
;
1391 if (!this->require_line())
1393 error_at(location
, "unterminated raw string");
1394 return Token::make_string_token(value
, location
);
1396 p
= this->linebuf_
+ this->lineoff_
;
1397 pend
= this->linebuf_
+ this->linesize_
;
1401 // If C1 C2 C3 are a three character operator, return the code.
1404 Lex::three_character_operator(char c1
, char c2
, char c3
)
1408 if (c1
== '<' && c2
== '<')
1409 return OPERATOR_LSHIFTEQ
;
1410 else if (c1
== '>' && c2
== '>')
1411 return OPERATOR_RSHIFTEQ
;
1412 else if (c1
== '&' && c2
== '^')
1413 return OPERATOR_BITCLEAREQ
;
1415 return OPERATOR_INVALID
;
1418 // If C1 C2 are a two character operator, return the code.
1421 Lex::two_character_operator(char c1
, char c2
)
1427 return OPERATOR_OROR
;
1429 return OPERATOR_OREQ
;
1433 return OPERATOR_ANDAND
;
1435 return OPERATOR_BITCLEAR
;
1437 return OPERATOR_ANDEQ
;
1441 return OPERATOR_XOREQ
;
1445 return OPERATOR_EQEQ
;
1449 return OPERATOR_NOTEQ
;
1455 return OPERATOR_LSHIFT
;
1457 return OPERATOR_CHANOP
;
1463 return OPERATOR_RSHIFT
;
1467 return OPERATOR_MULTEQ
;
1471 return OPERATOR_DIVEQ
;
1475 return OPERATOR_MODEQ
;
1480 this->add_semi_at_eol_
= true;
1481 return OPERATOR_PLUSPLUS
;
1484 return OPERATOR_PLUSEQ
;
1489 this->add_semi_at_eol_
= true;
1490 return OPERATOR_MINUSMINUS
;
1493 return OPERATOR_MINUSEQ
;
1497 return OPERATOR_COLONEQ
;
1502 return OPERATOR_INVALID
;
1505 // If character C is an operator, return the code.
1508 Lex::one_character_operator(char c
)
1517 return OPERATOR_PLUS
;
1519 return OPERATOR_MINUS
;
1523 return OPERATOR_XOR
;
1525 return OPERATOR_MULT
;
1527 return OPERATOR_DIV
;
1529 return OPERATOR_MOD
;
1531 return OPERATOR_AND
;
1533 return OPERATOR_NOT
;
1537 return OPERATOR_COLON
;
1539 return OPERATOR_SEMICOLON
;
1541 return OPERATOR_DOT
;
1543 return OPERATOR_COMMA
;
1545 return OPERATOR_LPAREN
;
1547 this->add_semi_at_eol_
= true;
1548 return OPERATOR_RPAREN
;
1550 return OPERATOR_LCURLY
;
1552 this->add_semi_at_eol_
= true;
1553 return OPERATOR_RCURLY
;
1555 return OPERATOR_LSQUARE
;
1557 this->add_semi_at_eol_
= true;
1558 return OPERATOR_RSQUARE
;
1560 return OPERATOR_INVALID
;
1564 // Skip a C-style comment.
1567 Lex::skip_c_comment()
1571 if (!this->require_line())
1573 error_at(this->location(), "unterminated comment");
1577 const char* p
= this->linebuf_
+ this->lineoff_
;
1578 const char* pend
= this->linebuf_
+ this->linesize_
;
1582 if (p
[0] == '*' && p
+ 1 < pend
&& p
[1] == '/')
1584 this->lineoff_
= p
+ 2 - this->linebuf_
;
1588 this->lineoff_
= p
- this->linebuf_
;
1591 p
= this->advance_one_utf8_char(p
, &c
, &issued_error
);
1594 this->lineoff_
= p
- this->linebuf_
;
1598 // Skip a C++-style comment.
1601 Lex::skip_cpp_comment()
1603 const char* p
= this->linebuf_
+ this->lineoff_
;
1604 const char* pend
= this->linebuf_
+ this->linesize_
;
1606 // By convention, a C++ comment at the start of the line of the form
1607 // //line FILE:LINENO
1608 // is interpreted as setting the file name and line number of the
1609 // next source line.
1611 if (this->lineoff_
== 2
1613 && memcmp(p
, "line ", 5) == 0)
1616 while (p
< pend
&& *p
== ' ')
1618 const char* pcolon
= static_cast<const char*>(memchr(p
, ':', pend
- p
));
1621 && pcolon
[1] <= '9')
1624 long lineno
= strtol(pcolon
+ 1, &plend
, 10);
1625 if (plend
> pcolon
+ 1
1630 && lineno
< 0x7fffffff)
1632 unsigned int filelen
= pcolon
- p
;
1633 char* file
= new char[filelen
+ 1];
1634 memcpy(file
, p
, filelen
);
1635 file
[filelen
] = '\0';
1637 linemap_add(line_table
, LC_LEAVE
, 0, NULL
, 0);
1638 linemap_add(line_table
, LC_ENTER
, 0, file
, lineno
);
1639 this->lineno_
= lineno
- 1;
1648 this->lineoff_
= p
- this->linebuf_
;
1651 p
= this->advance_one_utf8_char(p
, &c
, &issued_error
);
1655 // The Unicode tables use this struct.
1657 struct Unicode_range
1659 // The low end of the range.
1661 // The high end of the range.
1663 // The stride. This entries represents low, low + stride, low + 2 *
1664 // stride, etc., up to high.
1665 unsigned int stride
;
1668 // A table of Unicode digits--Unicode code points classified as
1671 static const Unicode_range unicode_digits
[] =
1673 { 0x0030, 0x0039, 1},
1674 { 0x0660, 0x0669, 1},
1675 { 0x06f0, 0x06f9, 1},
1676 { 0x07c0, 0x07c9, 1},
1677 { 0x0966, 0x096f, 1},
1678 { 0x09e6, 0x09ef, 1},
1679 { 0x0a66, 0x0a6f, 1},
1680 { 0x0ae6, 0x0aef, 1},
1681 { 0x0b66, 0x0b6f, 1},
1682 { 0x0be6, 0x0bef, 1},
1683 { 0x0c66, 0x0c6f, 1},
1684 { 0x0ce6, 0x0cef, 1},
1685 { 0x0d66, 0x0d6f, 1},
1686 { 0x0e50, 0x0e59, 1},
1687 { 0x0ed0, 0x0ed9, 1},
1688 { 0x0f20, 0x0f29, 1},
1689 { 0x1040, 0x1049, 1},
1690 { 0x17e0, 0x17e9, 1},
1691 { 0x1810, 0x1819, 1},
1692 { 0x1946, 0x194f, 1},
1693 { 0x19d0, 0x19d9, 1},
1694 { 0x1b50, 0x1b59, 1},
1695 { 0xff10, 0xff19, 1},
1696 { 0x104a0, 0x104a9, 1},
1697 { 0x1d7ce, 0x1d7ff, 1},
1700 // A table of Unicode letters--Unicode code points classified as
1703 static const Unicode_range unicode_letters
[] =
1705 { 0x0041, 0x005a, 1},
1706 { 0x0061, 0x007a, 1},
1707 { 0x00aa, 0x00b5, 11},
1708 { 0x00ba, 0x00ba, 1},
1709 { 0x00c0, 0x00d6, 1},
1710 { 0x00d8, 0x00f6, 1},
1711 { 0x00f8, 0x02c1, 1},
1712 { 0x02c6, 0x02d1, 1},
1713 { 0x02e0, 0x02e4, 1},
1714 { 0x02ec, 0x02ee, 2},
1715 { 0x0370, 0x0374, 1},
1716 { 0x0376, 0x0377, 1},
1717 { 0x037a, 0x037d, 1},
1718 { 0x0386, 0x0386, 1},
1719 { 0x0388, 0x038a, 1},
1720 { 0x038c, 0x038c, 1},
1721 { 0x038e, 0x03a1, 1},
1722 { 0x03a3, 0x03f5, 1},
1723 { 0x03f7, 0x0481, 1},
1724 { 0x048a, 0x0523, 1},
1725 { 0x0531, 0x0556, 1},
1726 { 0x0559, 0x0559, 1},
1727 { 0x0561, 0x0587, 1},
1728 { 0x05d0, 0x05ea, 1},
1729 { 0x05f0, 0x05f2, 1},
1730 { 0x0621, 0x064a, 1},
1731 { 0x066e, 0x066f, 1},
1732 { 0x0671, 0x06d3, 1},
1733 { 0x06d5, 0x06d5, 1},
1734 { 0x06e5, 0x06e6, 1},
1735 { 0x06ee, 0x06ef, 1},
1736 { 0x06fa, 0x06fc, 1},
1737 { 0x06ff, 0x0710, 17},
1738 { 0x0712, 0x072f, 1},
1739 { 0x074d, 0x07a5, 1},
1740 { 0x07b1, 0x07b1, 1},
1741 { 0x07ca, 0x07ea, 1},
1742 { 0x07f4, 0x07f5, 1},
1743 { 0x07fa, 0x07fa, 1},
1744 { 0x0904, 0x0939, 1},
1745 { 0x093d, 0x0950, 19},
1746 { 0x0958, 0x0961, 1},
1747 { 0x0971, 0x0972, 1},
1748 { 0x097b, 0x097f, 1},
1749 { 0x0985, 0x098c, 1},
1750 { 0x098f, 0x0990, 1},
1751 { 0x0993, 0x09a8, 1},
1752 { 0x09aa, 0x09b0, 1},
1753 { 0x09b2, 0x09b2, 1},
1754 { 0x09b6, 0x09b9, 1},
1755 { 0x09bd, 0x09ce, 17},
1756 { 0x09dc, 0x09dd, 1},
1757 { 0x09df, 0x09e1, 1},
1758 { 0x09f0, 0x09f1, 1},
1759 { 0x0a05, 0x0a0a, 1},
1760 { 0x0a0f, 0x0a10, 1},
1761 { 0x0a13, 0x0a28, 1},
1762 { 0x0a2a, 0x0a30, 1},
1763 { 0x0a32, 0x0a33, 1},
1764 { 0x0a35, 0x0a36, 1},
1765 { 0x0a38, 0x0a39, 1},
1766 { 0x0a59, 0x0a5c, 1},
1767 { 0x0a5e, 0x0a5e, 1},
1768 { 0x0a72, 0x0a74, 1},
1769 { 0x0a85, 0x0a8d, 1},
1770 { 0x0a8f, 0x0a91, 1},
1771 { 0x0a93, 0x0aa8, 1},
1772 { 0x0aaa, 0x0ab0, 1},
1773 { 0x0ab2, 0x0ab3, 1},
1774 { 0x0ab5, 0x0ab9, 1},
1775 { 0x0abd, 0x0ad0, 19},
1776 { 0x0ae0, 0x0ae1, 1},
1777 { 0x0b05, 0x0b0c, 1},
1778 { 0x0b0f, 0x0b10, 1},
1779 { 0x0b13, 0x0b28, 1},
1780 { 0x0b2a, 0x0b30, 1},
1781 { 0x0b32, 0x0b33, 1},
1782 { 0x0b35, 0x0b39, 1},
1783 { 0x0b3d, 0x0b3d, 1},
1784 { 0x0b5c, 0x0b5d, 1},
1785 { 0x0b5f, 0x0b61, 1},
1786 { 0x0b71, 0x0b83, 18},
1787 { 0x0b85, 0x0b8a, 1},
1788 { 0x0b8e, 0x0b90, 1},
1789 { 0x0b92, 0x0b95, 1},
1790 { 0x0b99, 0x0b9a, 1},
1791 { 0x0b9c, 0x0b9c, 1},
1792 { 0x0b9e, 0x0b9f, 1},
1793 { 0x0ba3, 0x0ba4, 1},
1794 { 0x0ba8, 0x0baa, 1},
1795 { 0x0bae, 0x0bb9, 1},
1796 { 0x0bd0, 0x0bd0, 1},
1797 { 0x0c05, 0x0c0c, 1},
1798 { 0x0c0e, 0x0c10, 1},
1799 { 0x0c12, 0x0c28, 1},
1800 { 0x0c2a, 0x0c33, 1},
1801 { 0x0c35, 0x0c39, 1},
1802 { 0x0c3d, 0x0c3d, 1},
1803 { 0x0c58, 0x0c59, 1},
1804 { 0x0c60, 0x0c61, 1},
1805 { 0x0c85, 0x0c8c, 1},
1806 { 0x0c8e, 0x0c90, 1},
1807 { 0x0c92, 0x0ca8, 1},
1808 { 0x0caa, 0x0cb3, 1},
1809 { 0x0cb5, 0x0cb9, 1},
1810 { 0x0cbd, 0x0cde, 33},
1811 { 0x0ce0, 0x0ce1, 1},
1812 { 0x0d05, 0x0d0c, 1},
1813 { 0x0d0e, 0x0d10, 1},
1814 { 0x0d12, 0x0d28, 1},
1815 { 0x0d2a, 0x0d39, 1},
1816 { 0x0d3d, 0x0d3d, 1},
1817 { 0x0d60, 0x0d61, 1},
1818 { 0x0d7a, 0x0d7f, 1},
1819 { 0x0d85, 0x0d96, 1},
1820 { 0x0d9a, 0x0db1, 1},
1821 { 0x0db3, 0x0dbb, 1},
1822 { 0x0dbd, 0x0dbd, 1},
1823 { 0x0dc0, 0x0dc6, 1},
1824 { 0x0e01, 0x0e30, 1},
1825 { 0x0e32, 0x0e33, 1},
1826 { 0x0e40, 0x0e46, 1},
1827 { 0x0e81, 0x0e82, 1},
1828 { 0x0e84, 0x0e84, 1},
1829 { 0x0e87, 0x0e88, 1},
1830 { 0x0e8a, 0x0e8d, 3},
1831 { 0x0e94, 0x0e97, 1},
1832 { 0x0e99, 0x0e9f, 1},
1833 { 0x0ea1, 0x0ea3, 1},
1834 { 0x0ea5, 0x0ea7, 2},
1835 { 0x0eaa, 0x0eab, 1},
1836 { 0x0ead, 0x0eb0, 1},
1837 { 0x0eb2, 0x0eb3, 1},
1838 { 0x0ebd, 0x0ebd, 1},
1839 { 0x0ec0, 0x0ec4, 1},
1840 { 0x0ec6, 0x0ec6, 1},
1841 { 0x0edc, 0x0edd, 1},
1842 { 0x0f00, 0x0f00, 1},
1843 { 0x0f40, 0x0f47, 1},
1844 { 0x0f49, 0x0f6c, 1},
1845 { 0x0f88, 0x0f8b, 1},
1846 { 0x1000, 0x102a, 1},
1847 { 0x103f, 0x103f, 1},
1848 { 0x1050, 0x1055, 1},
1849 { 0x105a, 0x105d, 1},
1850 { 0x1061, 0x1061, 1},
1851 { 0x1065, 0x1066, 1},
1852 { 0x106e, 0x1070, 1},
1853 { 0x1075, 0x1081, 1},
1854 { 0x108e, 0x108e, 1},
1855 { 0x10a0, 0x10c5, 1},
1856 { 0x10d0, 0x10fa, 1},
1857 { 0x10fc, 0x10fc, 1},
1858 { 0x1100, 0x1159, 1},
1859 { 0x115f, 0x11a2, 1},
1860 { 0x11a8, 0x11f9, 1},
1861 { 0x1200, 0x1248, 1},
1862 { 0x124a, 0x124d, 1},
1863 { 0x1250, 0x1256, 1},
1864 { 0x1258, 0x1258, 1},
1865 { 0x125a, 0x125d, 1},
1866 { 0x1260, 0x1288, 1},
1867 { 0x128a, 0x128d, 1},
1868 { 0x1290, 0x12b0, 1},
1869 { 0x12b2, 0x12b5, 1},
1870 { 0x12b8, 0x12be, 1},
1871 { 0x12c0, 0x12c0, 1},
1872 { 0x12c2, 0x12c5, 1},
1873 { 0x12c8, 0x12d6, 1},
1874 { 0x12d8, 0x1310, 1},
1875 { 0x1312, 0x1315, 1},
1876 { 0x1318, 0x135a, 1},
1877 { 0x1380, 0x138f, 1},
1878 { 0x13a0, 0x13f4, 1},
1879 { 0x1401, 0x166c, 1},
1880 { 0x166f, 0x1676, 1},
1881 { 0x1681, 0x169a, 1},
1882 { 0x16a0, 0x16ea, 1},
1883 { 0x1700, 0x170c, 1},
1884 { 0x170e, 0x1711, 1},
1885 { 0x1720, 0x1731, 1},
1886 { 0x1740, 0x1751, 1},
1887 { 0x1760, 0x176c, 1},
1888 { 0x176e, 0x1770, 1},
1889 { 0x1780, 0x17b3, 1},
1890 { 0x17d7, 0x17dc, 5},
1891 { 0x1820, 0x1877, 1},
1892 { 0x1880, 0x18a8, 1},
1893 { 0x18aa, 0x18aa, 1},
1894 { 0x1900, 0x191c, 1},
1895 { 0x1950, 0x196d, 1},
1896 { 0x1970, 0x1974, 1},
1897 { 0x1980, 0x19a9, 1},
1898 { 0x19c1, 0x19c7, 1},
1899 { 0x1a00, 0x1a16, 1},
1900 { 0x1b05, 0x1b33, 1},
1901 { 0x1b45, 0x1b4b, 1},
1902 { 0x1b83, 0x1ba0, 1},
1903 { 0x1bae, 0x1baf, 1},
1904 { 0x1c00, 0x1c23, 1},
1905 { 0x1c4d, 0x1c4f, 1},
1906 { 0x1c5a, 0x1c7d, 1},
1907 { 0x1d00, 0x1dbf, 1},
1908 { 0x1e00, 0x1f15, 1},
1909 { 0x1f18, 0x1f1d, 1},
1910 { 0x1f20, 0x1f45, 1},
1911 { 0x1f48, 0x1f4d, 1},
1912 { 0x1f50, 0x1f57, 1},
1913 { 0x1f59, 0x1f5d, 2},
1914 { 0x1f5f, 0x1f7d, 1},
1915 { 0x1f80, 0x1fb4, 1},
1916 { 0x1fb6, 0x1fbc, 1},
1917 { 0x1fbe, 0x1fbe, 1},
1918 { 0x1fc2, 0x1fc4, 1},
1919 { 0x1fc6, 0x1fcc, 1},
1920 { 0x1fd0, 0x1fd3, 1},
1921 { 0x1fd6, 0x1fdb, 1},
1922 { 0x1fe0, 0x1fec, 1},
1923 { 0x1ff2, 0x1ff4, 1},
1924 { 0x1ff6, 0x1ffc, 1},
1925 { 0x2071, 0x207f, 14},
1926 { 0x2090, 0x2094, 1},
1927 { 0x2102, 0x2107, 5},
1928 { 0x210a, 0x2113, 1},
1929 { 0x2115, 0x2115, 1},
1930 { 0x2119, 0x211d, 1},
1931 { 0x2124, 0x2128, 2},
1932 { 0x212a, 0x212d, 1},
1933 { 0x212f, 0x2139, 1},
1934 { 0x213c, 0x213f, 1},
1935 { 0x2145, 0x2149, 1},
1936 { 0x214e, 0x214e, 1},
1937 { 0x2183, 0x2184, 1},
1938 { 0x2c00, 0x2c2e, 1},
1939 { 0x2c30, 0x2c5e, 1},
1940 { 0x2c60, 0x2c6f, 1},
1941 { 0x2c71, 0x2c7d, 1},
1942 { 0x2c80, 0x2ce4, 1},
1943 { 0x2d00, 0x2d25, 1},
1944 { 0x2d30, 0x2d65, 1},
1945 { 0x2d6f, 0x2d6f, 1},
1946 { 0x2d80, 0x2d96, 1},
1947 { 0x2da0, 0x2da6, 1},
1948 { 0x2da8, 0x2dae, 1},
1949 { 0x2db0, 0x2db6, 1},
1950 { 0x2db8, 0x2dbe, 1},
1951 { 0x2dc0, 0x2dc6, 1},
1952 { 0x2dc8, 0x2dce, 1},
1953 { 0x2dd0, 0x2dd6, 1},
1954 { 0x2dd8, 0x2dde, 1},
1955 { 0x2e2f, 0x2e2f, 1},
1956 { 0x3005, 0x3006, 1},
1957 { 0x3031, 0x3035, 1},
1958 { 0x303b, 0x303c, 1},
1959 { 0x3041, 0x3096, 1},
1960 { 0x309d, 0x309f, 1},
1961 { 0x30a1, 0x30fa, 1},
1962 { 0x30fc, 0x30ff, 1},
1963 { 0x3105, 0x312d, 1},
1964 { 0x3131, 0x318e, 1},
1965 { 0x31a0, 0x31b7, 1},
1966 { 0x31f0, 0x31ff, 1},
1967 { 0x3400, 0x4db5, 1},
1968 { 0x4e00, 0x9fc3, 1},
1969 { 0xa000, 0xa48c, 1},
1970 { 0xa500, 0xa60c, 1},
1971 { 0xa610, 0xa61f, 1},
1972 { 0xa62a, 0xa62b, 1},
1973 { 0xa640, 0xa65f, 1},
1974 { 0xa662, 0xa66e, 1},
1975 { 0xa67f, 0xa697, 1},
1976 { 0xa717, 0xa71f, 1},
1977 { 0xa722, 0xa788, 1},
1978 { 0xa78b, 0xa78c, 1},
1979 { 0xa7fb, 0xa801, 1},
1980 { 0xa803, 0xa805, 1},
1981 { 0xa807, 0xa80a, 1},
1982 { 0xa80c, 0xa822, 1},
1983 { 0xa840, 0xa873, 1},
1984 { 0xa882, 0xa8b3, 1},
1985 { 0xa90a, 0xa925, 1},
1986 { 0xa930, 0xa946, 1},
1987 { 0xaa00, 0xaa28, 1},
1988 { 0xaa40, 0xaa42, 1},
1989 { 0xaa44, 0xaa4b, 1},
1990 { 0xac00, 0xd7a3, 1},
1991 { 0xf900, 0xfa2d, 1},
1992 { 0xfa30, 0xfa6a, 1},
1993 { 0xfa70, 0xfad9, 1},
1994 { 0xfb00, 0xfb06, 1},
1995 { 0xfb13, 0xfb17, 1},
1996 { 0xfb1d, 0xfb1d, 1},
1997 { 0xfb1f, 0xfb28, 1},
1998 { 0xfb2a, 0xfb36, 1},
1999 { 0xfb38, 0xfb3c, 1},
2000 { 0xfb3e, 0xfb3e, 1},
2001 { 0xfb40, 0xfb41, 1},
2002 { 0xfb43, 0xfb44, 1},
2003 { 0xfb46, 0xfbb1, 1},
2004 { 0xfbd3, 0xfd3d, 1},
2005 { 0xfd50, 0xfd8f, 1},
2006 { 0xfd92, 0xfdc7, 1},
2007 { 0xfdf0, 0xfdfb, 1},
2008 { 0xfe70, 0xfe74, 1},
2009 { 0xfe76, 0xfefc, 1},
2010 { 0xff21, 0xff3a, 1},
2011 { 0xff41, 0xff5a, 1},
2012 { 0xff66, 0xffbe, 1},
2013 { 0xffc2, 0xffc7, 1},
2014 { 0xffca, 0xffcf, 1},
2015 { 0xffd2, 0xffd7, 1},
2016 { 0xffda, 0xffdc, 1},
2017 { 0x10000, 0x1000b, 1},
2018 { 0x1000d, 0x10026, 1},
2019 { 0x10028, 0x1003a, 1},
2020 { 0x1003c, 0x1003d, 1},
2021 { 0x1003f, 0x1004d, 1},
2022 { 0x10050, 0x1005d, 1},
2023 { 0x10080, 0x100fa, 1},
2024 { 0x10280, 0x1029c, 1},
2025 { 0x102a0, 0x102d0, 1},
2026 { 0x10300, 0x1031e, 1},
2027 { 0x10330, 0x10340, 1},
2028 { 0x10342, 0x10349, 1},
2029 { 0x10380, 0x1039d, 1},
2030 { 0x103a0, 0x103c3, 1},
2031 { 0x103c8, 0x103cf, 1},
2032 { 0x10400, 0x1049d, 1},
2033 { 0x10800, 0x10805, 1},
2034 { 0x10808, 0x10808, 1},
2035 { 0x1080a, 0x10835, 1},
2036 { 0x10837, 0x10838, 1},
2037 { 0x1083c, 0x1083f, 3},
2038 { 0x10900, 0x10915, 1},
2039 { 0x10920, 0x10939, 1},
2040 { 0x10a00, 0x10a00, 1},
2041 { 0x10a10, 0x10a13, 1},
2042 { 0x10a15, 0x10a17, 1},
2043 { 0x10a19, 0x10a33, 1},
2044 { 0x12000, 0x1236e, 1},
2045 { 0x1d400, 0x1d454, 1},
2046 { 0x1d456, 0x1d49c, 1},
2047 { 0x1d49e, 0x1d49f, 1},
2048 { 0x1d4a2, 0x1d4a2, 1},
2049 { 0x1d4a5, 0x1d4a6, 1},
2050 { 0x1d4a9, 0x1d4ac, 1},
2051 { 0x1d4ae, 0x1d4b9, 1},
2052 { 0x1d4bb, 0x1d4bb, 1},
2053 { 0x1d4bd, 0x1d4c3, 1},
2054 { 0x1d4c5, 0x1d505, 1},
2055 { 0x1d507, 0x1d50a, 1},
2056 { 0x1d50d, 0x1d514, 1},
2057 { 0x1d516, 0x1d51c, 1},
2058 { 0x1d51e, 0x1d539, 1},
2059 { 0x1d53b, 0x1d53e, 1},
2060 { 0x1d540, 0x1d544, 1},
2061 { 0x1d546, 0x1d546, 1},
2062 { 0x1d54a, 0x1d550, 1},
2063 { 0x1d552, 0x1d6a5, 1},
2064 { 0x1d6a8, 0x1d6c0, 1},
2065 { 0x1d6c2, 0x1d6da, 1},
2066 { 0x1d6dc, 0x1d6fa, 1},
2067 { 0x1d6fc, 0x1d714, 1},
2068 { 0x1d716, 0x1d734, 1},
2069 { 0x1d736, 0x1d74e, 1},
2070 { 0x1d750, 0x1d76e, 1},
2071 { 0x1d770, 0x1d788, 1},
2072 { 0x1d78a, 0x1d7a8, 1},
2073 { 0x1d7aa, 0x1d7c2, 1},
2074 { 0x1d7c4, 0x1d7cb, 1},
2075 { 0x20000, 0x2a6d6, 1},
2076 { 0x2f800, 0x2fa1d, 1},
2079 // A table of Unicode uppercase letters--Unicode code points
2080 // classified as "Letter, uppercase".
2082 static const Unicode_range unicode_uppercase_letters
[] =
2084 { 0x0041, 0x005a, 1},
2085 { 0x00c0, 0x00d6, 1},
2086 { 0x00d8, 0x00de, 1},
2087 { 0x0100, 0x0136, 2},
2088 { 0x0139, 0x0147, 2},
2089 { 0x014a, 0x0176, 2},
2090 { 0x0178, 0x0179, 1},
2091 { 0x017b, 0x017d, 2},
2092 { 0x0181, 0x0182, 1},
2093 { 0x0184, 0x0184, 1},
2094 { 0x0186, 0x0187, 1},
2095 { 0x0189, 0x018b, 1},
2096 { 0x018e, 0x0191, 1},
2097 { 0x0193, 0x0194, 1},
2098 { 0x0196, 0x0198, 1},
2099 { 0x019c, 0x019d, 1},
2100 { 0x019f, 0x01a0, 1},
2101 { 0x01a2, 0x01a4, 2},
2102 { 0x01a6, 0x01a7, 1},
2103 { 0x01a9, 0x01ac, 3},
2104 { 0x01ae, 0x01af, 1},
2105 { 0x01b1, 0x01b3, 1},
2106 { 0x01b5, 0x01b5, 1},
2107 { 0x01b7, 0x01b8, 1},
2108 { 0x01bc, 0x01c4, 8},
2109 { 0x01c7, 0x01cd, 3},
2110 { 0x01cf, 0x01db, 2},
2111 { 0x01de, 0x01ee, 2},
2112 { 0x01f1, 0x01f4, 3},
2113 { 0x01f6, 0x01f8, 1},
2114 { 0x01fa, 0x0232, 2},
2115 { 0x023a, 0x023b, 1},
2116 { 0x023d, 0x023e, 1},
2117 { 0x0241, 0x0241, 1},
2118 { 0x0243, 0x0246, 1},
2119 { 0x0248, 0x024e, 2},
2120 { 0x0370, 0x0372, 2},
2121 { 0x0376, 0x0386, 16},
2122 { 0x0388, 0x038a, 1},
2123 { 0x038c, 0x038c, 1},
2124 { 0x038e, 0x038f, 1},
2125 { 0x0391, 0x03a1, 1},
2126 { 0x03a3, 0x03ab, 1},
2127 { 0x03cf, 0x03cf, 1},
2128 { 0x03d2, 0x03d4, 1},
2129 { 0x03d8, 0x03ee, 2},
2130 { 0x03f4, 0x03f7, 3},
2131 { 0x03f9, 0x03fa, 1},
2132 { 0x03fd, 0x042f, 1},
2133 { 0x0460, 0x0480, 2},
2134 { 0x048a, 0x04be, 2},
2135 { 0x04c0, 0x04c1, 1},
2136 { 0x04c3, 0x04cd, 2},
2137 { 0x04d0, 0x0522, 2},
2138 { 0x0531, 0x0556, 1},
2139 { 0x10a0, 0x10c5, 1},
2140 { 0x1e00, 0x1e94, 2},
2141 { 0x1e9e, 0x1efe, 2},
2142 { 0x1f08, 0x1f0f, 1},
2143 { 0x1f18, 0x1f1d, 1},
2144 { 0x1f28, 0x1f2f, 1},
2145 { 0x1f38, 0x1f3f, 1},
2146 { 0x1f48, 0x1f4d, 1},
2147 { 0x1f59, 0x1f5f, 2},
2148 { 0x1f68, 0x1f6f, 1},
2149 { 0x1fb8, 0x1fbb, 1},
2150 { 0x1fc8, 0x1fcb, 1},
2151 { 0x1fd8, 0x1fdb, 1},
2152 { 0x1fe8, 0x1fec, 1},
2153 { 0x1ff8, 0x1ffb, 1},
2154 { 0x2102, 0x2107, 5},
2155 { 0x210b, 0x210d, 1},
2156 { 0x2110, 0x2112, 1},
2157 { 0x2115, 0x2115, 1},
2158 { 0x2119, 0x211d, 1},
2159 { 0x2124, 0x2128, 2},
2160 { 0x212a, 0x212d, 1},
2161 { 0x2130, 0x2133, 1},
2162 { 0x213e, 0x213f, 1},
2163 { 0x2145, 0x2183, 62},
2164 { 0x2c00, 0x2c2e, 1},
2165 { 0x2c60, 0x2c60, 1},
2166 { 0x2c62, 0x2c64, 1},
2167 { 0x2c67, 0x2c6b, 2},
2168 { 0x2c6d, 0x2c6f, 1},
2169 { 0x2c72, 0x2c75, 3},
2170 { 0x2c80, 0x2ce2, 2},
2171 { 0xa640, 0xa65e, 2},
2172 { 0xa662, 0xa66c, 2},
2173 { 0xa680, 0xa696, 2},
2174 { 0xa722, 0xa72e, 2},
2175 { 0xa732, 0xa76e, 2},
2176 { 0xa779, 0xa77b, 2},
2177 { 0xa77d, 0xa77e, 1},
2178 { 0xa780, 0xa786, 2},
2179 { 0xa78b, 0xa78b, 1},
2180 { 0xff21, 0xff3a, 1},
2181 { 0x10400, 0x10427, 1},
2182 { 0x1d400, 0x1d419, 1},
2183 { 0x1d434, 0x1d44d, 1},
2184 { 0x1d468, 0x1d481, 1},
2185 { 0x1d49c, 0x1d49c, 1},
2186 { 0x1d49e, 0x1d49f, 1},
2187 { 0x1d4a2, 0x1d4a2, 1},
2188 { 0x1d4a5, 0x1d4a6, 1},
2189 { 0x1d4a9, 0x1d4ac, 1},
2190 { 0x1d4ae, 0x1d4b5, 1},
2191 { 0x1d4d0, 0x1d4e9, 1},
2192 { 0x1d504, 0x1d505, 1},
2193 { 0x1d507, 0x1d50a, 1},
2194 { 0x1d50d, 0x1d514, 1},
2195 { 0x1d516, 0x1d51c, 1},
2196 { 0x1d538, 0x1d539, 1},
2197 { 0x1d53b, 0x1d53e, 1},
2198 { 0x1d540, 0x1d544, 1},
2199 { 0x1d546, 0x1d546, 1},
2200 { 0x1d54a, 0x1d550, 1},
2201 { 0x1d56c, 0x1d585, 1},
2202 { 0x1d5a0, 0x1d5b9, 1},
2203 { 0x1d5d4, 0x1d5ed, 1},
2204 { 0x1d608, 0x1d621, 1},
2205 { 0x1d63c, 0x1d655, 1},
2206 { 0x1d670, 0x1d689, 1},
2207 { 0x1d6a8, 0x1d6c0, 1},
2208 { 0x1d6e2, 0x1d6fa, 1},
2209 { 0x1d71c, 0x1d734, 1},
2210 { 0x1d756, 0x1d76e, 1},
2211 { 0x1d790, 0x1d7a8, 1},
2212 { 0x1d7ca, 0x1d7ca, 1},
2215 // Return true if C is in RANGES.
2218 Lex::is_in_unicode_range(unsigned int c
, const Unicode_range
* ranges
,
2223 // The common case is a small value, and we know that it will be
2224 // in the first few entries of the table. Do a linear scan
2225 // rather than a binary search.
2226 for (size_t i
= 0; i
< range_size
; ++i
)
2228 const Unicode_range
* p
= &ranges
[i
];
2233 return (c
- p
->low
) % p
->stride
== 0;
2241 size_t hi
= range_size
;
2244 size_t mid
= lo
+ (hi
- lo
) / 2;
2245 const Unicode_range
* p
= &ranges
[mid
];
2248 else if (c
> p
->high
)
2251 return (c
- p
->low
) % p
->stride
== 0;
2257 // Return whether C is a Unicode digit--a Unicode code point
2258 // classified as "Digit".
2261 Lex::is_unicode_digit(unsigned int c
)
2263 return Lex::is_in_unicode_range(c
, unicode_digits
,
2264 ARRAY_SIZE(unicode_digits
));
2267 // Return whether C is a Unicode letter--a Unicode code point
2268 // classified as "Letter".
2271 Lex::is_unicode_letter(unsigned int c
)
2273 return Lex::is_in_unicode_range(c
, unicode_letters
,
2274 ARRAY_SIZE(unicode_letters
));
2277 // Return whether C is a Unicode uppercase letter. a Unicode code
2278 // point classified as "Letter, uppercase".
2281 Lex::is_unicode_uppercase(unsigned int c
)
2283 return Lex::is_in_unicode_range(c
, unicode_uppercase_letters
,
2284 ARRAY_SIZE(unicode_uppercase_letters
));
2287 // Return whether the identifier NAME should be exported. NAME is a
2288 // mangled name which includes only ASCII characters.
2291 Lex::is_exported_name(const std::string
& name
)
2293 unsigned char c
= name
[0];
2295 return c
>= 'A' && c
<= 'Z';
2298 const char* p
= name
.data();
2299 size_t len
= name
.length();
2300 if (len
< 2 || p
[1] != 'U')
2302 unsigned int ci
= 0;
2303 for (size_t i
= 2; i
< len
&& p
[i
] != '$'; ++i
)
2311 return Lex::is_unicode_uppercase(ci
);