1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000, 2001 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
7 Single-pass line tokenization by Neil Booth, April 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 2, or (at your option) any
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
28 /* MULTIBYTE_CHARS support only works for native compilers.
29 ??? Ideally what we want is to model widechar support after
30 the current floating point support. */
32 #undef MULTIBYTE_CHARS
35 #ifdef MULTIBYTE_CHARS
40 /* Tokens with SPELL_STRING store their spelling in the token list,
41 and it's length in the token->val.name.len. */
54 enum spell_type category
;
55 const unsigned char *name
;
58 static const unsigned char *const digraph_spellings
[] =
59 { U
"%:", U
"%:%:", U
"<:", U
":>", U
"<%", U
"%>" };
61 #define OP(e, s) { SPELL_OPERATOR, U s },
62 #define TK(e, s) { s, U STRINGX (e) },
63 static const struct token_spelling token_spellings
[N_TTYPES
] = { TTYPE_TABLE
};
67 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
68 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
69 #define BACKUP() do {buffer->cur = buffer->backup_to;} while (0)
71 static void handle_newline
PARAMS ((cpp_reader
*));
72 static cppchar_t skip_escaped_newlines
PARAMS ((cpp_reader
*));
73 static cppchar_t get_effective_char
PARAMS ((cpp_reader
*));
75 static int skip_block_comment
PARAMS ((cpp_reader
*));
76 static int skip_line_comment
PARAMS ((cpp_reader
*));
77 static void adjust_column
PARAMS ((cpp_reader
*));
78 static int skip_whitespace
PARAMS ((cpp_reader
*, cppchar_t
));
79 static cpp_hashnode
*parse_identifier
PARAMS ((cpp_reader
*));
80 static cpp_hashnode
*parse_identifier_slow
PARAMS ((cpp_reader
*,
82 static void parse_number
PARAMS ((cpp_reader
*, cpp_string
*, cppchar_t
, int));
83 static int unescaped_terminator_p
PARAMS ((cpp_reader
*, const U_CHAR
*));
84 static void parse_string
PARAMS ((cpp_reader
*, cpp_token
*, cppchar_t
));
85 static void unterminated
PARAMS ((cpp_reader
*, int));
86 static bool trigraph_p
PARAMS ((cpp_reader
*));
87 static void save_comment
PARAMS ((cpp_reader
*, cpp_token
*, const U_CHAR
*));
88 static int name_p
PARAMS ((cpp_reader
*, const cpp_string
*));
89 static int maybe_read_ucs
PARAMS ((cpp_reader
*, const unsigned char **,
90 const unsigned char *, unsigned int *));
91 static tokenrun
*next_tokenrun
PARAMS ((tokenrun
*));
93 static unsigned int hex_digit_value
PARAMS ((unsigned int));
94 static _cpp_buff
*new_buff
PARAMS ((size_t));
98 Compares, the token TOKEN to the NUL-terminated string STRING.
99 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
102 cpp_ideq (token
, string
)
103 const cpp_token
*token
;
106 if (token
->type
!= CPP_NAME
)
109 return !ustrcmp (NODE_NAME (token
->val
.node
), (const U_CHAR
*) string
);
112 /* Call when meeting a newline, assumed to be in buffer->cur[-1].
113 Returns with buffer->cur pointing to the character immediately
114 following the newline (combination). */
116 handle_newline (pfile
)
119 cpp_buffer
*buffer
= pfile
->buffer
;
121 /* Handle CR-LF and LF-CR. Most other implementations (e.g. java)
122 only accept CR-LF; maybe we should fall back to that behaviour? */
123 if (buffer
->cur
[-1] + buffer
->cur
[0] == '\r' + '\n')
126 buffer
->line_base
= buffer
->cur
;
127 buffer
->col_adjust
= 0;
131 /* Subroutine of skip_escaped_newlines; called when a 3-character
132 sequence beginning with "??" is encountered. buffer->cur points to
135 Warn if necessary, and returns true if the sequence forms a
136 trigraph and the trigraph should be honoured. */
141 cpp_buffer
*buffer
= pfile
->buffer
;
142 cppchar_t from_char
= buffer
->cur
[1];
145 if (!_cpp_trigraph_map
[from_char
])
148 accept
= CPP_OPTION (pfile
, trigraphs
);
150 /* Don't warn about trigraphs in comments. */
151 if (CPP_OPTION (pfile
, warn_trigraphs
) && !pfile
->state
.lexing_comment
)
154 cpp_warning_with_line (pfile
, pfile
->line
, CPP_BUF_COL (buffer
) - 1,
155 "trigraph ??%c converted to %c",
157 (int) _cpp_trigraph_map
[from_char
]);
158 else if (buffer
->cur
!= buffer
->last_Wtrigraphs
)
160 buffer
->last_Wtrigraphs
= buffer
->cur
;
161 cpp_warning_with_line (pfile
, pfile
->line
,
162 CPP_BUF_COL (buffer
) - 1,
163 "trigraph ??%c ignored", (int) from_char
);
170 /* Skips any escaped newlines introduced by '?' or a '\\', assumed to
171 lie in buffer->cur[-1]. Returns the next byte, which will be in
172 buffer->cur[-1]. This routine performs preprocessing stages 1 and
173 2 of the ISO C standard. */
175 skip_escaped_newlines (pfile
)
178 cpp_buffer
*buffer
= pfile
->buffer
;
179 cppchar_t next
= buffer
->cur
[-1];
181 /* Only do this if we apply stages 1 and 2. */
182 if (!buffer
->from_stage3
)
184 const unsigned char *saved_cur
;
191 if (buffer
->cur
[0] != '?' || !trigraph_p (pfile
))
194 /* Translate the trigraph. */
195 next
= _cpp_trigraph_map
[buffer
->cur
[1]];
201 if (buffer
->cur
== buffer
->rlimit
)
204 /* We have a backslash, and room for at least one more
205 character. Skip horizontal whitespace. */
206 saved_cur
= buffer
->cur
;
208 next1
= *buffer
->cur
++;
209 while (is_nvspace (next1
) && buffer
->cur
< buffer
->rlimit
);
211 if (!is_vspace (next1
))
213 buffer
->cur
= saved_cur
;
217 if (saved_cur
!= buffer
->cur
- 1
218 && !pfile
->state
.lexing_comment
)
219 cpp_warning (pfile
, "backslash and newline separated by space");
221 handle_newline (pfile
);
222 buffer
->backup_to
= buffer
->cur
;
223 if (buffer
->cur
== buffer
->rlimit
)
225 cpp_pedwarn (pfile
, "backslash-newline at end of file");
229 next
= *buffer
->cur
++;
231 while (next
== '\\' || next
== '?');
237 /* Obtain the next character, after trigraph conversion and skipping
238 an arbitrarily long string of escaped newlines. The common case of
239 no trigraphs or escaped newlines falls through quickly. On return,
240 buffer->backup_to points to where to return to if the character is
241 not to be processed. */
243 get_effective_char (pfile
)
247 cpp_buffer
*buffer
= pfile
->buffer
;
249 buffer
->backup_to
= buffer
->cur
;
250 next
= *buffer
->cur
++;
251 if (__builtin_expect (next
== '?' || next
== '\\', 0))
252 next
= skip_escaped_newlines (pfile
);
257 /* Skip a C-style block comment. We find the end of the comment by
258 seeing if an asterisk is before every '/' we encounter. Returns
259 non-zero if comment terminated by EOF, zero otherwise. */
261 skip_block_comment (pfile
)
264 cpp_buffer
*buffer
= pfile
->buffer
;
265 cppchar_t c
= EOF
, prevc
= EOF
;
267 pfile
->state
.lexing_comment
= 1;
268 while (buffer
->cur
!= buffer
->rlimit
)
270 prevc
= c
, c
= *buffer
->cur
++;
272 /* FIXME: For speed, create a new character class of characters
273 of interest inside block comments. */
274 if (c
== '?' || c
== '\\')
275 c
= skip_escaped_newlines (pfile
);
277 /* People like decorating comments with '*', so check for '/'
278 instead for efficiency. */
284 /* Warn about potential nested comments, but not if the '/'
285 comes immediately before the true comment delimiter.
286 Don't bother to get it right across escaped newlines. */
287 if (CPP_OPTION (pfile
, warn_comments
)
288 && buffer
->cur
[0] == '*' && buffer
->cur
[1] != '/')
289 cpp_warning_with_line (pfile
,
290 pfile
->line
, CPP_BUF_COL (buffer
),
291 "\"/*\" within comment");
293 else if (is_vspace (c
))
294 handle_newline (pfile
);
296 adjust_column (pfile
);
299 pfile
->state
.lexing_comment
= 0;
300 return c
!= '/' || prevc
!= '*';
303 /* Skip a C++ line comment, leaving buffer->cur pointing to the
304 terminating newline. Handles escaped newlines. Returns non-zero
305 if a multiline comment. */
307 skip_line_comment (pfile
)
310 cpp_buffer
*buffer
= pfile
->buffer
;
311 unsigned int orig_line
= pfile
->line
;
314 pfile
->state
.lexing_comment
= 1;
317 if (buffer
->cur
== buffer
->rlimit
)
321 if (c
== '?' || c
== '\\')
322 c
= skip_escaped_newlines (pfile
);
324 while (!is_vspace (c
));
326 /* Step back over the newline, except at EOF. */
330 pfile
->state
.lexing_comment
= 0;
331 return orig_line
!= pfile
->line
;
334 /* pfile->buffer->cur is one beyond the \t character. Update
335 col_adjust so we track the column correctly. */
337 adjust_column (pfile
)
340 cpp_buffer
*buffer
= pfile
->buffer
;
341 unsigned int col
= CPP_BUF_COL (buffer
) - 1; /* Zero-based column. */
343 /* Round it up to multiple of the tabstop, but subtract 1 since the
344 tab itself occupies a character position. */
345 buffer
->col_adjust
+= (CPP_OPTION (pfile
, tabstop
)
346 - col
% CPP_OPTION (pfile
, tabstop
)) - 1;
349 /* Skips whitespace, saving the next non-whitespace character.
350 Adjusts pfile->col_adjust to account for tabs. Without this,
351 tokens might be assigned an incorrect column. */
353 skip_whitespace (pfile
, c
)
357 cpp_buffer
*buffer
= pfile
->buffer
;
358 unsigned int warned
= 0;
362 /* Horizontal space always OK. */
366 adjust_column (pfile
);
367 /* Just \f \v or \0 left. */
370 if (buffer
->cur
- 1 == buffer
->rlimit
)
374 cpp_warning (pfile
, "null character(s) ignored");
378 else if (pfile
->state
.in_directive
&& CPP_PEDANTIC (pfile
))
379 cpp_pedwarn_with_line (pfile
, pfile
->line
,
380 CPP_BUF_COL (buffer
),
381 "%s in preprocessing directive",
382 c
== '\f' ? "form feed" : "vertical tab");
386 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
387 while (is_nvspace (c
));
393 /* See if the characters of a number token are valid in a name (no
396 name_p (pfile
, string
)
398 const cpp_string
*string
;
402 for (i
= 0; i
< string
->len
; i
++)
403 if (!is_idchar (string
->text
[i
]))
409 /* Parse an identifier, skipping embedded backslash-newlines. This is
410 a critical inner loop. The common case is an identifier which has
411 not been split by backslash-newline, does not contain a dollar
412 sign, and has already been scanned (roughly 10:1 ratio of
413 seen:unseen identifiers in normal code; the distribution is
414 Poisson-like). Second most common case is a new identifier, not
415 split and no dollar sign. The other possibilities are rare and
416 have been relegated to parse_identifier_slow. */
418 static cpp_hashnode
*
419 parse_identifier (pfile
)
422 cpp_hashnode
*result
;
425 /* Fast-path loop. Skim over a normal identifier.
426 N.B. ISIDNUM does not include $. */
427 cur
= pfile
->buffer
->cur
;
428 while (ISIDNUM (*cur
))
431 /* Check for slow-path cases. */
432 if (*cur
== '?' || *cur
== '\\' || *cur
== '$')
433 result
= parse_identifier_slow (pfile
, cur
);
436 const U_CHAR
*base
= pfile
->buffer
->cur
- 1;
437 result
= (cpp_hashnode
*)
438 ht_lookup (pfile
->hash_table
, base
, cur
- base
, HT_ALLOC
);
439 pfile
->buffer
->cur
= cur
;
442 /* Rarely, identifiers require diagnostics when lexed.
443 XXX Has to be forced out of the fast path. */
444 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
445 && !pfile
->state
.skipping
, 0))
447 /* It is allowed to poison the same identifier twice. */
448 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
449 cpp_error (pfile
, "attempt to use poisoned \"%s\"",
452 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
453 replacement list of a variadic macro. */
454 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
455 && !pfile
->state
.va_args_ok
)
457 "__VA_ARGS__ can only appear in the expansion of a C99 variadic macro");
463 /* Slow path. This handles identifiers which have been split, and
464 identifiers which contain dollar signs. The part of the identifier
465 from PFILE->buffer->cur-1 to CUR has already been scanned. */
466 static cpp_hashnode
*
467 parse_identifier_slow (pfile
, cur
)
471 cpp_buffer
*buffer
= pfile
->buffer
;
472 const U_CHAR
*base
= buffer
->cur
- 1;
473 struct obstack
*stack
= &pfile
->hash_table
->stack
;
474 unsigned int c
, saw_dollar
= 0, len
;
476 /* Copy the part of the token which is known to be okay. */
477 obstack_grow (stack
, base
, cur
- base
);
479 /* Now process the part which isn't. We are looking at one of
480 '$', '\\', or '?' on entry to this loop. */
485 while (is_idchar (c
))
487 obstack_1grow (stack
, c
);
495 /* Potential escaped newline? */
496 buffer
->backup_to
= buffer
->cur
- 1;
497 if (c
!= '?' && c
!= '\\')
499 c
= skip_escaped_newlines (pfile
);
501 while (is_idchar (c
));
503 /* Step back over the unwanted char. */
506 /* $ is not an identifier character in the standard, but is commonly
507 accepted as an extension. Don't warn about it in skipped
508 conditional blocks. */
509 if (saw_dollar
&& CPP_PEDANTIC (pfile
) && ! pfile
->state
.skipping
)
510 cpp_pedwarn (pfile
, "'$' character(s) in identifier");
512 /* Identifiers are null-terminated. */
513 len
= obstack_object_size (stack
);
514 obstack_1grow (stack
, '\0');
516 return (cpp_hashnode
*)
517 ht_lookup (pfile
->hash_table
, obstack_finish (stack
), len
, HT_ALLOCED
);
520 /* Parse a number, skipping embedded backslash-newlines. */
522 parse_number (pfile
, number
, c
, leading_period
)
528 cpp_buffer
*buffer
= pfile
->buffer
;
529 unsigned char *dest
, *limit
;
531 dest
= BUFF_FRONT (pfile
->u_buff
);
532 limit
= BUFF_LIMIT (pfile
->u_buff
);
534 /* Place a leading period. */
539 _cpp_extend_buff (pfile
, &pfile
->u_buff
, 1);
540 dest
= BUFF_FRONT (pfile
->u_buff
);
541 limit
= BUFF_LIMIT (pfile
->u_buff
);
550 /* Need room for terminating null. */
551 if ((size_t) (limit
- dest
) < 2)
553 size_t len_so_far
= dest
- BUFF_FRONT (pfile
->u_buff
);
554 _cpp_extend_buff (pfile
, &pfile
->u_buff
, 2);
555 dest
= BUFF_FRONT (pfile
->u_buff
) + len_so_far
;
556 limit
= BUFF_LIMIT (pfile
->u_buff
);
562 while (is_numchar (c
) || c
== '.' || VALID_SIGN (c
, dest
[-1]));
564 /* Potential escaped newline? */
565 buffer
->backup_to
= buffer
->cur
- 1;
566 if (c
!= '?' && c
!= '\\')
568 c
= skip_escaped_newlines (pfile
);
570 while (is_numchar (c
) || c
== '.' || VALID_SIGN (c
, dest
[-1]));
572 /* Step back over the unwanted char. */
575 /* Null-terminate the number. */
578 number
->text
= BUFF_FRONT (pfile
->u_buff
);
579 number
->len
= dest
- number
->text
;
580 BUFF_FRONT (pfile
->u_buff
) = dest
+ 1;
583 /* Subroutine of parse_string. Emits error for unterminated strings. */
585 unterminated (pfile
, term
)
589 cpp_error (pfile
, "missing terminating %c character", term
);
591 if (term
== '\"' && pfile
->mls_line
&& pfile
->mls_line
!= pfile
->line
)
593 cpp_error_with_line (pfile
, pfile
->mls_line
, pfile
->mls_col
,
594 "possible start of unterminated string literal");
599 /* Subroutine of parse_string. */
601 unescaped_terminator_p (pfile
, dest
)
603 const unsigned char *dest
;
605 const unsigned char *start
, *temp
;
607 /* In #include-style directives, terminators are not escapeable. */
608 if (pfile
->state
.angled_headers
)
611 start
= BUFF_FRONT (pfile
->u_buff
);
613 /* An odd number of consecutive backslashes represents an escaped
615 for (temp
= dest
; temp
> start
&& temp
[-1] == '\\'; temp
--)
618 return ((dest
- temp
) & 1) == 0;
621 /* Parses a string, character constant, or angle-bracketed header file
622 name. Handles embedded trigraphs and escaped newlines. The stored
623 string is guaranteed NUL-terminated, but it is not guaranteed that
624 this is the first NUL since embedded NULs are preserved.
625 Multi-line strings are allowed, but they are deprecated.
627 When this function returns, buffer->cur points to the next
628 character to be processed. */
630 parse_string (pfile
, token
, terminator
)
633 cppchar_t terminator
;
635 cpp_buffer
*buffer
= pfile
->buffer
;
636 unsigned char *dest
, *limit
;
638 bool warned_nulls
= false, warned_multi
= false;
640 dest
= BUFF_FRONT (pfile
->u_buff
);
641 limit
= BUFF_LIMIT (pfile
->u_buff
);
645 /* We need room for another char, possibly the terminating NUL. */
646 if ((size_t) (limit
- dest
) < 1)
648 size_t len_so_far
= dest
- BUFF_FRONT (pfile
->u_buff
);
649 _cpp_extend_buff (pfile
, &pfile
->u_buff
, 2);
650 dest
= BUFF_FRONT (pfile
->u_buff
) + len_so_far
;
651 limit
= BUFF_LIMIT (pfile
->u_buff
);
654 /* Handle trigraphs, escaped newlines etc. */
656 if (c
== '?' || c
== '\\')
657 c
= skip_escaped_newlines (pfile
);
661 if (unescaped_terminator_p (pfile
, dest
))
664 else if (is_vspace (c
))
666 /* In assembly language, silently terminate string and
667 character literals at end of line. This is a kludge
668 around not knowing where comments are. */
669 if (CPP_OPTION (pfile
, lang
) == CLK_ASM
&& terminator
!= '>')
675 /* Character constants and header names may not extend over
676 multiple lines. In Standard C, neither may strings.
677 Unfortunately, we accept multiline strings as an
678 extension, except in #include family directives. */
679 if (terminator
!= '"' || pfile
->state
.angled_headers
)
681 unterminated (pfile
, terminator
);
689 cpp_pedwarn (pfile
, "multi-line string literals are deprecated");
692 if (pfile
->mls_line
== 0)
694 pfile
->mls_line
= token
->line
;
695 pfile
->mls_col
= token
->col
;
698 handle_newline (pfile
);
703 if (buffer
->cur
- 1 == buffer
->rlimit
)
705 unterminated (pfile
, terminator
);
712 cpp_warning (pfile
, "null character(s) preserved in literal");
721 token
->val
.str
.text
= BUFF_FRONT (pfile
->u_buff
);
722 token
->val
.str
.len
= dest
- BUFF_FRONT (pfile
->u_buff
);
723 BUFF_FRONT (pfile
->u_buff
) = dest
+ 1;
726 /* The stored comment includes the comment start and any terminator. */
728 save_comment (pfile
, token
, from
)
731 const unsigned char *from
;
733 unsigned char *buffer
;
736 len
= pfile
->buffer
->cur
- from
+ 1; /* + 1 for the initial '/'. */
738 /* C++ comments probably (not definitely) have moved past a new
739 line, which we don't want to save in the comment. */
740 if (is_vspace (pfile
->buffer
->cur
[-1]))
742 buffer
= _cpp_unaligned_alloc (pfile
, len
);
744 token
->type
= CPP_COMMENT
;
745 token
->val
.str
.len
= len
;
746 token
->val
.str
.text
= buffer
;
749 memcpy (buffer
+ 1, from
, len
- 1);
752 /* Allocate COUNT tokens for RUN. */
754 _cpp_init_tokenrun (run
, count
)
758 run
->base
= xnewvec (cpp_token
, count
);
759 run
->limit
= run
->base
+ count
;
763 /* Returns the next tokenrun, or creates one if there is none. */
768 if (run
->next
== NULL
)
770 run
->next
= xnew (tokenrun
);
771 run
->next
->prev
= run
;
772 _cpp_init_tokenrun (run
->next
, 250);
778 /* Allocate a single token that is invalidated at the same time as the
779 rest of the tokens on the line. Has its line and col set to the
780 same as the last lexed token, so that diagnostics appear in the
783 _cpp_temp_token (pfile
)
786 cpp_token
*old
, *result
;
788 old
= pfile
->cur_token
- 1;
789 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
791 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
792 pfile
->cur_token
= pfile
->cur_run
->base
;
795 result
= pfile
->cur_token
++;
796 result
->line
= old
->line
;
797 result
->col
= old
->col
;
801 /* Lex a token into RESULT (external interface). Takes care of issues
802 like directive handling, token lookahead, multiple include
803 optimization and skipping. */
805 _cpp_lex_token (pfile
)
812 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
814 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
815 pfile
->cur_token
= pfile
->cur_run
->base
;
818 if (pfile
->lookaheads
)
821 result
= pfile
->cur_token
++;
824 result
= _cpp_lex_direct (pfile
);
826 if (result
->flags
& BOL
)
828 /* Is this a directive. If _cpp_handle_directive returns
829 false, it is an assembler #. */
830 if (result
->type
== CPP_HASH
831 && !pfile
->state
.parsing_args
832 && _cpp_handle_directive (pfile
, result
->flags
& PREV_WHITE
))
834 if (pfile
->cb
.line_change
&& !pfile
->state
.skipping
)
835 (*pfile
->cb
.line_change
)(pfile
, result
, pfile
->state
.parsing_args
);
838 /* We don't skip tokens in directives. */
839 if (pfile
->state
.in_directive
)
842 /* Outside a directive, invalidate controlling macros. At file
843 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
844 get here and MI optimisation works. */
845 pfile
->mi_valid
= false;
847 if (!pfile
->state
.skipping
|| result
->type
== CPP_EOF
)
854 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
856 if (get_effective_char (pfile) == CHAR) \
857 result->type = THEN_TYPE; \
861 result->type = ELSE_TYPE; \
865 /* Lex a token into pfile->cur_token, which is also incremented, to
866 get diagnostics pointing to the correct location.
868 Does not handle issues such as token lookahead, multiple-include
869 optimisation, directives, skipping etc. This function is only
870 suitable for use by _cpp_lex_token, and in special cases like
871 lex_expansion_token which doesn't care for any of these issues.
873 When meeting a newline, returns CPP_EOF if parsing a directive,
874 otherwise returns to the start of the token buffer if permissible.
875 Returns the location of the lexed token. */
877 _cpp_lex_direct (pfile
)
882 const unsigned char *comment_start
;
883 cpp_token
*result
= pfile
->cur_token
++;
886 buffer
= pfile
->buffer
;
887 result
->flags
= buffer
->saved_flags
;
888 buffer
->saved_flags
= 0;
890 result
->line
= pfile
->line
;
894 result
->col
= CPP_BUF_COLUMN (buffer
, buffer
->cur
);
899 case ' ': case '\t': case '\f': case '\v': case '\0':
900 result
->flags
|= PREV_WHITE
;
901 if (skip_whitespace (pfile
, c
))
906 buffer
->saved_flags
= BOL
;
907 if (!pfile
->state
.parsing_args
&& !pfile
->state
.in_directive
)
909 if (buffer
->cur
!= buffer
->line_base
)
911 /* Non-empty files should end in a newline. Don't warn
912 for command line and _Pragma buffers. */
913 if (!buffer
->from_stage3
)
914 cpp_pedwarn (pfile
, "no newline at end of file");
915 handle_newline (pfile
);
918 /* Don't pop the last buffer. */
921 unsigned char stop
= buffer
->return_at_eof
;
923 _cpp_pop_buffer (pfile
);
928 result
->type
= CPP_EOF
;
931 case '\n': case '\r':
932 handle_newline (pfile
);
933 buffer
->saved_flags
= BOL
;
934 if (! pfile
->state
.in_directive
)
936 if (pfile
->state
.parsing_args
== 2)
937 buffer
->saved_flags
|= PREV_WHITE
;
938 if (!pfile
->keep_tokens
)
940 pfile
->cur_run
= &pfile
->base_run
;
941 result
= pfile
->base_run
.base
;
942 pfile
->cur_token
= result
+ 1;
946 result
->type
= CPP_EOF
;
951 /* These could start an escaped newline, or '?' a trigraph. Let
952 skip_escaped_newlines do all the work. */
954 unsigned int line
= pfile
->line
;
956 c
= skip_escaped_newlines (pfile
);
957 if (line
!= pfile
->line
)
960 /* We had at least one escaped newline of some sort.
961 Update the token's line and column. */
962 goto update_tokens_line
;
966 /* We are either the original '?' or '\\', or a trigraph. */
968 result
->type
= CPP_QUERY
;
975 case '0': case '1': case '2': case '3': case '4':
976 case '5': case '6': case '7': case '8': case '9':
977 result
->type
= CPP_NUMBER
;
978 parse_number (pfile
, &result
->val
.str
, c
, 0);
982 if (!CPP_OPTION (pfile
, dollars_in_ident
))
984 /* Fall through... */
987 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
988 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
989 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
990 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
992 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
993 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
994 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
995 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
997 result
->type
= CPP_NAME
;
998 result
->val
.node
= parse_identifier (pfile
);
1000 /* 'L' may introduce wide characters or strings. */
1001 if (result
->val
.node
== pfile
->spec_nodes
.n_L
)
1004 if (c
== '\'' || c
== '"')
1007 result
->type
= (c
== '"' ? CPP_WSTRING
: CPP_WCHAR
);
1008 parse_string (pfile
, result
, c
);
1011 /* Convert named operators to their proper types. */
1012 else if (result
->val
.node
->flags
& NODE_OPERATOR
)
1014 result
->flags
|= NAMED_OP
;
1015 result
->type
= result
->val
.node
->value
.operator;
1021 result
->type
= c
== '"' ? CPP_STRING
: CPP_CHAR
;
1022 parse_string (pfile
, result
, c
);
1026 /* A potential block or line comment. */
1027 comment_start
= buffer
->cur
;
1028 c
= get_effective_char (pfile
);
1032 if (skip_block_comment (pfile
))
1033 cpp_error (pfile
, "unterminated comment");
1035 else if (c
== '/' && (CPP_OPTION (pfile
, cplusplus_comments
)
1036 || CPP_IN_SYSTEM_HEADER (pfile
)))
1038 /* Warn about comments only if pedantically GNUC89, and not
1039 in system headers. */
1040 if (CPP_OPTION (pfile
, lang
) == CLK_GNUC89
&& CPP_PEDANTIC (pfile
)
1041 && ! buffer
->warned_cplusplus_comments
)
1044 "C++ style comments are not allowed in ISO C89");
1046 "(this will be reported only once per input file)");
1047 buffer
->warned_cplusplus_comments
= 1;
1050 if (skip_line_comment (pfile
) && CPP_OPTION (pfile
, warn_comments
))
1051 cpp_warning (pfile
, "multi-line comment");
1055 result
->type
= CPP_DIV_EQ
;
1061 result
->type
= CPP_DIV
;
1065 if (!pfile
->state
.save_comments
)
1067 result
->flags
|= PREV_WHITE
;
1068 goto update_tokens_line
;
1071 /* Save the comment as a token in its own right. */
1072 save_comment (pfile
, result
, comment_start
);
1076 if (pfile
->state
.angled_headers
)
1078 result
->type
= CPP_HEADER_NAME
;
1079 parse_string (pfile
, result
, '>');
1083 c
= get_effective_char (pfile
);
1085 result
->type
= CPP_LESS_EQ
;
1087 IF_NEXT_IS ('=', CPP_LSHIFT_EQ
, CPP_LSHIFT
);
1088 else if (c
== '?' && CPP_OPTION (pfile
, cplusplus
))
1089 IF_NEXT_IS ('=', CPP_MIN_EQ
, CPP_MIN
);
1090 else if (c
== ':' && CPP_OPTION (pfile
, digraphs
))
1092 result
->type
= CPP_OPEN_SQUARE
;
1093 result
->flags
|= DIGRAPH
;
1095 else if (c
== '%' && CPP_OPTION (pfile
, digraphs
))
1097 result
->type
= CPP_OPEN_BRACE
;
1098 result
->flags
|= DIGRAPH
;
1103 result
->type
= CPP_LESS
;
1108 c
= get_effective_char (pfile
);
1110 result
->type
= CPP_GREATER_EQ
;
1112 IF_NEXT_IS ('=', CPP_RSHIFT_EQ
, CPP_RSHIFT
);
1113 else if (c
== '?' && CPP_OPTION (pfile
, cplusplus
))
1114 IF_NEXT_IS ('=', CPP_MAX_EQ
, CPP_MAX
);
1118 result
->type
= CPP_GREATER
;
1123 c
= get_effective_char (pfile
);
1125 result
->type
= CPP_MOD_EQ
;
1126 else if (CPP_OPTION (pfile
, digraphs
) && c
== ':')
1128 result
->flags
|= DIGRAPH
;
1129 result
->type
= CPP_HASH
;
1130 if (get_effective_char (pfile
) == '%')
1132 const unsigned char *pos
= buffer
->cur
;
1134 if (get_effective_char (pfile
) == ':')
1135 result
->type
= CPP_PASTE
;
1137 buffer
->cur
= pos
- 1;
1142 else if (CPP_OPTION (pfile
, digraphs
) && c
== '>')
1144 result
->flags
|= DIGRAPH
;
1145 result
->type
= CPP_CLOSE_BRACE
;
1150 result
->type
= CPP_MOD
;
1155 result
->type
= CPP_DOT
;
1156 c
= get_effective_char (pfile
);
1159 const unsigned char *pos
= buffer
->cur
;
1161 if (get_effective_char (pfile
) == '.')
1162 result
->type
= CPP_ELLIPSIS
;
1164 buffer
->cur
= pos
- 1;
1166 /* All known character sets have 0...9 contiguous. */
1167 else if (ISDIGIT (c
))
1169 result
->type
= CPP_NUMBER
;
1170 parse_number (pfile
, &result
->val
.str
, c
, 1);
1172 else if (c
== '*' && CPP_OPTION (pfile
, cplusplus
))
1173 result
->type
= CPP_DOT_STAR
;
1179 c
= get_effective_char (pfile
);
1181 result
->type
= CPP_PLUS_PLUS
;
1183 result
->type
= CPP_PLUS_EQ
;
1187 result
->type
= CPP_PLUS
;
1192 c
= get_effective_char (pfile
);
1195 result
->type
= CPP_DEREF
;
1196 if (CPP_OPTION (pfile
, cplusplus
))
1198 if (get_effective_char (pfile
) == '*')
1199 result
->type
= CPP_DEREF_STAR
;
1205 result
->type
= CPP_MINUS_MINUS
;
1207 result
->type
= CPP_MINUS_EQ
;
1211 result
->type
= CPP_MINUS
;
1216 c
= get_effective_char (pfile
);
1218 result
->type
= CPP_AND_AND
;
1220 result
->type
= CPP_AND_EQ
;
1224 result
->type
= CPP_AND
;
1229 c
= get_effective_char (pfile
);
1231 result
->type
= CPP_OR_OR
;
1233 result
->type
= CPP_OR_EQ
;
1237 result
->type
= CPP_OR
;
1242 c
= get_effective_char (pfile
);
1243 if (c
== ':' && CPP_OPTION (pfile
, cplusplus
))
1244 result
->type
= CPP_SCOPE
;
1245 else if (c
== '>' && CPP_OPTION (pfile
, digraphs
))
1247 result
->flags
|= DIGRAPH
;
1248 result
->type
= CPP_CLOSE_SQUARE
;
1253 result
->type
= CPP_COLON
;
1257 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ
, CPP_MULT
); break;
1258 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ
, CPP_EQ
); break;
1259 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ
, CPP_NOT
); break;
1260 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ
, CPP_XOR
); break;
1261 case '#': IF_NEXT_IS ('#', CPP_PASTE
, CPP_HASH
); break;
1263 case '~': result
->type
= CPP_COMPL
; break;
1264 case ',': result
->type
= CPP_COMMA
; break;
1265 case '(': result
->type
= CPP_OPEN_PAREN
; break;
1266 case ')': result
->type
= CPP_CLOSE_PAREN
; break;
1267 case '[': result
->type
= CPP_OPEN_SQUARE
; break;
1268 case ']': result
->type
= CPP_CLOSE_SQUARE
; break;
1269 case '{': result
->type
= CPP_OPEN_BRACE
; break;
1270 case '}': result
->type
= CPP_CLOSE_BRACE
; break;
1271 case ';': result
->type
= CPP_SEMICOLON
; break;
1273 /* @ is a punctuator in Objective C. */
1274 case '@': result
->type
= CPP_ATSIGN
; break;
1278 result
->type
= CPP_OTHER
;
1286 /* An upper bound on the number of bytes needed to spell a token,
1287 including preceding whitespace. */
1289 cpp_token_len (token
)
1290 const cpp_token
*token
;
1294 switch (TOKEN_SPELL (token
))
1296 default: len
= 0; break;
1298 case SPELL_STRING
: len
= token
->val
.str
.len
; break;
1299 case SPELL_IDENT
: len
= NODE_LEN (token
->val
.node
); break;
1301 /* 1 for whitespace, 4 for comment delimiters. */
1305 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
1306 already contain the enough space to hold the token's spelling.
1307 Returns a pointer to the character after the last character
1310 cpp_spell_token (pfile
, token
, buffer
)
1311 cpp_reader
*pfile
; /* Would be nice to be rid of this... */
1312 const cpp_token
*token
;
1313 unsigned char *buffer
;
1315 switch (TOKEN_SPELL (token
))
1317 case SPELL_OPERATOR
:
1319 const unsigned char *spelling
;
1322 if (token
->flags
& DIGRAPH
)
1324 = digraph_spellings
[(int) token
->type
- (int) CPP_FIRST_DIGRAPH
];
1325 else if (token
->flags
& NAMED_OP
)
1328 spelling
= TOKEN_NAME (token
);
1330 while ((c
= *spelling
++) != '\0')
1336 *buffer
++ = token
->val
.c
;
1341 memcpy (buffer
, NODE_NAME (token
->val
.node
), NODE_LEN (token
->val
.node
));
1342 buffer
+= NODE_LEN (token
->val
.node
);
1346 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
1347 buffer
+= token
->val
.str
.len
;
1352 int left
, right
, tag
;
1353 switch (token
->type
)
1355 case CPP_STRING
: left
= '"'; right
= '"'; tag
= '\0'; break;
1356 case CPP_WSTRING
: left
= '"'; right
= '"'; tag
= 'L'; break;
1357 case CPP_CHAR
: left
= '\''; right
= '\''; tag
= '\0'; break;
1358 case CPP_WCHAR
: left
= '\''; right
= '\''; tag
= 'L'; break;
1359 case CPP_HEADER_NAME
: left
= '<'; right
= '>'; tag
= '\0'; break;
1361 cpp_ice (pfile
, "unknown string token %s\n", TOKEN_NAME (token
));
1364 if (tag
) *buffer
++ = tag
;
1366 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
1367 buffer
+= token
->val
.str
.len
;
1373 cpp_ice (pfile
, "Unspellable token %s", TOKEN_NAME (token
));
1380 /* Returns a token as a null-terminated string. The string is
1381 temporary, and automatically freed later. Useful for diagnostics. */
1383 cpp_token_as_text (pfile
, token
)
1385 const cpp_token
*token
;
1387 unsigned int len
= cpp_token_len (token
);
1388 unsigned char *start
= _cpp_unaligned_alloc (pfile
, len
), *end
;
1390 end
= cpp_spell_token (pfile
, token
, start
);
1396 /* Used by C front ends. Should really move to using cpp_token_as_text. */
1398 cpp_type2name (type
)
1399 enum cpp_ttype type
;
1401 return (const char *) token_spellings
[type
].name
;
1404 /* Writes the spelling of token to FP, without any preceding space.
1405 Separated from cpp_spell_token for efficiency - to avoid stdio
1406 double-buffering. */
1408 cpp_output_token (token
, fp
)
1409 const cpp_token
*token
;
1412 switch (TOKEN_SPELL (token
))
1414 case SPELL_OPERATOR
:
1416 const unsigned char *spelling
;
1419 if (token
->flags
& DIGRAPH
)
1421 = digraph_spellings
[(int) token
->type
- (int) CPP_FIRST_DIGRAPH
];
1422 else if (token
->flags
& NAMED_OP
)
1425 spelling
= TOKEN_NAME (token
);
1430 while ((c
= *++spelling
) != '\0');
1435 putc (token
->val
.c
, fp
);
1440 fwrite (NODE_NAME (token
->val
.node
), 1, NODE_LEN (token
->val
.node
), fp
);
1444 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
1449 int left
, right
, tag
;
1450 switch (token
->type
)
1452 case CPP_STRING
: left
= '"'; right
= '"'; tag
= '\0'; break;
1453 case CPP_WSTRING
: left
= '"'; right
= '"'; tag
= 'L'; break;
1454 case CPP_CHAR
: left
= '\''; right
= '\''; tag
= '\0'; break;
1455 case CPP_WCHAR
: left
= '\''; right
= '\''; tag
= 'L'; break;
1456 case CPP_HEADER_NAME
: left
= '<'; right
= '>'; tag
= '\0'; break;
1458 fprintf (stderr
, "impossible STRING token %s\n", TOKEN_NAME (token
));
1461 if (tag
) putc (tag
, fp
);
1463 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
1469 /* An error, most probably. */
1474 /* Compare two tokens. */
1476 _cpp_equiv_tokens (a
, b
)
1477 const cpp_token
*a
, *b
;
1479 if (a
->type
== b
->type
&& a
->flags
== b
->flags
)
1480 switch (TOKEN_SPELL (a
))
1482 default: /* Keep compiler happy. */
1483 case SPELL_OPERATOR
:
1486 return a
->val
.c
== b
->val
.c
; /* Character. */
1488 return (a
->type
!= CPP_MACRO_ARG
|| a
->val
.arg_no
== b
->val
.arg_no
);
1490 return a
->val
.node
== b
->val
.node
;
1493 return (a
->val
.str
.len
== b
->val
.str
.len
1494 && !memcmp (a
->val
.str
.text
, b
->val
.str
.text
,
1501 /* Returns nonzero if a space should be inserted to avoid an
1502 accidental token paste for output. For simplicity, it is
1503 conservative, and occasionally advises a space where one is not
1504 needed, e.g. "." and ".2". */
1507 cpp_avoid_paste (pfile
, token1
, token2
)
1509 const cpp_token
*token1
, *token2
;
1511 enum cpp_ttype a
= token1
->type
, b
= token2
->type
;
1514 if (token1
->flags
& NAMED_OP
)
1516 if (token2
->flags
& NAMED_OP
)
1520 if (token2
->flags
& DIGRAPH
)
1521 c
= digraph_spellings
[(int) b
- (int) CPP_FIRST_DIGRAPH
][0];
1522 else if (token_spellings
[b
].category
== SPELL_OPERATOR
)
1523 c
= token_spellings
[b
].name
[0];
1525 /* Quickly get everything that can paste with an '='. */
1526 if ((int) a
<= (int) CPP_LAST_EQ
&& c
== '=')
1531 case CPP_GREATER
: return c
== '>' || c
== '?';
1532 case CPP_LESS
: return c
== '<' || c
== '?' || c
== '%' || c
== ':';
1533 case CPP_PLUS
: return c
== '+';
1534 case CPP_MINUS
: return c
== '-' || c
== '>';
1535 case CPP_DIV
: return c
== '/' || c
== '*'; /* Comments. */
1536 case CPP_MOD
: return c
== ':' || c
== '>';
1537 case CPP_AND
: return c
== '&';
1538 case CPP_OR
: return c
== '|';
1539 case CPP_COLON
: return c
== ':' || c
== '>';
1540 case CPP_DEREF
: return c
== '*';
1541 case CPP_DOT
: return c
== '.' || c
== '%' || b
== CPP_NUMBER
;
1542 case CPP_HASH
: return c
== '#' || c
== '%'; /* Digraph form. */
1543 case CPP_NAME
: return ((b
== CPP_NUMBER
1544 && name_p (pfile
, &token2
->val
.str
))
1546 || b
== CPP_CHAR
|| b
== CPP_STRING
); /* L */
1547 case CPP_NUMBER
: return (b
== CPP_NUMBER
|| b
== CPP_NAME
1548 || c
== '.' || c
== '+' || c
== '-');
1549 case CPP_OTHER
: return (CPP_OPTION (pfile
, objc
)
1550 && token1
->val
.c
== '@'
1551 && (b
== CPP_NAME
|| b
== CPP_STRING
));
1558 /* Output all the remaining tokens on the current line, and a newline
1559 character, to FP. Leading whitespace is removed. If there are
1560 macros, special token padding is not performed. */
1562 cpp_output_line (pfile
, fp
)
1566 const cpp_token
*token
;
1568 token
= cpp_get_token (pfile
);
1569 while (token
->type
!= CPP_EOF
)
1571 cpp_output_token (token
, fp
);
1572 token
= cpp_get_token (pfile
);
1573 if (token
->flags
& PREV_WHITE
)
1580 /* Returns the value of a hexadecimal digit. */
1586 return hex_value (c
);
1591 /* Parse a '\uNNNN' or '\UNNNNNNNN' sequence. Returns 1 to indicate
1592 failure if cpplib is not parsing C++ or C99. Such failure is
1593 silent, and no variables are updated. Otherwise returns 0, and
1594 warns if -Wtraditional.
1596 [lex.charset]: The character designated by the universal character
1597 name \UNNNNNNNN is that character whose character short name in
1598 ISO/IEC 10646 is NNNNNNNN; the character designated by the
1599 universal character name \uNNNN is that character whose character
1600 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
1601 for a universal character name is less than 0x20 or in the range
1602 0x7F-0x9F (inclusive), or if the universal character name
1603 designates a character in the basic source character set, then the
1604 program is ill-formed.
1606 We assume that wchar_t is Unicode, so we don't need to do any
1607 mapping. Is this ever wrong?
1609 PC points to the 'u' or 'U', PSTR is points to the byte after PC,
1610 LIMIT is the end of the string or charconst. PSTR is updated to
1611 point after the UCS on return, and the UCS is written into PC. */
1614 maybe_read_ucs (pfile
, pstr
, limit
, pc
)
1616 const unsigned char **pstr
;
1617 const unsigned char *limit
;
1620 const unsigned char *p
= *pstr
;
1621 unsigned int code
= 0;
1622 unsigned int c
= *pc
, length
;
1624 /* Only attempt to interpret a UCS for C++ and C99. */
1625 if (! (CPP_OPTION (pfile
, cplusplus
) || CPP_OPTION (pfile
, c99
)))
1628 if (CPP_WTRADITIONAL (pfile
))
1629 cpp_warning (pfile
, "the meaning of '\\%c' varies with -traditional", c
);
1631 length
= (c
== 'u' ? 4: 8);
1633 if ((size_t) (limit
- p
) < length
)
1635 cpp_error (pfile
, "incomplete universal-character-name");
1636 /* Skip to the end to avoid more diagnostics. */
1641 for (; length
; length
--, p
++)
1645 code
= (code
<< 4) + hex_digit_value (c
);
1649 "non-hex digit '%c' in universal-character-name", c
);
1650 /* We shouldn't skip in case there are multibyte chars. */
1656 #ifdef TARGET_EBCDIC
1657 cpp_error (pfile
, "universal-character-name on EBCDIC target");
1658 code
= 0x3f; /* EBCDIC invalid character */
1660 /* True extended characters are OK. */
1662 && !(code
& 0x80000000)
1663 && !(code
>= 0xD800 && code
<= 0xDFFF))
1665 /* The standard permits $, @ and ` to be specified as UCNs. We use
1666 hex escapes so that this also works with EBCDIC hosts. */
1667 else if (code
== 0x24 || code
== 0x40 || code
== 0x60)
1669 /* Don't give another error if one occurred above. */
1670 else if (length
== 0)
1671 cpp_error (pfile
, "universal-character-name out of range");
1679 /* Interpret an escape sequence, and return its value. PSTR points to
1680 the input pointer, which is just after the backslash. LIMIT is how
1681 much text we have. MASK is a bitmask for the precision for the
1682 destination type (char or wchar_t). TRADITIONAL, if true, does not
1683 interpret escapes that did not exist in traditional C.
1685 Handles all relevant diagnostics. */
1688 cpp_parse_escape (pfile
, pstr
, limit
, mask
, traditional
)
1690 const unsigned char **pstr
;
1691 const unsigned char *limit
;
1692 unsigned HOST_WIDE_INT mask
;
1696 const unsigned char *str
= *pstr
;
1697 unsigned int c
= *str
++;
1701 case '\\': case '\'': case '"': case '?': break;
1702 case 'b': c
= TARGET_BS
; break;
1703 case 'f': c
= TARGET_FF
; break;
1704 case 'n': c
= TARGET_NEWLINE
; break;
1705 case 'r': c
= TARGET_CR
; break;
1706 case 't': c
= TARGET_TAB
; break;
1707 case 'v': c
= TARGET_VT
; break;
1709 case '(': case '{': case '[': case '%':
1710 /* '\(', etc, are used at beginning of line to avoid confusing Emacs.
1711 '\%' is used to prevent SCCS from getting confused. */
1712 unknown
= CPP_PEDANTIC (pfile
);
1716 if (CPP_WTRADITIONAL (pfile
))
1717 cpp_warning (pfile
, "the meaning of '\\a' varies with -traditional");
1723 if (CPP_PEDANTIC (pfile
))
1724 cpp_pedwarn (pfile
, "non-ISO-standard escape sequence, '\\%c'", c
);
1729 unknown
= maybe_read_ucs (pfile
, &str
, limit
, &c
);
1733 if (CPP_WTRADITIONAL (pfile
))
1734 cpp_warning (pfile
, "the meaning of '\\x' varies with -traditional");
1738 unsigned int i
= 0, overflow
= 0;
1739 int digits_found
= 0;
1747 overflow
|= i
^ (i
<< 4 >> 4);
1748 i
= (i
<< 4) + hex_digit_value (c
);
1753 cpp_error (pfile
, "\\x used with no following hex digits");
1755 if (overflow
| (i
!= (i
& mask
)))
1757 cpp_pedwarn (pfile
, "hex escape sequence out of range");
1764 case '0': case '1': case '2': case '3':
1765 case '4': case '5': case '6': case '7':
1767 unsigned int i
= c
- '0';
1770 while (str
< limit
&& ++count
< 3)
1773 if (c
< '0' || c
> '7')
1776 i
= (i
<< 3) + c
- '0';
1779 if (i
!= (i
& mask
))
1781 cpp_pedwarn (pfile
, "octal escape sequence out of range");
1796 cpp_pedwarn (pfile
, "unknown escape sequence '\\%c'", c
);
1798 cpp_pedwarn (pfile
, "unknown escape sequence: '\\%03o'", c
);
1802 cpp_pedwarn (pfile
, "escape sequence out of range for character");
1808 #ifndef MAX_CHAR_TYPE_SIZE
1809 #define MAX_CHAR_TYPE_SIZE CHAR_TYPE_SIZE
1812 #ifndef MAX_WCHAR_TYPE_SIZE
1813 #define MAX_WCHAR_TYPE_SIZE WCHAR_TYPE_SIZE
1816 /* Interpret a (possibly wide) character constant in TOKEN.
1817 WARN_MULTI warns about multi-character charconsts, if not
1818 TRADITIONAL. TRADITIONAL also indicates not to interpret escapes
1819 that did not exist in traditional C. PCHARS_SEEN points to a
1820 variable that is filled in with the number of characters seen. */
1822 cpp_interpret_charconst (pfile
, token
, warn_multi
, traditional
, pchars_seen
)
1824 const cpp_token
*token
;
1827 unsigned int *pchars_seen
;
1829 const unsigned char *str
= token
->val
.str
.text
;
1830 const unsigned char *limit
= str
+ token
->val
.str
.len
;
1831 unsigned int chars_seen
= 0;
1832 unsigned int width
, max_chars
, c
;
1833 unsigned HOST_WIDE_INT mask
;
1834 HOST_WIDE_INT result
= 0;
1836 #ifdef MULTIBYTE_CHARS
1837 (void) local_mbtowc (NULL
, NULL
, 0);
1840 /* Width in bits. */
1841 if (token
->type
== CPP_CHAR
)
1842 width
= MAX_CHAR_TYPE_SIZE
;
1844 width
= MAX_WCHAR_TYPE_SIZE
;
1846 if (width
< HOST_BITS_PER_WIDE_INT
)
1847 mask
= ((unsigned HOST_WIDE_INT
) 1 << width
) - 1;
1850 max_chars
= HOST_BITS_PER_WIDE_INT
/ width
;
1854 #ifdef MULTIBYTE_CHARS
1858 char_len
= local_mbtowc (&wc
, str
, limit
- str
);
1861 cpp_warning (pfile
, "ignoring invalid multibyte character");
1874 c
= cpp_parse_escape (pfile
, &str
, limit
, mask
, traditional
);
1876 #ifdef MAP_CHARACTER
1878 c
= MAP_CHARACTER (c
);
1881 /* Merge character into result; ignore excess chars. */
1882 if (++chars_seen
<= max_chars
)
1884 if (width
< HOST_BITS_PER_WIDE_INT
)
1885 result
= (result
<< width
) | (c
& mask
);
1891 if (chars_seen
== 0)
1892 cpp_error (pfile
, "empty character constant");
1893 else if (chars_seen
> max_chars
)
1895 chars_seen
= max_chars
;
1896 cpp_warning (pfile
, "character constant too long");
1898 else if (chars_seen
> 1 && !traditional
&& warn_multi
)
1899 cpp_warning (pfile
, "multi-character character constant");
1901 /* If char type is signed, sign-extend the constant. The
1902 __CHAR_UNSIGNED__ macro is set by the driver if appropriate. */
1903 if (token
->type
== CPP_CHAR
&& chars_seen
)
1905 unsigned int nbits
= chars_seen
* width
;
1907 mask
= (unsigned HOST_WIDE_INT
) ~0 >> (HOST_BITS_PER_WIDE_INT
- nbits
);
1908 if (pfile
->spec_nodes
.n__CHAR_UNSIGNED__
->type
== NT_MACRO
1909 || ((result
>> (nbits
- 1)) & 1) == 0)
1915 *pchars_seen
= chars_seen
;
1919 /* Memory buffers. Changing these three constants can have a dramatic
1920 effect on performance. The values here are reasonable defaults,
1921 but might be tuned. If you adjust them, be sure to test across a
1922 range of uses of cpplib, including heavy nested function-like macro
1923 expansion. Also check the change in peak memory usage (NJAMD is a
1924 good tool for this). */
1925 #define MIN_BUFF_SIZE 8000
1926 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
1927 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
1928 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
1930 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
1931 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
1944 #define DEFAULT_ALIGNMENT (offsetof (struct dummy, u))
1945 #define CPP_ALIGN(size, align) (((size) + ((align) - 1)) & ~((align) - 1))
1947 /* Create a new allocation buffer. Place the control block at the end
1948 of the buffer, so that buffer overflows will cause immediate chaos. */
1954 unsigned char *base
;
1956 if (len
< MIN_BUFF_SIZE
)
1957 len
= MIN_BUFF_SIZE
;
1958 len
= CPP_ALIGN (len
, DEFAULT_ALIGNMENT
);
1960 base
= xmalloc (len
+ sizeof (_cpp_buff
));
1961 result
= (_cpp_buff
*) (base
+ len
);
1962 result
->base
= base
;
1964 result
->limit
= base
+ len
;
1965 result
->next
= NULL
;
1969 /* Place a chain of unwanted allocation buffers on the free list. */
1971 _cpp_release_buff (pfile
, buff
)
1975 _cpp_buff
*end
= buff
;
1979 end
->next
= pfile
->free_buffs
;
1980 pfile
->free_buffs
= buff
;
1983 /* Return a free buffer of size at least MIN_SIZE. */
1985 _cpp_get_buff (pfile
, min_size
)
1989 _cpp_buff
*result
, **p
;
1991 for (p
= &pfile
->free_buffs
;; p
= &(*p
)->next
)
1996 return new_buff (min_size
);
1998 size
= result
->limit
- result
->base
;
1999 /* Return a buffer that's big enough, but don't waste one that's
2001 if (size
>= min_size
&& size
<= BUFF_SIZE_UPPER_BOUND (min_size
))
2006 result
->next
= NULL
;
2007 result
->cur
= result
->base
;
2011 /* Creates a new buffer with enough space to hold the uncommitted
2012 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
2013 the excess bytes to the new buffer. Chains the new buffer after
2014 BUFF, and returns the new buffer. */
2016 _cpp_append_extend_buff (pfile
, buff
, min_extra
)
2021 size_t size
= EXTENDED_BUFF_SIZE (buff
, min_extra
);
2022 _cpp_buff
*new_buff
= _cpp_get_buff (pfile
, size
);
2024 buff
->next
= new_buff
;
2025 memcpy (new_buff
->base
, buff
->cur
, BUFF_ROOM (buff
));
2029 /* Creates a new buffer with enough space to hold the uncommitted
2030 remaining bytes of the buffer pointed to by BUFF, and at least
2031 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
2032 Chains the new buffer before the buffer pointed to by BUFF, and
2033 updates the pointer to point to the new buffer. */
2035 _cpp_extend_buff (pfile
, pbuff
, min_extra
)
2040 _cpp_buff
*new_buff
, *old_buff
= *pbuff
;
2041 size_t size
= EXTENDED_BUFF_SIZE (old_buff
, min_extra
);
2043 new_buff
= _cpp_get_buff (pfile
, size
);
2044 memcpy (new_buff
->base
, old_buff
->cur
, BUFF_ROOM (old_buff
));
2045 new_buff
->next
= old_buff
;
2049 /* Free a chain of buffers starting at BUFF. */
2051 _cpp_free_buff (buff
)
2056 for (; buff
; buff
= next
)
2063 /* Allocate permanent, unaligned storage of length LEN. */
2065 _cpp_unaligned_alloc (pfile
, len
)
2069 _cpp_buff
*buff
= pfile
->u_buff
;
2070 unsigned char *result
= buff
->cur
;
2072 if (len
> (size_t) (buff
->limit
- result
))
2074 buff
= _cpp_get_buff (pfile
, len
);
2075 buff
->next
= pfile
->u_buff
;
2076 pfile
->u_buff
= buff
;
2080 buff
->cur
= result
+ len
;
2084 /* Allocate permanent, unaligned storage of length LEN from a_buff.
2085 That buffer is used for growing allocations when saving macro
2086 replacement lists in a #define, and when parsing an answer to an
2087 assertion in #assert, #unassert or #if (and therefore possibly
2088 whilst expanding macros). It therefore must not be used by any
2089 code that they might call: specifically the lexer and the guts of
2092 All existing other uses clearly fit this restriction: storing
2093 registered pragmas during initialization. */
2095 _cpp_aligned_alloc (pfile
, len
)
2099 _cpp_buff
*buff
= pfile
->a_buff
;
2100 unsigned char *result
= buff
->cur
;
2102 if (len
> (size_t) (buff
->limit
- result
))
2104 buff
= _cpp_get_buff (pfile
, len
);
2105 buff
->next
= pfile
->a_buff
;
2106 pfile
->a_buff
= buff
;
2110 buff
->cur
= result
+ len
;