1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
7 Single-pass line tokenization by Neil Booth, April 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 2, or (at your option) any
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
30 # include <sys/mman.h>
33 #define PEEKBUF(BUFFER, N) \
34 ((BUFFER)->rlimit - (BUFFER)->cur > (N) ? (BUFFER)->cur[N] : EOF)
35 #define GETBUF(BUFFER) \
36 ((BUFFER)->cur < (BUFFER)->rlimit ? *(BUFFER)->cur++ : EOF)
37 #define FORWARDBUF(BUFFER, N) ((BUFFER)->cur += (N))
39 #define PEEKN(N) PEEKBUF (CPP_BUFFER (pfile), N)
40 #define FORWARD(N) FORWARDBUF (CPP_BUFFER (pfile), (N))
41 #define GETC() GETBUF (CPP_BUFFER (pfile))
42 #define PEEKC() PEEKBUF (CPP_BUFFER (pfile), 0)
44 static void skip_block_comment
PARAMS ((cpp_reader
*));
45 static void skip_line_comment
PARAMS ((cpp_reader
*));
46 static int maybe_macroexpand
PARAMS ((cpp_reader
*, long));
47 static int skip_comment
PARAMS ((cpp_reader
*, int));
48 static int copy_comment
PARAMS ((cpp_reader
*, int));
49 static void skip_string
PARAMS ((cpp_reader
*, int));
50 static void parse_string
PARAMS ((cpp_reader
*, int));
51 static U_CHAR
*find_position
PARAMS ((U_CHAR
*, U_CHAR
*, unsigned long *));
52 static void null_warning
PARAMS ((cpp_reader
*, unsigned int));
54 static void safe_fwrite
PARAMS ((cpp_reader
*, const U_CHAR
*,
56 static void output_line_command
PARAMS ((cpp_reader
*, cpp_printer
*,
58 static void bump_column
PARAMS ((cpp_printer
*, unsigned int,
60 static void expand_name_space
PARAMS ((cpp_toklist
*, unsigned int));
61 static void pedantic_whitespace
PARAMS ((cpp_reader
*, U_CHAR
*,
64 #define auto_expand_name_space(list) \
65 expand_name_space ((list), 1 + (list)->name_cap / 2)
69 void init_trigraph_map
PARAMS ((void));
70 static unsigned char* trigraph_replace
PARAMS ((cpp_reader
*, unsigned char *,
72 static const unsigned char *backslash_start
PARAMS ((cpp_reader
*,
73 const unsigned char *));
74 static int skip_block_comment2
PARAMS ((cpp_reader
*));
75 static int skip_line_comment2
PARAMS ((cpp_reader
*));
76 static void skip_whitespace
PARAMS ((cpp_reader
*, int));
77 static void parse_name
PARAMS ((cpp_reader
*, cpp_toklist
*, cpp_name
*));
78 static void parse_number
PARAMS ((cpp_reader
*, cpp_toklist
*, cpp_name
*));
79 static void parse_string2
PARAMS ((cpp_reader
*, cpp_toklist
*, cpp_name
*,
81 static int trigraph_ok
PARAMS ((cpp_reader
*, const unsigned char *));
82 static void save_comment
PARAMS ((cpp_toklist
*, cpp_token
*,
83 const unsigned char *,
84 unsigned int, unsigned int));
85 void _cpp_lex_line
PARAMS ((cpp_reader
*, cpp_toklist
*));
87 static void _cpp_output_list
PARAMS ((cpp_reader
*, cpp_toklist
*));
89 static unsigned char * spell_token
PARAMS ((cpp_reader
*, const cpp_token
*,
90 unsigned char *, int));
92 typedef unsigned int (* speller
) PARAMS ((unsigned char *, cpp_toklist
*,
95 /* Macros on a cpp_name. */
96 #define INIT_TOKEN_NAME(list, token) \
97 do {(token)->val.name.len = 0; \
98 (token)->val.name.text = (list)->namebuf + (list)->name_used; \
99 (list)->tokens_used = token - (list)->tokens + 1; \
102 /* Maybe put these in the ISTABLE eventually. */
103 #define IS_HSPACE(c) ((c) == ' ' || (c) == '\t')
104 #define IS_NEWLINE(c) ((c) == '\n' || (c) == '\r')
106 /* Handle LF, CR, CR-LF and LF-CR style newlines. Assumes next
107 character, if any, is in buffer. */
108 #define handle_newline(cur, limit, c) \
110 if ((cur) < (limit) && *(cur) == '\r' + '\n' - c) \
112 CPP_BUMP_LINE_CUR (pfile, (cur)); \
113 pfile->col_adjust = 0; \
116 #define IMMED_TOKEN() (!(cur_token->flags & PREV_WHITESPACE))
117 #define PREV_TOKEN_TYPE (cur_token[-1].type)
119 #define PUSH_TOKEN(ttype) cur_token++->type = ttype
120 #define REVISE_TOKEN(ttype) cur_token[-1].type = ttype
121 #define BACKUP_TOKEN(ttype) (--cur_token)->type = ttype
122 #define BACKUP_DIGRAPH(ttype) do { \
123 BACKUP_TOKEN(ttype); cur_token->flags |= DIGRAPH;} while (0)
125 /* An upper bound on the number of bytes needed to spell a token,
126 including preceding whitespace. */
127 #define TOKEN_LEN(token) (5 + (token_spellings[(token)->type].type > \
128 SPELL_NONE ? (token)->val.name.len: 0))
132 /* Order here matters. Those beyond SPELL_NONE store their spelling
133 in the token list, and it's length in the token->val.name.len. */
138 SPELL_CHAR
, /* FIXME: revert order of NONE and CHAR after transition. */
143 #define T(e, s) {SPELL_OPERATOR, (const U_CHAR *) s},
144 #define I(e, s) {SPELL_IDENT, s},
145 #define S(e, s) {SPELL_STRING, s},
146 #define C(e, s) {SPELL_CHAR, s},
147 #define N(e, s) {SPELL_NONE, s},
149 static const struct token_spelling
151 ENUM_BITFIELD(spell_type
) type
: CHAR_BIT
;
152 const U_CHAR
*spelling
;
153 } token_spellings
[N_TTYPES
+ 1] = {TTYPE_TABLE
{0, 0} };
161 /* Re-allocates PFILE->token_buffer so it will hold at least N more chars. */
164 _cpp_grow_token_buffer (pfile
, n
)
168 long old_written
= CPP_WRITTEN (pfile
);
169 pfile
->token_buffer_size
= n
+ 2 * pfile
->token_buffer_size
;
170 pfile
->token_buffer
= (U_CHAR
*)
171 xrealloc(pfile
->token_buffer
, pfile
->token_buffer_size
);
172 CPP_SET_WRITTEN (pfile
, old_written
);
175 /* Allocate a new cpp_buffer for PFILE, and push it on the input buffer stack.
176 If BUFFER != NULL, then use the LENGTH characters in BUFFER
177 as the new input buffer.
178 Return the new buffer, or NULL on failure. */
181 cpp_push_buffer (pfile
, buffer
, length
)
183 const U_CHAR
*buffer
;
186 cpp_buffer
*buf
= CPP_BUFFER (pfile
);
188 if (++pfile
->buffer_stack_depth
== CPP_STACK_MAX
)
190 cpp_fatal (pfile
, "macro or `#include' recursion too deep");
194 new = (cpp_buffer
*) xcalloc (1, sizeof (cpp_buffer
));
196 new->buf
= new->cur
= buffer
;
197 new->rlimit
= buffer
+ length
;
200 new->line_base
= NULL
;
202 CPP_BUFFER (pfile
) = new;
207 cpp_pop_buffer (pfile
)
210 cpp_buffer
*buf
= CPP_BUFFER (pfile
);
211 if (ACTIVE_MARK_P (pfile
))
212 cpp_ice (pfile
, "mark active in cpp_pop_buffer");
216 _cpp_unwind_if_stack (pfile
, buf
);
218 free ((PTR
) buf
->buf
);
219 if (pfile
->system_include_depth
)
220 pfile
->system_include_depth
--;
221 if (pfile
->potential_control_macro
)
223 buf
->ihash
->cmacro
= pfile
->potential_control_macro
;
224 pfile
->potential_control_macro
= 0;
226 pfile
->input_stack_listing_current
= 0;
230 cpp_hashnode
*m
= buf
->macro
;
233 if ((m
->type
== T_FMACRO
&& buf
->mapped
)
234 || m
->type
== T_SPECLINE
|| m
->type
== T_FILE
235 || m
->type
== T_BASE_FILE
|| m
->type
== T_INCLUDE_LEVEL
236 || m
->type
== T_STDC
)
237 free ((PTR
) buf
->buf
);
239 CPP_BUFFER (pfile
) = CPP_PREV_BUFFER (buf
);
241 pfile
->buffer_stack_depth
--;
242 return CPP_BUFFER (pfile
);
245 /* Deal with the annoying semantics of fwrite. */
247 safe_fwrite (pfile
, buf
, len
, fp
)
257 count
= fwrite (buf
, 1, len
, fp
);
266 cpp_notice_from_errno (pfile
, CPP_OPTION (pfile
, out_fname
));
269 /* Notify the compiler proper that the current line number has jumped,
270 or the current file name has changed. */
273 output_line_command (pfile
, print
, line
)
278 cpp_buffer
*ip
= cpp_file_buffer (pfile
);
279 enum { same
= 0, enter
, leave
, rname
} change
;
280 static const char * const codes
[] = { "", " 1", " 2", "" };
282 if (CPP_OPTION (pfile
, no_line_commands
))
285 /* Determine whether the current filename has changed, and if so,
286 how. 'nominal_fname' values are unique, so they can be compared
287 by comparing pointers. */
288 if (ip
->nominal_fname
== print
->last_fname
)
292 if (pfile
->buffer_stack_depth
== print
->last_bsd
)
296 if (pfile
->buffer_stack_depth
> print
->last_bsd
)
300 print
->last_bsd
= pfile
->buffer_stack_depth
;
302 print
->last_fname
= ip
->nominal_fname
;
304 /* If the current file has not changed, we can output a few newlines
305 instead if we want to increase the line number by a small amount.
306 We cannot do this if print->lineno is zero, because that means we
307 haven't output any line commands yet. (The very first line
308 command output is a `same_file' command.) */
309 if (change
== same
&& print
->lineno
!= 0
310 && line
>= print
->lineno
&& line
< print
->lineno
+ 8)
312 while (line
> print
->lineno
)
314 putc ('\n', print
->outf
);
320 #ifndef NO_IMPLICIT_EXTERN_C
321 if (CPP_OPTION (pfile
, cplusplus
))
322 fprintf (print
->outf
, "# %u \"%s\"%s%s%s\n", line
, ip
->nominal_fname
,
324 ip
->system_header_p
? " 3" : "",
325 (ip
->system_header_p
== 2) ? " 4" : "");
328 fprintf (print
->outf
, "# %u \"%s\"%s%s\n", line
, ip
->nominal_fname
,
330 ip
->system_header_p
? " 3" : "");
331 print
->lineno
= line
;
334 /* Write the contents of the token_buffer to the output stream, and
335 clear the token_buffer. Also handles generating line commands and
336 keeping track of file transitions. */
339 cpp_output_tokens (pfile
, print
)
345 if (CPP_WRITTEN (pfile
) - print
->written
)
347 if (CPP_PWRITTEN (pfile
)[-1] == '\n' && print
->lineno
)
349 safe_fwrite (pfile
, pfile
->token_buffer
,
350 CPP_WRITTEN (pfile
) - print
->written
, print
->outf
);
353 ip
= cpp_file_buffer (pfile
);
355 output_line_command (pfile
, print
, CPP_BUF_LINE (ip
));
357 CPP_SET_WRITTEN (pfile
, print
->written
);
360 /* Helper for cpp_output_list - increases the column number to match
361 what we expect it to be. */
364 bump_column (print
, from
, to
)
366 unsigned int from
, to
;
368 unsigned int tabs
, spcs
;
369 unsigned int delta
= to
- from
;
371 /* Only if FROM is 0, advance by tabs. */
373 tabs
= delta
/ 8, spcs
= delta
% 8;
375 tabs
= 0, spcs
= delta
;
377 while (tabs
--) putc ('\t', print
->outf
);
378 while (spcs
--) putc (' ', print
->outf
);
381 /* Write out the list L onto pfile->token_buffer. This function is
384 1) pfile->token_buffer is not going to continue to exist.
385 2) At the moment, tokens don't carry the information described
386 in cpplib.h; they are all strings.
387 3) The list has to be a complete line, and has to be written starting
388 at the beginning of a line. */
391 cpp_output_list (pfile
, print
, list
)
394 const cpp_toklist
*list
;
397 unsigned int curcol
= 1;
399 /* XXX Probably does not do what is intended. */
400 if (print
->lineno
!= list
->line
)
401 output_line_command (pfile
, print
, list
->line
);
403 for (i
= 0; i
< list
->tokens_used
; i
++)
405 if (TOK_TYPE (list
, i
) == CPP_VSPACE
)
407 output_line_command (pfile
, print
, list
->tokens
[i
].aux
);
411 if (curcol
< TOK_COL (list
, i
))
413 /* Insert space to bring the column to what it should be. */
414 bump_column (print
, curcol
- 1, TOK_COL (list
, i
));
415 curcol
= TOK_COL (list
, i
);
417 /* XXX We may have to insert space to prevent an accidental
419 safe_fwrite (pfile
, TOK_NAME (list
, i
), TOK_LEN (list
, i
), print
->outf
);
420 curcol
+= TOK_LEN (list
, i
);
424 /* Scan a string (which may have escape marks), perform macro expansion,
425 and write the result to the token_buffer. */
428 _cpp_expand_to_buffer (pfile
, buf
, length
)
434 enum cpp_ttype token
;
439 cpp_ice (pfile
, "length < 0 in cpp_expand_to_buffer");
443 /* Copy the buffer, because it might be in an unsafe place - for
444 example, a sequence on the token_buffer, where the pointers will
445 be invalidated if we enlarge the token_buffer. */
446 buf1
= alloca (length
);
447 memcpy (buf1
, buf
, length
);
449 /* Set up the input on the input stack. */
450 stop
= CPP_BUFFER (pfile
);
451 if (cpp_push_buffer (pfile
, buf1
, length
) == NULL
)
453 CPP_BUFFER (pfile
)->has_escapes
= 1;
455 /* Scan the input, create the output. */
458 token
= cpp_get_token (pfile
);
459 if (token
== CPP_EOF
&& CPP_BUFFER (pfile
) == stop
)
464 /* Scan until CPP_BUFFER (PFILE) is exhausted, discarding output. */
467 cpp_scan_buffer_nooutput (pfile
)
470 cpp_buffer
*stop
= CPP_PREV_BUFFER (CPP_BUFFER (pfile
));
471 enum cpp_ttype token
;
472 unsigned int old_written
= CPP_WRITTEN (pfile
);
473 /* In no-output mode, we can ignore everything but directives. */
476 if (! pfile
->only_seen_white
)
477 _cpp_skip_rest_of_line (pfile
);
478 token
= cpp_get_token (pfile
);
479 if (token
== CPP_EOF
&& CPP_BUFFER (pfile
) == stop
)
482 CPP_SET_WRITTEN (pfile
, old_written
);
485 /* Scan until CPP_BUFFER (pfile) is exhausted, writing output to PRINT. */
488 cpp_scan_buffer (pfile
, print
)
492 cpp_buffer
*stop
= CPP_PREV_BUFFER (CPP_BUFFER (pfile
));
493 enum cpp_ttype token
;
497 token
= cpp_get_token (pfile
);
498 if (token
== CPP_VSPACE
|| token
== CPP_EOF
499 /* XXX Temporary kluge - force flush after #include only */
500 || (token
== CPP_DIRECTIVE
501 && CPP_BUFFER (pfile
)->nominal_fname
!= print
->last_fname
))
503 cpp_output_tokens (pfile
, print
);
504 if (token
== CPP_EOF
&& CPP_BUFFER (pfile
) == stop
)
510 /* Return the topmost cpp_buffer that corresponds to a file (not a macro). */
513 cpp_file_buffer (pfile
)
518 for (ip
= CPP_BUFFER (pfile
); ip
; ip
= CPP_PREV_BUFFER (ip
))
519 if (ip
->ihash
!= NULL
)
524 /* Token-buffer helper functions. */
526 /* Expand a token list's string space. It is *vital* that
527 list->tokens_used is correct, to get pointer fix-up right. */
529 expand_name_space (list
, len
)
533 const U_CHAR
*old_namebuf
;
535 old_namebuf
= list
->namebuf
;
536 list
->name_cap
+= len
;
537 list
->namebuf
= (unsigned char *) xrealloc (list
->namebuf
, list
->name_cap
);
539 /* Fix up token text pointers. */
540 if (list
->namebuf
!= old_namebuf
)
544 for (i
= 0; i
< list
->tokens_used
; i
++)
545 if (token_spellings
[list
->tokens
[i
].type
].type
> SPELL_NONE
)
546 list
->tokens
[i
].val
.name
.text
+= (list
->namebuf
- old_namebuf
);
550 /* Expand the number of tokens in a list. */
552 _cpp_expand_token_space (list
, count
)
558 list
->tokens_cap
+= count
;
559 n
= list
->tokens_cap
;
560 if (list
->flags
& LIST_OFFSET
)
562 list
->tokens
= (cpp_token
*)
563 xrealloc (list
->tokens
, n
* sizeof (cpp_token
));
564 if (list
->flags
& LIST_OFFSET
)
565 list
->tokens
++; /* Skip the dummy. */
568 /* Initialize a token list. If flags is DUMMY_TOKEN, we allocate
569 an extra token in front of the token list, as this allows the lexer
570 to always peek at the previous token without worrying about
571 underflowing the list, and some initial space. Otherwise, no
572 token- or name-space is allocated, and there is no dummy token. */
574 _cpp_init_toklist (list
, flags
)
578 /* We malloc zero bytes because we may want to realloc later, and
579 some old implementations don't like realloc-ing a null pointer. */
580 if (flags
== NO_DUMMY_TOKEN
)
582 list
->tokens_cap
= 0;
583 list
->tokens
= (cpp_token
*) malloc (0);
589 /* Initialize token space. Put a dummy token before the start
590 that will fail matches. */
591 list
->tokens_cap
= 256; /* 4K's worth. */
592 list
->tokens
= (cpp_token
*)
593 xmalloc ((list
->tokens_cap
+ 1) * sizeof (cpp_token
));
594 list
->tokens
[0].type
= CPP_EOF
;
597 /* Initialize name space. */
598 list
->name_cap
= 1024;
599 list
->flags
= LIST_OFFSET
;
602 /* Allocate name space. */
603 list
->namebuf
= (unsigned char *) xmalloc (list
->name_cap
);
605 _cpp_clear_toklist (list
);
608 /* Clear a token list. */
610 _cpp_clear_toklist (list
)
613 list
->tokens_used
= 0;
616 list
->flags
&= LIST_OFFSET
; /* clear all but that one */
619 /* Free a token list. Does not free the list itself, which may be
620 embedded in a larger structure. */
622 _cpp_free_toklist (list
)
625 if (list
->flags
& LIST_OFFSET
)
626 free (list
->tokens
- 1); /* Backup over dummy token. */
629 free (list
->namebuf
);
632 /* Slice a token list: copy the sublist [START, FINISH) into COPY.
633 COPY is assumed not to be initialized. The comment space is not
636 _cpp_slice_toklist (copy
, start
, finish
)
638 const cpp_token
*start
, *finish
;
644 copy
->tokens_cap
= n
;
645 copy
->tokens
= (cpp_token
*) xmalloc (n
* sizeof (cpp_token
));
646 memcpy (copy
->tokens
, start
, n
* sizeof (cpp_token
));
649 for (i
= 0; i
< n
; i
++)
650 if (token_spellings
[start
[i
].type
].type
> SPELL_NONE
)
651 bytes
+= start
[i
].val
.name
.len
;
653 copy
->namebuf
= xmalloc (bytes
);
655 for (i
= 0; i
< n
; i
++)
656 if (token_spellings
[start
[i
].type
].type
> SPELL_NONE
)
658 memcpy (copy
->namebuf
+ bytes
,
659 start
[i
].val
.name
.text
, start
[i
].val
.name
.len
);
660 copy
->tokens
[i
].val
.name
.text
= copy
->namebuf
+ bytes
;
661 bytes
+= start
[i
].val
.name
.len
;
664 copy
->tokens_cap
= n
;
665 copy
->tokens_used
= n
;
666 copy
->name_used
= bytes
;
667 copy
->name_cap
= bytes
;
673 /* Shrink a token list down to the minimum size. */
675 _cpp_squeeze_toklist (list
)
679 const U_CHAR
*old_namebuf
;
681 if (list
->flags
& LIST_OFFSET
)
684 memmove (list
->tokens
, list
->tokens
+ 1,
685 list
->tokens_used
* sizeof (cpp_token
));
686 list
->tokens
= xrealloc (list
->tokens
,
687 list
->tokens_used
* sizeof (cpp_token
));
688 list
->flags
&= ~LIST_OFFSET
;
691 list
->tokens
= xrealloc (list
->tokens
,
692 list
->tokens_used
* sizeof (cpp_token
));
693 list
->tokens_cap
= list
->tokens_used
;
695 old_namebuf
= list
->namebuf
;
696 list
->namebuf
= xrealloc (list
->namebuf
, list
->name_used
);
697 list
->name_cap
= list
->name_used
;
699 /* Fix up token text pointers. */
700 delta
= list
->namebuf
- old_namebuf
;
705 for (i
= 0; i
< list
->tokens_used
; i
++)
706 if (token_spellings
[list
->tokens
[i
].type
].type
> SPELL_NONE
)
707 list
->tokens
[i
].val
.name
.text
+= delta
;
711 /* Compare two tokens. */
713 _cpp_equiv_tokens (a
, b
)
714 const cpp_token
*a
, *b
;
716 if (a
->type
!= b
->type
717 || a
->flags
!= b
->flags
721 if (token_spellings
[a
->type
].type
> SPELL_NONE
)
723 if (a
->val
.name
.len
!= b
->val
.name
.len
724 || ustrncmp(a
->val
.name
.text
,
732 /* Compare two token lists. */
734 _cpp_equiv_toklists (a
, b
)
735 const cpp_toklist
*a
, *b
;
739 if (a
->tokens_used
!= b
->tokens_used
)
742 for (i
= 0; i
< a
->tokens_used
; i
++)
743 if (! _cpp_equiv_tokens (&a
->tokens
[i
], &b
->tokens
[i
]))
748 /* Scan until we encounter a token of type STOP or a newline, and
749 create a token list for it. Does not macro-expand or execute
750 directives. The final token is not included in the list or
751 consumed from the input. Returns the type of the token stopped at. */
754 _cpp_scan_until (pfile
, list
, stop
)
764 _cpp_clear_toklist (list
);
765 list
->line
= CPP_BUF_LINE (CPP_BUFFER (pfile
));
767 written
= CPP_WRITTEN (pfile
);
772 col
= CPP_BUFFER (pfile
)->cur
- CPP_BUFFER (pfile
)->line_base
;
773 type
= _cpp_lex_token (pfile
);
774 len
= CPP_WRITTEN (pfile
) - written
;
775 CPP_SET_WRITTEN (pfile
, written
);
776 if (type
== CPP_HSPACE
)
778 if (CPP_PEDANTIC (pfile
))
779 pedantic_whitespace (pfile
, pfile
->token_buffer
+ written
, len
);
783 else if (type
== CPP_COMMENT
)
784 /* Only happens when processing -traditional macro definitions.
785 Do not give this a token entry, but do not change space_before
789 if (list
->tokens_used
>= list
->tokens_cap
)
790 _cpp_expand_token_space (list
, 256);
791 if (list
->name_used
+ len
>= list
->name_cap
)
792 expand_name_space (list
, list
->name_used
+ len
+ 1 - list
->name_cap
);
794 if (type
== CPP_MACRO
)
797 if (type
== CPP_VSPACE
|| type
== stop
)
801 TOK_TYPE (list
, i
) = type
;
802 TOK_COL (list
, i
) = col
;
803 TOK_AUX (list
, i
) = 0;
804 TOK_FLAGS (list
, i
) = space_before
? PREV_WHITESPACE
: 0;
806 TOK_LEN (list
, i
) = len
;
807 if (token_spellings
[type
].type
> SPELL_NONE
)
809 memcpy (list
->namebuf
+ list
->name_used
, CPP_PWRITTEN (pfile
), len
);
810 TOK_NAME (list
, i
) = list
->namebuf
+ list
->name_used
;
811 list
->name_used
+= len
;
814 TOK_NAME (list
, i
) = token_spellings
[type
].spelling
;
819 /* XXX Temporary kluge: put back the newline (or whatever). */
822 /* Don't consider the first token to have white before. */
823 TOK_FLAGS (list
, 0) &= ~PREV_WHITESPACE
;
827 /* Skip a C-style block comment. We know it's a comment, and point is
828 at the second character of the starter. */
830 skip_block_comment (pfile
)
833 unsigned int line
, col
;
834 const U_CHAR
*limit
, *cur
;
837 line
= CPP_BUF_LINE (CPP_BUFFER (pfile
));
838 col
= CPP_BUF_COL (CPP_BUFFER (pfile
));
839 limit
= CPP_BUFFER (pfile
)->rlimit
;
840 cur
= CPP_BUFFER (pfile
)->cur
;
845 if (c
== '\n' || c
== '\r')
847 /* \r cannot be a macro escape marker here. */
848 if (!ACTIVE_MARK_P (pfile
))
849 CPP_BUMP_LINE_CUR (pfile
, cur
);
853 /* Check for teminator. */
854 if (cur
< limit
&& *cur
== '/')
857 /* Warn about comment starter embedded in comment. */
858 if (cur
[-2] == '/' && CPP_OPTION (pfile
, warn_comments
))
859 cpp_warning_with_line (pfile
, CPP_BUFFER (pfile
)->lineno
,
860 cur
- CPP_BUFFER (pfile
)->line_base
,
861 "'/*' within comment");
865 cpp_error_with_line (pfile
, line
, col
, "unterminated comment");
868 CPP_BUFFER (pfile
)->cur
= cur
+ 1;
871 /* Skip a C++/Chill line comment. We know it's a comment, and point
872 is at the second character of the initiator. */
874 skip_line_comment (pfile
)
882 /* We don't have to worry about EOF in here. */
885 /* Don't consider final '\n' to be part of comment. */
891 /* \r cannot be a macro escape marker here. */
892 if (!ACTIVE_MARK_P (pfile
))
893 CPP_BUMP_LINE (pfile
);
894 if (CPP_OPTION (pfile
, warn_comments
))
895 cpp_warning (pfile
, "backslash-newline within line comment");
900 /* Skip a comment - C, C++, or Chill style. M is the first character
901 of the comment marker. If this really is a comment, skip to its
902 end and return ' '. If this is not a comment, return M (which will
906 skip_comment (pfile
, m
)
910 if (m
== '/' && PEEKC() == '*')
912 skip_block_comment (pfile
);
915 else if (m
== '/' && PEEKC() == '/')
917 if (CPP_BUFFER (pfile
)->system_header_p
)
919 /* We silently allow C++ comments in system headers, irrespective
920 of conformance mode, because lots of busted systems do that
921 and trying to clean it up in fixincludes is a nightmare. */
922 skip_line_comment (pfile
);
925 else if (CPP_OPTION (pfile
, cplusplus_comments
))
927 if (! CPP_BUFFER (pfile
)->warned_cplusplus_comments
)
929 if (CPP_WTRADITIONAL (pfile
))
931 "C++ style comments are not allowed in traditional C");
932 else if (CPP_OPTION (pfile
, c89
) && CPP_PEDANTIC (pfile
))
934 "C++ style comments are not allowed in ISO C89");
935 if (CPP_WTRADITIONAL (pfile
)
936 || (CPP_OPTION (pfile
, c89
) && CPP_PEDANTIC (pfile
)))
938 "(this will be reported only once per input file)");
939 CPP_BUFFER (pfile
)->warned_cplusplus_comments
= 1;
941 skip_line_comment (pfile
);
947 else if (m
== '-' && PEEKC() == '-'
948 && CPP_OPTION (pfile
, chill
))
950 skip_line_comment (pfile
);
957 /* Identical to skip_comment except that it copies the comment into the
958 token_buffer. This is used if !discard_comments. */
960 copy_comment (pfile
, m
)
964 const U_CHAR
*start
= CPP_BUFFER (pfile
)->cur
; /* XXX Layering violation */
967 if (skip_comment (pfile
, m
) == m
)
970 limit
= CPP_BUFFER (pfile
)->cur
;
971 CPP_RESERVE (pfile
, limit
- start
+ 2);
972 CPP_PUTC_Q (pfile
, m
);
973 for (; start
<= limit
; start
++)
975 CPP_PUTC_Q (pfile
, *start
);
981 null_warning (pfile
, count
)
986 cpp_warning (pfile
, "embedded null character ignored");
988 cpp_warning (pfile
, "embedded null characters ignored");
991 /* Skip whitespace \-newline and comments. Does not macro-expand. */
994 _cpp_skip_hspace (pfile
)
997 unsigned int null_count
= 0;
1005 else if (is_hspace(c
))
1007 if ((c
== '\f' || c
== '\v') && CPP_PEDANTIC (pfile
))
1008 cpp_pedwarn (pfile
, "%s in preprocessing directive",
1009 c
== '\f' ? "formfeed" : "vertical tab");
1015 /* \r is a backslash-newline marker if !has_escapes, and
1016 a deletable-whitespace or no-reexpansion marker otherwise. */
1017 if (CPP_BUFFER (pfile
)->has_escapes
)
1025 CPP_BUMP_LINE (pfile
);
1027 else if (c
== '/' || c
== '-')
1029 c
= skip_comment (pfile
, c
);
1039 null_warning (pfile
, null_count
);
1042 /* Read and discard the rest of the current line. */
1045 _cpp_skip_rest_of_line (pfile
)
1059 if (! CPP_BUFFER (pfile
)->has_escapes
)
1060 CPP_BUMP_LINE (pfile
);
1065 skip_string (pfile
, c
);
1070 skip_comment (pfile
, c
);
1075 if (CPP_PEDANTIC (pfile
))
1076 cpp_pedwarn (pfile
, "%s in preprocessing directive",
1077 c
== '\f' ? "formfeed" : "vertical tab");
1084 /* Parse an identifier starting with C. */
1087 _cpp_parse_name (pfile
, c
)
1099 if (c
== '$' && CPP_PEDANTIC (pfile
))
1100 cpp_pedwarn (pfile
, "`$' in identifier");
1102 CPP_RESERVE(pfile
, 2); /* One more for final NUL. */
1103 CPP_PUTC_Q (pfile
, c
);
1111 /* Parse and skip over a string starting with C. A single quoted
1112 string is treated like a double -- some programs (e.g., troff) are
1113 perverse this way. (However, a single quoted string is not allowed
1114 to extend over multiple lines.) */
1116 skip_string (pfile
, c
)
1120 unsigned int start_line
, start_column
;
1121 unsigned int null_count
= 0;
1123 start_line
= CPP_BUF_LINE (CPP_BUFFER (pfile
));
1124 start_column
= CPP_BUF_COL (CPP_BUFFER (pfile
));
1131 cpp_error_with_line (pfile
, start_line
, start_column
,
1132 "unterminated string or character constant");
1133 if (pfile
->multiline_string_line
!= start_line
1134 && pfile
->multiline_string_line
!= 0)
1135 cpp_error_with_line (pfile
,
1136 pfile
->multiline_string_line
, -1,
1137 "possible real start of unterminated constant");
1138 pfile
->multiline_string_line
= 0;
1146 CPP_BUMP_LINE (pfile
);
1147 /* In Fortran and assembly language, silently terminate
1148 strings of either variety at end of line. This is a
1149 kludge around not knowing where comments are in these
1151 if (CPP_OPTION (pfile
, lang_fortran
)
1152 || CPP_OPTION (pfile
, lang_asm
))
1157 /* Character constants may not extend over multiple lines.
1158 In Standard C, neither may strings. We accept multiline
1159 strings as an extension. */
1162 cpp_error_with_line (pfile
, start_line
, start_column
,
1163 "unterminated character constant");
1167 if (CPP_PEDANTIC (pfile
) && pfile
->multiline_string_line
== 0)
1168 cpp_pedwarn_with_line (pfile
, start_line
, start_column
,
1169 "string constant runs past end of line");
1170 if (pfile
->multiline_string_line
== 0)
1171 pfile
->multiline_string_line
= start_line
;
1175 if (CPP_BUFFER (pfile
)->has_escapes
)
1177 cpp_ice (pfile
, "\\r escape inside string constant");
1181 /* Backslash newline is replaced by nothing at all. */
1182 CPP_BUMP_LINE (pfile
);
1198 if (null_count
== 1)
1199 cpp_warning (pfile
, "null character in string or character constant");
1200 else if (null_count
> 1)
1201 cpp_warning (pfile
, "null characters in string or character constant");
1204 /* Parse a string and copy it to the output. */
1207 parse_string (pfile
, c
)
1211 const U_CHAR
*start
= CPP_BUFFER (pfile
)->cur
; /* XXX Layering violation */
1212 const U_CHAR
*limit
;
1214 skip_string (pfile
, c
);
1216 limit
= CPP_BUFFER (pfile
)->cur
;
1217 CPP_RESERVE (pfile
, limit
- start
+ 2);
1218 CPP_PUTC_Q (pfile
, c
);
1219 for (; start
< limit
; start
++)
1221 CPP_PUTC_Q (pfile
, *start
);
1224 /* Get the next token, and add it to the text in pfile->token_buffer.
1225 Return the kind of token we got. */
1228 _cpp_lex_token (pfile
)
1232 enum cpp_ttype token
;
1234 if (CPP_BUFFER (pfile
) == NULL
)
1245 if (PEEKC () == '=')
1249 if (CPP_OPTION (pfile
, discard_comments
))
1250 c
= skip_comment (pfile
, c
);
1252 c
= copy_comment (pfile
, c
);
1256 /* Comments are equivalent to spaces.
1257 For -traditional, a comment is equivalent to nothing. */
1258 if (!CPP_OPTION (pfile
, discard_comments
))
1260 else if (CPP_TRADITIONAL (pfile
))
1264 CPP_PUTC (pfile
, c
);
1269 CPP_PUTC (pfile
, c
);
1276 CPP_PUTC (pfile
, c2
);
1279 else if (c2
== '%' && PEEKN (1) == ':')
1281 /* Digraph: "%:" == "#". */
1283 CPP_RESERVE (pfile
, 2);
1284 CPP_PUTC_Q (pfile
, c2
);
1285 CPP_PUTC_Q (pfile
, GETC ());
1293 parse_string (pfile
, c
);
1294 return c
== '\'' ? CPP_CHAR
: CPP_STRING
;
1297 if (!CPP_OPTION (pfile
, dollars_in_ident
))
1303 /* Digraph: ":>" == "]". */
1305 || (c2
== ':' && CPP_OPTION (pfile
, cplusplus
)))
1313 if (c2
== c
|| c2
== '=')
1318 /* Digraphs: "%:" == "#", "%>" == "}". */
1323 CPP_RESERVE (pfile
, 2);
1324 CPP_PUTC_Q (pfile
, c
);
1325 CPP_PUTC_Q (pfile
, c2
);
1331 CPP_RESERVE (pfile
, 2);
1332 CPP_PUTC_Q (pfile
, c
);
1333 CPP_PUTC_Q (pfile
, c2
);
1334 return CPP_OPEN_BRACE
;
1336 /* else fall through */
1342 if (PEEKC () == '=')
1350 if (CPP_OPTION (pfile
, chill
))
1351 goto comment
; /* Chill style comment */
1359 if (CPP_OPTION (pfile
, cplusplus
) && PEEKN (1) == '*')
1361 /* In C++, there's a ->* operator. */
1363 CPP_RESERVE (pfile
, 4);
1364 CPP_PUTC_Q (pfile
, c
);
1365 CPP_PUTC_Q (pfile
, GETC ());
1366 CPP_PUTC_Q (pfile
, GETC ());
1374 if (pfile
->parsing_include_directive
)
1378 CPP_PUTC (pfile
, c
);
1382 if (c
== '\n' || c
== EOF
)
1385 "missing '>' in `#include <FILENAME>'");
1390 if (!CPP_BUFFER (pfile
)->has_escapes
)
1392 /* Backslash newline is replaced by nothing. */
1393 CPP_ADJUST_WRITTEN (pfile
, -1);
1394 CPP_BUMP_LINE (pfile
);
1398 /* We might conceivably get \r- or \r<space> in
1399 here. Just delete 'em. */
1401 if (d
!= '-' && d
!= ' ')
1402 cpp_ice (pfile
, "unrecognized escape \\r%c", d
);
1403 CPP_ADJUST_WRITTEN (pfile
, -1);
1409 /* Digraphs: "<%" == "{", "<:" == "[". */
1414 CPP_RESERVE (pfile
, 2);
1415 CPP_PUTC_Q (pfile
, c
);
1416 CPP_PUTC_Q (pfile
, c2
);
1417 return CPP_CLOSE_BRACE
;
1421 /* else fall through */
1426 /* GNU C++ supports MIN and MAX operators <? and >?. */
1427 if (c2
!= c
&& (!CPP_OPTION (pfile
, cplusplus
) || c2
!= '?'))
1430 CPP_RESERVE (pfile
, 3);
1431 CPP_PUTC_Q (pfile
, c
);
1432 CPP_PUTC_Q (pfile
, c2
);
1433 if (PEEKC () == '=')
1434 CPP_PUTC_Q (pfile
, GETC ());
1441 CPP_PUTC (pfile
, c
);
1446 /* In C++ there's a .* operator. */
1447 if (CPP_OPTION (pfile
, cplusplus
) && c2
== '*')
1450 if (c2
== '.' && PEEKN(1) == '.')
1452 CPP_RESERVE (pfile
, 3);
1453 CPP_PUTC_Q (pfile
, '.');
1454 CPP_PUTC_Q (pfile
, '.');
1455 CPP_PUTC_Q (pfile
, '.');
1457 return CPP_ELLIPSIS
;
1462 CPP_RESERVE (pfile
, 2);
1463 CPP_PUTC_Q (pfile
, c
);
1464 CPP_PUTC_Q (pfile
, GETC ());
1469 if ((c2
== '\'' || c2
== '\"') && !CPP_TRADITIONAL (pfile
))
1471 CPP_PUTC (pfile
, c
);
1473 parse_string (pfile
, c
);
1474 return c
== '\'' ? CPP_WCHAR
: CPP_WSTRING
;
1478 case '0': case '1': case '2': case '3': case '4':
1479 case '5': case '6': case '7': case '8': case '9':
1484 CPP_RESERVE (pfile
, 2);
1485 CPP_PUTC_Q (pfile
, c
);
1489 if (!is_numchar(c
) && c
!= '.'
1490 && ((c2
!= 'e' && c2
!= 'E'
1491 && ((c2
!= 'p' && c2
!= 'P')
1492 || CPP_OPTION (pfile
, c89
)))
1493 || (c
!= '+' && c
!= '-')))
1499 case 'b': case 'c': case 'd': case 'h': case 'o':
1500 case 'B': case 'C': case 'D': case 'H': case 'O':
1501 if (CPP_OPTION (pfile
, chill
) && PEEKC () == '\'')
1503 CPP_RESERVE (pfile
, 2);
1504 CPP_PUTC_Q (pfile
, c
);
1505 CPP_PUTC_Q (pfile
, '\'');
1511 goto chill_number_eof
;
1514 CPP_PUTC (pfile
, c
);
1518 CPP_RESERVE (pfile
, 2);
1519 CPP_PUTC_Q (pfile
, c
);
1532 case 'a': case 'e': case 'f': case 'g': case 'i': case 'j':
1533 case 'k': case 'l': case 'm': case 'n': case 'p': case 'q':
1534 case 'r': case 's': case 't': case 'u': case 'v': case 'w':
1535 case 'x': case 'y': case 'z':
1536 case 'A': case 'E': case 'F': case 'G': case 'I': case 'J':
1537 case 'K': case 'M': case 'N': case 'P': case 'Q': case 'R':
1538 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1541 _cpp_parse_name (pfile
, c
);
1544 case ' ': case '\t': case '\v': case '\f': case '\0':
1553 CPP_PUTC (pfile
, c
);
1555 if (c
== EOF
|| !is_hspace(c
))
1560 null_warning (pfile
, null_count
);
1565 if (CPP_BUFFER (pfile
)->has_escapes
)
1570 if (pfile
->output_escapes
)
1571 CPP_PUTS (pfile
, "\r-", 2);
1572 _cpp_parse_name (pfile
, GETC ());
1577 /* "\r " means a space, but only if necessary to prevent
1578 accidental token concatenation. */
1579 CPP_RESERVE (pfile
, 2);
1580 if (pfile
->output_escapes
)
1581 CPP_PUTC_Q (pfile
, '\r');
1582 CPP_PUTC_Q (pfile
, c
);
1587 cpp_ice (pfile
, "unrecognized escape \\r%c", c
);
1593 /* Backslash newline is ignored. */
1594 if (!ACTIVE_MARK_P (pfile
))
1595 CPP_BUMP_LINE (pfile
);
1600 CPP_PUTC (pfile
, c
);
1603 case '(': token
= CPP_OPEN_PAREN
; goto char1
;
1604 case ')': token
= CPP_CLOSE_PAREN
; goto char1
;
1605 case '{': token
= CPP_OPEN_BRACE
; goto char1
;
1606 case '}': token
= CPP_CLOSE_BRACE
; goto char1
;
1607 case ',': token
= CPP_COMMA
; goto char1
;
1608 case ';': token
= CPP_SEMICOLON
; goto char1
;
1614 CPP_PUTC (pfile
, c
);
1619 /* Check for and expand a macro, which is from WRITTEN to CPP_WRITTEN (pfile).
1620 Caller is expected to have checked no_macro_expand. */
1622 maybe_macroexpand (pfile
, written
)
1626 U_CHAR
*macro
= pfile
->token_buffer
+ written
;
1627 size_t len
= CPP_WRITTEN (pfile
) - written
;
1628 cpp_hashnode
*hp
= cpp_lookup (pfile
, macro
, len
);
1630 /* cpp_lookup never returns null. */
1631 if (hp
->type
== T_VOID
)
1633 if (hp
->disabled
|| hp
->type
== T_IDENTITY
)
1635 if (pfile
->output_escapes
)
1637 /* Insert a no-reexpand marker before IDENT. */
1638 CPP_RESERVE (pfile
, 2);
1639 CPP_ADJUST_WRITTEN (pfile
, 2);
1640 macro
= pfile
->token_buffer
+ written
;
1642 memmove (macro
+ 2, macro
, len
);
1648 if (hp
->type
== T_EMPTY
)
1650 /* Special case optimization: macro expands to nothing. */
1651 CPP_SET_WRITTEN (pfile
, written
);
1652 CPP_PUTC_Q (pfile
, ' ');
1656 /* If macro wants an arglist, verify that a '(' follows. */
1657 if (hp
->type
== T_FMACRO
)
1659 int macbuf_whitespace
= 0;
1662 while (CPP_IS_MACRO_BUFFER (CPP_BUFFER (pfile
)))
1664 const U_CHAR
*point
= CPP_BUFFER (pfile
)->cur
;
1667 _cpp_skip_hspace (pfile
);
1674 if (point
!= CPP_BUFFER (pfile
)->cur
)
1675 macbuf_whitespace
= 1;
1679 goto not_macro_call
;
1680 cpp_pop_buffer (pfile
);
1683 CPP_SET_MARK (pfile
);
1686 _cpp_skip_hspace (pfile
);
1693 CPP_GOTO_MARK (pfile
);
1698 if (macbuf_whitespace
)
1699 CPP_PUTC (pfile
, ' ');
1701 /* K+R treated this as a hard error. */
1702 if (CPP_WTRADITIONAL (pfile
))
1704 "function macro %s must be used with arguments in traditional C",
1711 /* This is now known to be a macro call.
1712 Expand the macro, reading arguments as needed,
1713 and push the expansion on the input stack. */
1714 _cpp_macroexpand (pfile
, hp
);
1715 CPP_SET_WRITTEN (pfile
, written
);
1719 /* Complain about \v or \f in a preprocessing directive (constraint
1720 violation, C99 6.10 para 5). Caller has checked CPP_PEDANTIC. */
1722 pedantic_whitespace (pfile
, p
, len
)
1730 cpp_pedwarn (pfile
, "vertical tab in preprocessing directive");
1731 else if (*p
== '\f')
1732 cpp_pedwarn (pfile
, "form feed in preprocessing directive");
1740 cpp_get_token (pfile
)
1743 enum cpp_ttype token
;
1744 long written
= CPP_WRITTEN (pfile
);
1748 token
= _cpp_lex_token (pfile
);
1753 if (pfile
->skipping
)
1755 pfile
->potential_control_macro
= 0;
1756 pfile
->only_seen_white
= 0;
1764 if (pfile
->only_seen_white
== 0)
1765 pfile
->only_seen_white
= 1;
1766 CPP_BUMP_LINE (pfile
);
1770 pfile
->potential_control_macro
= 0;
1771 if (!pfile
->only_seen_white
)
1773 /* XXX shouldn't have to do this - remove the hash or %: from
1774 the token buffer. */
1775 if (CPP_PWRITTEN (pfile
)[-1] == '#')
1776 CPP_ADJUST_WRITTEN (pfile
, -1);
1778 CPP_ADJUST_WRITTEN (pfile
, -2);
1780 if (_cpp_handle_directive (pfile
))
1782 token
= CPP_DIRECTIVE
;
1785 pfile
->only_seen_white
= 0;
1786 CPP_PUTC (pfile
, '#');
1790 if (pfile
->skipping
)
1792 pfile
->potential_control_macro
= 0;
1793 pfile
->only_seen_white
= 0;
1794 if (! pfile
->no_macro_expand
1795 && maybe_macroexpand (pfile
, written
))
1800 /* Do not run this case through the 'skipping' logic. */
1802 if (CPP_BUFFER (pfile
) == NULL
)
1804 macro_buffer
= CPP_IS_MACRO_BUFFER (CPP_BUFFER (pfile
));
1806 cpp_pop_buffer (pfile
);
1812 if (pfile
->skipping
)
1814 CPP_SET_WRITTEN (pfile
, written
);
1820 /* Like cpp_get_token, but skip spaces and comments. */
1823 cpp_get_non_space_token (pfile
)
1826 int old_written
= CPP_WRITTEN (pfile
);
1829 enum cpp_ttype token
= cpp_get_token (pfile
);
1830 if (token
!= CPP_COMMENT
&& token
!= CPP_HSPACE
&& token
!= CPP_VSPACE
)
1832 CPP_SET_WRITTEN (pfile
, old_written
);
1836 /* Like cpp_get_token, except that it does not execute directives,
1837 does not consume vertical space, and discards horizontal space. */
1839 _cpp_get_directive_token (pfile
)
1843 enum cpp_ttype token
;
1847 at_bol
= (CPP_BUFFER (pfile
)->cur
== CPP_BUFFER (pfile
)->line_base
);
1848 old_written
= CPP_WRITTEN (pfile
);
1849 token
= _cpp_lex_token (pfile
);
1856 /* Put it back and return VSPACE. */
1858 CPP_ADJUST_WRITTEN (pfile
, -1);
1862 /* The purpose of this rather strange check is to prevent pedantic
1863 warnings for ^L in an #ifdefed out block. */
1864 if (CPP_PEDANTIC (pfile
) && ! at_bol
)
1865 pedantic_whitespace (pfile
, pfile
->token_buffer
+ old_written
,
1866 CPP_WRITTEN (pfile
) - old_written
);
1867 CPP_SET_WRITTEN (pfile
, old_written
);
1872 if (! pfile
->no_macro_expand
1873 && maybe_macroexpand (pfile
, old_written
))
1878 if (CPP_IS_MACRO_BUFFER (CPP_BUFFER (pfile
)))
1880 cpp_pop_buffer (pfile
);
1884 /* This can happen for files that don't end with a newline,
1885 and for cpp_define and friends. Pretend they do, so
1886 callers don't have to deal. A warning will be issued by
1887 someone else, if necessary. */
1892 /* Determine the current line and column. Used only by read_and_prescan. */
1894 find_position (start
, limit
, linep
)
1897 unsigned long *linep
;
1899 unsigned long line
= *linep
;
1900 U_CHAR
*lbase
= start
;
1901 while (start
< limit
)
1903 U_CHAR ch
= *start
++;
1904 if (ch
== '\n' || ch
== '\r')
1914 /* The following table is used by _cpp_prescan. If we have
1915 designated initializers, it can be constant data; otherwise, it is
1916 set up at runtime by _cpp_init_input_buffer. */
1918 #if (GCC_VERSION >= 2007)
1919 #define init_chartab() /* nothing */
1920 #define CHARTAB __extension__ static const U_CHAR chartab[UCHAR_MAX + 1] = {
1922 #define s(p, v) [p] = v,
1924 #define CHARTAB static U_CHAR chartab[UCHAR_MAX + 1] = { 0 }; \
1925 static void init_chartab PARAMS ((void)) { \
1926 unsigned char *x = chartab;
1928 #define s(p, v) x[p] = v;
1931 /* Table of characters that can't be handled in the inner loop.
1932 Also contains the mapping between trigraph third characters and their
1934 #define SPECCASE_CR 1
1935 #define SPECCASE_BACKSLASH 2
1936 #define SPECCASE_QUESTION 3
1939 s('\r', SPECCASE_CR
)
1940 s('\\', SPECCASE_BACKSLASH
)
1941 s('?', SPECCASE_QUESTION
)
1943 s('=', '#') s(')', ']') s('!', '|')
1944 s('(', '[') s('\'', '^') s('>', '}')
1945 s('/', '\\') s('<', '{') s('-', '~')
1952 #define NORMAL(c) ((chartab[c]) == 0 || (chartab[c]) > SPECCASE_QUESTION)
1953 #define NONTRI(c) ((c) <= SPECCASE_QUESTION)
1955 /* Prescan pass over a file already loaded into BUF. This is
1956 translation phases 1 and 2 (C99 5.1.1.2).
1958 Convert end-of-line markers (\n, \r, \r\n, \n\r) to
1959 canonical form (\n). If enabled, convert and/or warn about
1960 trigraphs. Convert backslash-newline to a one-character escape
1961 (\r) and remove it from "embarrassing" places (i.e. the middle of a
1962 token). If there is no newline at the end of the file, add one and
1963 warn. Returns -1 on failure, or the actual length of the data to
1966 This function does a lot of work, and can be a serious performance
1967 bottleneck. It has been tuned heavily; make sure you understand it
1968 before hacking. The common case - no trigraphs, Unix style line
1969 breaks, backslash-newline set off by whitespace, newline at EOF -
1970 has been optimized at the expense of the others. The performance
1971 penalty for DOS style line breaks (\r\n) is about 15%.
1973 Warnings lose particularly heavily since we have to determine the
1974 line number, which involves scanning from the beginning of the file
1975 or from the last warning. The penalty for the absence of a newline
1976 at the end of reload1.c is about 60%. (reload1.c is 329k.)
1978 If your file has more than one kind of end-of-line marker, you
1979 will get messed-up line numbering. */
1982 _cpp_prescan (pfile
, fp
, len
)
1988 const U_CHAR
*ibase
, *ip
, *ilimit
;
1991 unsigned int deferred_newlines
;
1993 /* Allocate an extra byte in case we must add a trailing \n. */
1994 buf
= (U_CHAR
*) xmalloc (len
+ 1);
1995 line_base
= op
= buf
;
1996 ip
= ibase
= fp
->buf
;
1997 ilimit
= ibase
+ len
;
1999 deferred_newlines
= 0;
2005 /* Deal with \-newline, potentially in the middle of a token. */
2006 if (deferred_newlines
)
2008 if (op
!= buf
&& ! is_space (op
[-1]) && op
[-1] != '\r')
2010 /* Previous was not white space. Skip to white
2011 space, if we can, before outputting the \r's */
2019 memcpy (op
, ip
, iq
- ip
);
2025 while (deferred_newlines
)
2026 deferred_newlines
--, *op
++ = '\r';
2029 /* Copy as much as we can without special treatment. */
2031 while (iq
< ilimit
&& NORMAL (*iq
)) iq
++;
2032 memcpy (op
, ip
, iq
- ip
);
2040 switch (chartab
[*ip
++])
2042 case SPECCASE_CR
: /* \r */
2045 if (ip
< ilimit
&& *ip
== '\n')
2051 case SPECCASE_BACKSLASH
: /* \ */
2057 deferred_newlines
++;
2059 if (*ip
== '\r') ip
++;
2062 else if (*ip
== '\r')
2064 deferred_newlines
++;
2066 if (*ip
== '\n') ip
++;
2074 case SPECCASE_QUESTION
: /* ? */
2078 *op
++ = '?'; /* Normal non-trigraph case */
2079 if (ip
> ilimit
- 2 || ip
[0] != '?')
2087 if (CPP_OPTION (pfile
, warn_trigraphs
))
2090 line_base
= find_position (line_base
, op
, &line
);
2091 col
= op
- line_base
+ 1;
2092 if (CPP_OPTION (pfile
, trigraphs
))
2093 cpp_warning_with_line (pfile
, line
, col
,
2094 "trigraph ??%c converted to %c", d
, t
);
2096 cpp_warning_with_line (pfile
, line
, col
,
2097 "trigraph ??%c ignored", d
);
2101 if (CPP_OPTION (pfile
, trigraphs
))
2103 op
[-1] = t
; /* Overwrite '?' */
2120 #ifdef HAVE_MMAP_FILE
2122 munmap ((caddr_t
) fp
->buf
, len
);
2125 free ((PTR
) fp
->buf
);
2130 line_base
= find_position (line_base
, op
, &line
);
2131 col
= op
- line_base
+ 1;
2132 cpp_warning_with_line (pfile
, line
, col
, "no newline at end of file");
2140 /* Allocate pfile->input_buffer, and initialize chartab[]
2141 if it hasn't happened already. */
2144 _cpp_init_input_buffer (pfile
)
2150 _cpp_init_toklist (&pfile
->directbuf
, NO_DUMMY_TOKEN
);
2152 /* Determine the appropriate size for the input buffer. Normal C
2153 source files are smaller than eight K. */
2154 /* 8Kbytes of buffer proper, 1 to detect running off the end without
2155 address arithmetic all the time, and 3 for pushback during buffer
2156 refill, in case there's a potential trigraph or end-of-line
2157 digraph at the end of a block. */
2159 tmp
= (U_CHAR
*) xmalloc (8192 + 1 + 3);
2160 pfile
->input_buffer
= tmp
;
2161 pfile
->input_buffer_len
= 8192;
2165 Compares, in the manner of strcmp(3), the token beginning at TOKEN
2166 and extending for LEN characters to the NUL-terminated string
2167 STRING. Typical usage:
2169 if (! cpp_idcmp (pfile->token_buffer + here, CPP_WRITTEN (pfile) - here,
2175 cpp_idcmp (token
, len
, string
)
2176 const U_CHAR
*token
;
2180 size_t len2
= strlen (string
);
2183 if ((r
= memcmp (token
, string
, MIN (len
, len2
))))
2186 /* The longer of the two strings sorts after the shorter. */
2189 else if (len
< len2
)
2197 /* Lexing algorithm.
2199 The original lexer in cpplib was made up of two passes: a first pass
2200 that replaced trigraphs and deleted esacped newlines, and a second
2201 pass that tokenized the result of the first pass. Tokenisation was
2202 performed by peeking at the next character in the input stream. For
2203 example, if the input stream contained "!=", the handler for the !
2204 character would peek at the next character, and if it were a '='
2205 would skip over it, and return a "!=" token, otherwise it would
2206 return just the "!" token.
2208 To implement a single-pass lexer, this peeking ahead is unworkable.
2209 An arbitrary number of escaped newlines, and trigraphs (in particular
2210 ??/ which translates to the escape \), could separate the '!' and '='
2211 in the input stream, yet the next token is still a "!=".
2213 Suppose instead that we lex by one logical line at a time, producing
2214 a token list or stack for each logical line, and when seeing the '!'
2215 push a CPP_NOT token on the list. Then if the '!' is part of a
2216 longer token ("!=") we know we must see the remainder of the token by
2217 the time we reach the end of the logical line. Thus we can have the
2218 '=' handler look at the previous token (at the end of the list / top
2219 of the stack) and see if it is a "!" token, and if so, instead of
2220 pushing a "=" token revise the existing token to be a "!=" token.
2222 This works in the presence of escaped newlines, because the '\' would
2223 have been pushed on the top of the stack as a CPP_BACKSLASH. The
2224 newline ('\n' or '\r') handler looks at the token at the top of the
2225 stack to see if it is a CPP_BACKSLASH, and if so discards both.
2226 Otherwise it pushes the newline (CPP_VSPACE) token as normal. Hence
2227 the '=' handler would never see any intervening escaped newlines.
2229 To make trigraphs work in this context, as in precedence trigraphs
2230 are highest and converted before anything else, the '?' handler does
2231 lookahead to see if it is a trigraph, and if so skips the trigraph
2232 and pushes the token it represents onto the top of the stack. This
2233 also works in the particular case of a CPP_BACKSLASH trigraph.
2235 To the preprocessor, whitespace is only significant to the point of
2236 knowing whether whitespace precedes a particular token. For example,
2237 the '=' handler needs to know whether there was whitespace between it
2238 and a "!" token on the top of the stack, to make the token conversion
2239 decision correctly. So each token has a PREV_WHITESPACE flag to
2240 indicate this - the standard permits consecutive whitespace to be
2241 regarded as a single space. The compiler front ends are not
2242 interested in whitespace at all; they just require a token stream.
2243 Another place where whitespace is significant to the preprocessor is
2244 a #define statment - if there is whitespace between the macro name
2245 and an initial "(" token the macro is "object-like", otherwise it is
2246 a function-like macro that takes arguments.
2248 However, all is not rosy. Parsing of identifiers, numbers, comments
2249 and strings becomes trickier because of the possibility of raw
2250 trigraphs and escaped newlines in the input stream.
2252 The trigraphs are three consecutive characters beginning with two
2253 question marks. A question mark is not valid as part of a number or
2254 identifier, so parsing of a number or identifier terminates normally
2255 upon reaching it, returning to the mainloop which handles the
2256 trigraph just like it would in any other position. Similarly for the
2257 backslash of a backslash-newline combination. So we just need the
2258 escaped-newline dropper in the mainloop to check if the token on the
2259 top of the stack after dropping the escaped newline is a number or
2260 identifier, and if so to continue the processing it as if nothing had
2263 For strings, we replace trigraphs whenever we reach a quote or
2264 newline, because there might be a backslash trigraph escaping them.
2265 We need to be careful that we start trigraph replacing from where we
2266 left off previously, because it is possible for a first scan to leave
2267 "fake" trigraphs that a second scan would pick up as real (e.g. the
2268 sequence "????/\n=" would find a fake ??= trigraph after removing the
2271 For line comments, on reaching a newline we scan the previous
2272 character(s) to see if it escaped, and continue if it is. Block
2273 comments ignore everything and just focus on finding the comment
2274 termination mark. The only difficult thing, and it is surprisingly
2275 tricky, is checking if an asterisk precedes the final slash since
2276 they could be separated by escaped newlines. If the preprocessor is
2277 invoked with the output comments option, we don't bother removing
2278 escaped newlines and replacing trigraphs for output.
2280 Finally, numbers can begin with a period, which is pushed initially
2281 as a CPP_DOT token in its own right. The digit handler checks if the
2282 previous token was a CPP_DOT not separated by whitespace, and if so
2283 pops it off the stack and pushes a period into the number's buffer
2284 before calling the number parser.
2288 static const unsigned char *digraph_spellings
[] = {U
"%:", U
"%:%:", U
"<:",
2289 U
":>", U
"<%", U
"%>"};
2290 static unsigned char trigraph_map
[256];
2293 init_trigraph_map ()
2295 trigraph_map
['='] = '#';
2296 trigraph_map
['('] = '[';
2297 trigraph_map
[')'] = ']';
2298 trigraph_map
['/'] = '\\';
2299 trigraph_map
['\''] = '^';
2300 trigraph_map
['<'] = '{';
2301 trigraph_map
['>'] = '}';
2302 trigraph_map
['!'] = '|';
2303 trigraph_map
['-'] = '~';
2306 /* Call when a trigraph is encountered. It warns if necessary, and
2307 returns true if the trigraph should be honoured. END is the third
2308 character of a trigraph in the input stream. */
2310 trigraph_ok (pfile
, end
)
2312 const unsigned char *end
;
2314 int accept
= CPP_OPTION (pfile
, trigraphs
);
2316 if (CPP_OPTION (pfile
, warn_trigraphs
))
2318 unsigned int col
= end
- 1 - pfile
->buffer
->line_base
;
2320 cpp_warning_with_line (pfile
, pfile
->buffer
->lineno
, col
,
2321 "trigraph ??%c converted to %c",
2322 (int) *end
, (int) trigraph_map
[*end
]);
2324 cpp_warning_with_line (pfile
, pfile
->buffer
->lineno
, col
,
2325 "trigraph ??%c ignored", (int) *end
);
2330 /* Scan a string for trigraphs, warning or replacing them inline as
2331 appropriate. When parsing a string, we must call this routine
2332 before processing a newline character (if trigraphs are enabled),
2333 since the newline might be escaped by a preceding backslash
2334 trigraph sequence. Returns a pointer to the end of the name after
2337 static unsigned char*
2338 trigraph_replace (pfile
, src
, limit
)
2341 unsigned char* limit
;
2343 unsigned char *dest
;
2345 /* Starting with src[1], find two consecutive '?'. The case of no
2346 trigraphs is streamlined. */
2348 for (; src
+ 1 < limit
; src
+= 2)
2353 /* Make src point to the 1st (NOT 2nd) of two consecutive '?'s. */
2356 else if (src
+ 2 == limit
|| src
[1] != '?')
2359 /* Check if it really is a trigraph. */
2360 if (trigraph_map
[src
[2]] == 0)
2364 goto trigraph_found
;
2368 /* Now we have a trigraph, we need to scan the remaining buffer, and
2369 copy-shifting its contents left if replacement is enabled. */
2370 for (; src
+ 2 < limit
; dest
++, src
++)
2371 if ((*dest
= *src
) == '?' && src
[1] == '?' && trigraph_map
[src
[2]])
2375 if (trigraph_ok (pfile
, pfile
->buffer
->cur
- (limit
- src
)))
2376 *dest
= trigraph_map
[*src
];
2379 /* Copy remaining (at most 2) characters. */
2385 /* If CUR is a backslash or the end of a trigraphed backslash, return
2386 a pointer to its beginning, otherwise NULL. We don't read beyond
2387 the buffer start, because there is the start of the comment in the
2389 static const unsigned char *
2390 backslash_start (pfile
, cur
)
2392 const unsigned char *cur
;
2396 if (cur
[0] == '/' && cur
[-1] == '?' && cur
[-2] == '?'
2397 && trigraph_ok (pfile
, cur
))
2402 /* Skip a C-style block comment. This is probably the trickiest
2403 handler. We find the end of the comment by seeing if an asterisk
2404 is before every '/' we encounter. The nasty complication is that a
2405 previous asterisk may be separated by one or more escaped newlines.
2406 Returns non-zero if comment terminated by EOF, zero otherwise. */
2408 skip_block_comment2 (pfile
)
2411 cpp_buffer
*buffer
= pfile
->buffer
;
2412 const unsigned char *char_after_star
= 0;
2413 register const unsigned char *cur
= buffer
->cur
;
2416 /* Inner loop would think the comment has ended if the first comment
2417 character is a '/'. Avoid this and keep the inner loop clean by
2418 skipping such a character. */
2419 if (cur
< buffer
->rlimit
&& cur
[0] == '/')
2422 for (; cur
< buffer
->rlimit
; )
2424 unsigned char c
= *cur
++;
2426 /* People like decorating comments with '*', so check for
2427 '/' instead for efficiency. */
2430 if (cur
[-2] == '*' || cur
- 1 == char_after_star
)
2433 /* Warn about potential nested comments, but not when
2434 the final character inside the comment is a '/'.
2435 Don't bother to get it right across escaped newlines. */
2436 if (CPP_OPTION (pfile
, warn_comments
) && cur
+ 1 < buffer
->rlimit
2437 && cur
[0] == '*' && cur
[1] != '/')
2440 cpp_warning (pfile
, "'/*' within comment");
2443 else if (IS_NEWLINE(c
))
2445 const unsigned char* bslash
= backslash_start (pfile
, cur
- 2);
2447 handle_newline (cur
, buffer
->rlimit
, c
);
2448 /* Work correctly if there is an asterisk before an
2449 arbirtrarily long sequence of escaped newlines. */
2450 if (bslash
&& (bslash
[-1] == '*' || bslash
== char_after_star
))
2451 char_after_star
= cur
;
2453 char_after_star
= 0;
2463 /* Skip a C++ or Chill line comment. Handles escaped newlines.
2464 Returns non-zero if a multiline comment. */
2466 skip_line_comment2 (pfile
)
2469 cpp_buffer
*buffer
= pfile
->buffer
;
2470 register const unsigned char *cur
= buffer
->cur
;
2473 for (; cur
< buffer
->rlimit
; )
2475 unsigned char c
= *cur
++;
2479 /* Check for a (trigaph?) backslash escaping the newline. */
2480 if (!backslash_start (pfile
, cur
- 2))
2483 handle_newline (cur
, buffer
->rlimit
, c
);
2489 buffer
->cur
= cur
- 1; /* Leave newline for caller. */
2493 /* Skips whitespace, stopping at next non-whitespace character.
2494 Adjusts pfile->col_adjust to account for tabs. This enables tokens
2495 to be assigned the correct column. */
2497 skip_whitespace (pfile
, in_directive
)
2501 cpp_buffer
*buffer
= pfile
->buffer
;
2502 register const unsigned char *cur
= buffer
->cur
;
2503 unsigned short null_count
= 0;
2505 for (; cur
< buffer
->rlimit
; )
2507 unsigned char c
= *cur
++;
2511 unsigned int col
= CPP_BUF_COLUMN (buffer
, cur
- 1);
2512 pfile
->col_adjust
+= (CPP_OPTION (pfile
, tabstop
) - 1
2513 - col
% CPP_OPTION(pfile
, tabstop
));
2515 if (IS_HSPACE(c
)) /* FIXME: Fix ISTABLE. */
2517 if (!is_space(c
) || IS_NEWLINE (c
)) /* Main loop handles newlines. */
2521 /* Mut be '\f' or '\v' */
2522 else if (in_directive
&& CPP_PEDANTIC (pfile
))
2523 cpp_pedwarn (pfile
, "%s in preprocessing directive",
2524 c
== '\f' ? "formfeed" : "vertical tab");
2529 buffer
->cur
= cur
- 1;
2531 cpp_warning (pfile
, null_count
> 1 ? "embedded null characters ignored"
2532 : "embedded null character ignored");
2535 /* Parse (append) an identifier. */
2537 parse_name (pfile
, list
, name
)
2542 const unsigned char *name_limit
;
2543 unsigned char *namebuf
;
2544 cpp_buffer
*buffer
= pfile
->buffer
;
2545 register const unsigned char *cur
= buffer
->cur
;
2548 name_limit
= list
->namebuf
+ list
->name_cap
;
2549 namebuf
= list
->namebuf
+ list
->name_used
;
2551 for (; cur
< buffer
->rlimit
&& namebuf
< name_limit
; )
2553 unsigned char c
= *namebuf
= *cur
; /* Copy a single char. */
2559 if (c
== '$' && CPP_PEDANTIC (pfile
))
2562 cpp_pedwarn (pfile
, "'$' character in identifier");
2566 /* Run out of name space? */
2567 if (cur
< buffer
->rlimit
)
2569 list
->name_used
= namebuf
- list
->namebuf
;
2570 auto_expand_name_space (list
);
2576 name
->len
= namebuf
- name
->text
;
2577 list
->name_used
= namebuf
- list
->namebuf
;
2580 /* Parse (append) a number. */
2582 #define VALID_SIGN(c, prevc) \
2583 (((c) == '+' || (c) == '-') && \
2584 ((prevc) == 'e' || (prevc) == 'E' \
2585 || (((prevc) == 'p' || (prevc) == 'P') && !CPP_OPTION (pfile, c89))))
2588 parse_number (pfile
, list
, name
)
2593 const unsigned char *name_limit
;
2594 unsigned char *namebuf
;
2595 cpp_buffer
*buffer
= pfile
->buffer
;
2596 register const unsigned char *cur
= buffer
->cur
;
2599 name_limit
= list
->namebuf
+ list
->name_cap
;
2600 namebuf
= list
->namebuf
+ list
->name_used
;
2602 for (; cur
< buffer
->rlimit
&& namebuf
< name_limit
; )
2604 unsigned char c
= *namebuf
= *cur
; /* Copy a single char. */
2606 /* Perhaps we should accept '$' here if we accept it for
2607 identifiers. We know namebuf[-1] is safe, because for c to
2608 be a sign we must have pushed at least one character. */
2609 if (!is_numchar (c
) && c
!= '.' && ! VALID_SIGN (c
, namebuf
[-1]))
2616 /* Run out of name space? */
2617 if (cur
< buffer
->rlimit
)
2619 list
->name_used
= namebuf
- list
->namebuf
;
2620 auto_expand_name_space (list
);
2626 name
->len
= namebuf
- name
->text
;
2627 list
->name_used
= namebuf
- list
->namebuf
;
2630 /* Places a string terminated by an unescaped TERMINATOR into a
2631 cpp_name, which should be expandable and thus at the top of the
2632 list's stack. Handles embedded trigraphs, if necessary, and
2635 Can be used for character constants (terminator = '\''), string
2636 constants ('"') and angled headers ('>'). Multi-line strings are
2637 allowed, except for within directives. */
2640 parse_string2 (pfile
, list
, name
, terminator
, multiline_ok
)
2644 unsigned int terminator
;
2647 cpp_buffer
*buffer
= pfile
->buffer
;
2648 register const unsigned char *cur
= buffer
->cur
;
2649 const unsigned char *name_limit
;
2650 unsigned char *namebuf
;
2651 unsigned int null_count
= 0;
2652 int trigraphed_len
= 0;
2655 name_limit
= list
->namebuf
+ list
->name_cap
;
2656 namebuf
= list
->namebuf
+ list
->name_used
;
2658 for (; cur
< buffer
->rlimit
&& namebuf
< name_limit
; )
2660 unsigned int c
= *namebuf
++ = *cur
++; /* Copy a single char. */
2664 else if (c
== terminator
|| IS_NEWLINE (c
))
2666 /* Needed for trigraph_replace and multiline string warning. */
2669 /* Scan for trigraphs before checking if backslash-escaped. */
2670 if (CPP_OPTION (pfile
, trigraphs
)
2671 || CPP_OPTION (pfile
, warn_trigraphs
))
2673 namebuf
= trigraph_replace (pfile
, name
->text
+ trigraphed_len
,
2675 trigraphed_len
= namebuf
- 2 - (name
->text
+ trigraphed_len
);
2676 if (trigraphed_len
< 0)
2680 namebuf
--; /* Drop the newline / terminator from the name. */
2683 /* Drop a backslash newline, and continue. */
2684 if (namebuf
[-1] == '\\')
2686 handle_newline (cur
, buffer
->rlimit
, c
);
2693 /* In Fortran and assembly language, silently terminate
2694 strings of either variety at end of line. This is a
2695 kludge around not knowing where comments are in these
2697 if (CPP_OPTION (pfile
, lang_fortran
)
2698 || CPP_OPTION (pfile
, lang_asm
))
2701 /* Character constants, headers and asserts may not
2702 extend over multiple lines. In Standard C, neither
2703 may strings. We accept multiline strings as an
2704 extension, but not in directives. */
2708 cur
++; /* Move forwards again. */
2710 if (pfile
->multiline_string_line
== 0)
2712 pfile
->multiline_string_line
= list
->line
;
2713 if (CPP_PEDANTIC (pfile
))
2714 cpp_pedwarn (pfile
, "multi-line string constant");
2718 handle_newline (cur
, buffer
->rlimit
, c
);
2722 unsigned char *temp
;
2724 /* An odd number of consecutive backslashes represents
2725 an escaped terminator. */
2727 while (temp
>= name
->text
&& *temp
== '\\')
2730 if ((namebuf
- temp
) & 1)
2737 /* Run out of name space? */
2738 if (cur
< buffer
->rlimit
)
2740 list
->name_used
= namebuf
- list
->namebuf
;
2741 auto_expand_name_space (list
);
2745 /* We may not have trigraph-replaced the input for this code path,
2746 but as the input is in error by being unterminated we don't
2747 bother. Prevent warnings about no newlines at EOF. */
2748 if (IS_NEWLINE(cur
[-1]))
2752 cpp_error (pfile
, "missing terminating %c character", (int) terminator
);
2754 if (terminator
== '\"' && pfile
->multiline_string_line
!= list
->line
2755 && pfile
->multiline_string_line
!= 0)
2757 cpp_error_with_line (pfile
, pfile
->multiline_string_line
, -1,
2758 "possible start of unterminated string literal");
2759 pfile
->multiline_string_line
= 0;
2764 name
->len
= namebuf
- name
->text
;
2765 list
->name_used
= namebuf
- list
->namebuf
;
2768 cpp_warning (pfile
, (null_count
> 1 ? "null characters preserved"
2769 : "null character preserved"));
2772 /* The character TYPE helps us distinguish comment types: '*' = C
2773 style, '-' = Chill-style and '/' = C++ style. For code simplicity,
2774 the stored comment includes the comment start and any terminator. */
2776 #define COMMENT_START_LEN 2
2778 save_comment (list
, token
, from
, len
, type
)
2781 const unsigned char *from
;
2785 unsigned char *buffer
;
2787 len
+= COMMENT_START_LEN
;
2789 if (list
->name_used
+ len
> list
->name_cap
)
2790 expand_name_space (list
, len
);
2792 INIT_TOKEN_NAME (list
, token
);
2793 token
->type
= CPP_COMMENT
;
2794 token
->val
.name
.len
= len
;
2796 buffer
= list
->namebuf
+ list
->name_used
;
2797 list
->name_used
+= len
;
2799 /* Copy the comment. */
2810 memcpy (buffer
, from
, len
- COMMENT_START_LEN
);
2814 * The tokenizer's main loop. Returns a token list, representing a
2815 * logical line in the input file. On EOF after some tokens have
2816 * been processed, we return immediately. Then in next call, or if
2817 * EOF occurred at the beginning of a logical line, a single CPP_EOF
2818 * token is placed in the list.
2820 * Implementation relies almost entirely on lookback, rather than
2821 * looking forwards. This means that tokenization requires just
2822 * a single pass of the file, even in the presence of trigraphs and
2823 * escaped newlines, providing significant performance benefits.
2824 * Trigraph overhead is negligible if they are disabled, and low
2825 * even when enabled.
2828 #define IS_DIRECTIVE() (list->tokens[first_token].type == CPP_HASH)
2831 _cpp_lex_line (pfile
, list
)
2835 cpp_token
*cur_token
, *token_limit
;
2836 cpp_buffer
*buffer
= pfile
->buffer
;
2837 register const unsigned char *cur
= buffer
->cur
;
2838 unsigned char flags
= 0;
2839 unsigned int first_token
= list
->tokens_used
;
2841 list
->line
= CPP_BUF_LINE (buffer
);
2842 pfile
->col_adjust
= 0;
2844 token_limit
= list
->tokens
+ list
->tokens_cap
;
2845 cur_token
= list
->tokens
+ list
->tokens_used
;
2847 for (; cur
< buffer
->rlimit
&& cur_token
< token_limit
;)
2849 unsigned char c
= *cur
++;
2851 /* Optimize whitespace skipping, as most tokens are probably
2852 separated by whitespace. (' ' '\t' '\v' '\f' '\0'). */
2854 if (is_hspace ((unsigned int) c
))
2856 /* Step back to get the null warning and tab correction. */
2857 buffer
->cur
= cur
- 1;
2858 skip_whitespace (pfile
, IS_DIRECTIVE ());
2861 flags
= PREV_WHITESPACE
;
2862 if (cur
== buffer
->rlimit
)
2867 /* Initialize current token. Its type is set in the switch. */
2868 cur_token
->col
= CPP_BUF_COLUMN (buffer
, cur
);
2869 cur_token
->flags
= flags
;
2874 case '0': case '1': case '2': case '3': case '4':
2875 case '5': case '6': case '7': case '8': case '9':
2879 cur
--; /* Backup character. */
2880 prev_dot
= PREV_TOKEN_TYPE
== CPP_DOT
&& IMMED_TOKEN ();
2883 INIT_TOKEN_NAME (list
, cur_token
);
2884 /* Prepend an immediately previous CPP_DOT token. */
2887 if (list
->name_cap
== list
->name_used
)
2888 auto_expand_name_space (list
);
2890 cur_token
->val
.name
.len
= 1;
2891 list
->namebuf
[list
->name_used
++] = '.';
2895 cur_token
->type
= CPP_NUMBER
; /* Before parse_number. */
2897 parse_number (pfile
, list
, &cur_token
->val
.name
);
2905 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2906 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2907 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2908 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
2910 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2911 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
2912 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
2913 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
2915 cur
--; /* Backup character. */
2916 INIT_TOKEN_NAME (list
, cur_token
);
2917 cur_token
->type
= CPP_NAME
; /* Identifier, macro etc. */
2921 parse_name (pfile
, list
, &cur_token
->val
.name
);
2924 /* Find handler for newly created / extended directive. */
2925 if (IS_DIRECTIVE () && cur_token
== &list
->tokens
[first_token
+ 1])
2926 _cpp_check_directive (list
, cur_token
);
2933 cur_token
->type
= c
== '\'' ? CPP_CHAR
: CPP_STRING
;
2934 /* Do we have a wide string? */
2935 if (cur_token
[-1].type
== CPP_NAME
&& IMMED_TOKEN ()
2936 && cur_token
[-1].val
.name
.len
== 1
2937 && cur_token
[-1].val
.name
.text
[0] == 'L'
2938 && !CPP_TRADITIONAL (pfile
))
2940 /* No need for 'L' any more. */
2942 (--cur_token
)->type
= (c
== '\'' ? CPP_WCHAR
: CPP_WSTRING
);
2946 /* Here c is one of ' " or >. */
2947 INIT_TOKEN_NAME (list
, cur_token
);
2949 parse_string2 (pfile
, list
, &cur_token
->val
.name
, c
,
2950 c
== '"' && !IS_DIRECTIVE());
2956 cur_token
->type
= CPP_DIV
;
2959 if (PREV_TOKEN_TYPE
== CPP_DIV
)
2961 /* We silently allow C++ comments in system headers,
2962 irrespective of conformance mode, because lots of
2963 broken systems do that and trying to clean it up
2964 in fixincludes is a nightmare. */
2965 if (buffer
->system_header_p
)
2966 goto do_line_comment
;
2967 else if (CPP_OPTION (pfile
, cplusplus_comments
))
2969 if (CPP_OPTION (pfile
, c89
) && CPP_PEDANTIC (pfile
)
2970 && ! buffer
->warned_cplusplus_comments
)
2974 "C++ style comments are not allowed in ISO C89");
2976 "(this will be reported only once per input file)");
2977 buffer
->warned_cplusplus_comments
= 1;
2983 "comment start split across lines");
2984 if (skip_line_comment2 (pfile
))
2985 cpp_error_with_line (pfile
, list
->line
,
2987 "multi-line comment");
2989 /* Back-up to first '-' or '/'. */
2991 if (!CPP_OPTION (pfile
, discard_comments
)
2992 && (!IS_DIRECTIVE() || list
->dirno
== 0))
2993 save_comment (list
, cur_token
++, cur
,
2994 buffer
->cur
- cur
, c
);
2997 if (!CPP_OPTION (pfile
, traditional
))
2998 flags
= PREV_WHITESPACE
;
3007 cur_token
->type
= CPP_MULT
;
3010 if (PREV_TOKEN_TYPE
== CPP_DIV
)
3015 "comment start '/*' split across lines");
3016 if (skip_block_comment2 (pfile
))
3017 cpp_error_with_line (pfile
, list
->line
, cur_token
[-1].col
,
3018 "unterminated comment");
3019 else if (buffer
->cur
[-2] != '*')
3021 "comment end '*/' split across lines");
3023 /* Back up to opening '/'. */
3025 if (!CPP_OPTION (pfile
, discard_comments
)
3026 && (!IS_DIRECTIVE() || list
->dirno
== 0))
3027 save_comment (list
, cur_token
++, cur
,
3028 buffer
->cur
- cur
, c
);
3031 if (!CPP_OPTION (pfile
, traditional
))
3032 flags
= PREV_WHITESPACE
;
3035 else if (CPP_OPTION (pfile
, cplusplus
))
3037 /* In C++, there are .* and ->* operators. */
3038 if (PREV_TOKEN_TYPE
== CPP_DEREF
)
3039 BACKUP_TOKEN (CPP_DEREF_STAR
);
3040 else if (PREV_TOKEN_TYPE
== CPP_DOT
)
3041 BACKUP_TOKEN (CPP_DOT_STAR
);
3049 handle_newline (cur
, buffer
->rlimit
, c
);
3050 if (PREV_TOKEN_TYPE
== CPP_BACKSLASH
&& IMMED_TOKEN ())
3052 /* Remove the escaped newline. Then continue to process
3053 any interrupted name or number. */
3058 if (cur_token
->type
== CPP_NAME
)
3060 else if (cur_token
->type
== CPP_NUMBER
)
3061 goto continue_number
;
3064 /* Remember whitespace setting. */
3065 flags
= cur_token
->flags
;
3068 if (PREV_TOKEN_TYPE
== CPP_BACKSLASH
)
3071 cpp_warning (pfile
, "backslash and newline separated by space");
3073 /* Skip vertical space until we have at least one token to
3075 if (cur_token
!= &list
->tokens
[first_token
])
3077 list
->line
= CPP_BUF_LINE (buffer
);
3081 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
== CPP_MINUS
)
3083 if (CPP_OPTION (pfile
, chill
))
3084 goto do_line_comment
;
3085 REVISE_TOKEN (CPP_MINUS_MINUS
);
3088 PUSH_TOKEN (CPP_MINUS
);
3091 /* The digraph flag checking ensures that ## and %:%:
3092 are interpreted as CPP_PASTE, but #%: and %:# are not. */
3095 if (PREV_TOKEN_TYPE
== CPP_HASH
&& IMMED_TOKEN ()
3096 && ((cur_token
->flags
^ cur_token
[-1].flags
) & DIGRAPH
) == 0)
3097 REVISE_TOKEN (CPP_PASTE
);
3099 PUSH_TOKEN (CPP_HASH
);
3103 cur_token
->type
= CPP_COLON
;
3106 if (PREV_TOKEN_TYPE
== CPP_COLON
3107 && CPP_OPTION (pfile
, cplusplus
))
3108 BACKUP_TOKEN (CPP_SCOPE
);
3109 /* Digraph: "<:" is a '[' */
3110 else if (PREV_TOKEN_TYPE
== CPP_LESS
)
3111 BACKUP_DIGRAPH (CPP_OPEN_SQUARE
);
3112 /* Digraph: "%:" is a '#' */
3113 else if (PREV_TOKEN_TYPE
== CPP_MOD
)
3115 (--cur_token
)->flags
|= DIGRAPH
;
3123 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
== CPP_AND
)
3124 REVISE_TOKEN (CPP_AND_AND
);
3126 PUSH_TOKEN (CPP_AND
);
3131 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
== CPP_OR
)
3132 REVISE_TOKEN (CPP_OR_OR
);
3134 PUSH_TOKEN (CPP_OR
);
3138 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
== CPP_PLUS
)
3139 REVISE_TOKEN (CPP_PLUS_PLUS
);
3141 PUSH_TOKEN (CPP_PLUS
);
3145 /* This relies on equidistance of "?=" and "?" tokens. */
3146 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
<= CPP_LAST_EQ
)
3147 REVISE_TOKEN (PREV_TOKEN_TYPE
+ (CPP_EQ_EQ
- CPP_EQ
));
3149 PUSH_TOKEN (CPP_EQ
);
3153 cur_token
->type
= CPP_GREATER
;
3156 if (PREV_TOKEN_TYPE
== CPP_GREATER
)
3157 BACKUP_TOKEN (CPP_RSHIFT
);
3158 else if (PREV_TOKEN_TYPE
== CPP_MINUS
)
3159 BACKUP_TOKEN (CPP_DEREF
);
3160 /* Digraph: ":>" is a ']' */
3161 else if (PREV_TOKEN_TYPE
== CPP_COLON
)
3162 BACKUP_DIGRAPH (CPP_CLOSE_SQUARE
);
3163 /* Digraph: "%>" is a '}' */
3164 else if (PREV_TOKEN_TYPE
== CPP_MOD
)
3165 BACKUP_DIGRAPH (CPP_CLOSE_BRACE
);
3171 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
== CPP_LESS
)
3173 REVISE_TOKEN (CPP_LSHIFT
);
3176 /* Is this the beginning of a header name? */
3177 if (list
->flags
& SYNTAX_INCLUDE
)
3179 c
= '>'; /* Terminator. */
3180 cur_token
->type
= CPP_HEADER_NAME
;
3181 goto do_parse_string
;
3183 PUSH_TOKEN (CPP_LESS
);
3187 /* Digraph: "<%" is a '{' */
3188 cur_token
->type
= CPP_MOD
;
3189 if (IMMED_TOKEN () && PREV_TOKEN_TYPE
== CPP_LESS
)
3190 BACKUP_DIGRAPH (CPP_OPEN_BRACE
);
3195 if (cur
+ 1 < buffer
->rlimit
&& *cur
== '?'
3196 && trigraph_map
[cur
[1]] && trigraph_ok (pfile
, cur
+ 1))
3198 /* Handle trigraph. */
3202 case '(': goto make_open_square
;
3203 case ')': goto make_close_square
;
3204 case '<': goto make_open_brace
;
3205 case '>': goto make_close_brace
;
3206 case '=': goto make_hash
;
3207 case '!': goto make_or
;
3208 case '-': goto make_complement
;
3209 case '/': goto make_backslash
;
3210 case '\'': goto make_xor
;
3213 if (IMMED_TOKEN () && CPP_OPTION (pfile
, cplusplus
))
3215 /* GNU C++ defines <? and >? operators. */
3216 if (PREV_TOKEN_TYPE
== CPP_LESS
)
3218 REVISE_TOKEN (CPP_MIN
);
3221 else if (PREV_TOKEN_TYPE
== CPP_GREATER
)
3223 REVISE_TOKEN (CPP_MAX
);
3227 PUSH_TOKEN (CPP_QUERY
);
3231 if (PREV_TOKEN_TYPE
== CPP_DOT
&& cur_token
[-2].type
== CPP_DOT
3233 && !(cur_token
[-1].flags
& PREV_WHITESPACE
))
3236 PUSH_TOKEN (CPP_ELLIPSIS
);
3239 PUSH_TOKEN (CPP_DOT
);
3243 case '~': PUSH_TOKEN (CPP_COMPL
); break;
3245 case '^': PUSH_TOKEN (CPP_XOR
); break;
3247 case '{': PUSH_TOKEN (CPP_OPEN_BRACE
); break;
3249 case '}': PUSH_TOKEN (CPP_CLOSE_BRACE
); break;
3251 case '[': PUSH_TOKEN (CPP_OPEN_SQUARE
); break;
3253 case ']': PUSH_TOKEN (CPP_CLOSE_SQUARE
); break;
3255 case '\\': PUSH_TOKEN (CPP_BACKSLASH
); break;
3256 case '!': PUSH_TOKEN (CPP_NOT
); break;
3257 case ',': PUSH_TOKEN (CPP_COMMA
); break;
3258 case ';': PUSH_TOKEN (CPP_SEMICOLON
); break;
3259 case '(': PUSH_TOKEN (CPP_OPEN_PAREN
); break;
3260 case ')': PUSH_TOKEN (CPP_CLOSE_PAREN
); break;
3263 if (CPP_OPTION (pfile
, dollars_in_ident
))
3268 cur_token
->val
.name
.len
= 0; /* FIXME: needed for transition only */
3269 PUSH_TOKEN (CPP_OTHER
);
3274 /* Run out of token space? */
3275 if (cur_token
== token_limit
)
3277 list
->tokens_used
= cur_token
- list
->tokens
;
3278 _cpp_expand_token_space (list
, 256);
3282 cur_token
->flags
= flags
;
3283 if (cur_token
== &list
->tokens
[first_token
])
3285 /* FIXME: move this warning to callers who care. */
3286 if (cur
> buffer
->buf
&& !IS_NEWLINE (cur
[-1]))
3287 cpp_warning (pfile
, "no newline at end of file");
3288 cur_token
++->type
= CPP_EOF
;
3292 list
->tokens
[first_token
].flags
|= BOL
;
3294 list
->tokens_used
= cur_token
- list
->tokens
;
3297 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
3298 already contain the enough space to hold the token's spelling. If
3299 WHITESPACE is true, and the token was preceded by whitespace,
3300 output a single space before the token proper. Returns a pointer
3301 to the character after the last character written. */
3303 static unsigned char *
3304 spell_token (pfile
, token
, buffer
, whitespace
)
3305 cpp_reader
*pfile
; /* Would be nice to be rid of this... */
3306 const cpp_token
*token
;
3307 unsigned char *buffer
;
3310 /* Whitespace will not be wanted by handlers of the # and ##
3311 operators calling this function, but will be wanted by the
3312 function that writes out the preprocessed file. */
3313 if (whitespace
&& token
->flags
& PREV_WHITESPACE
)
3316 switch (token_spellings
[token
->type
].type
)
3318 case SPELL_OPERATOR
:
3320 const unsigned char *spelling
;
3323 if (token
->flags
& DIGRAPH
)
3324 spelling
= digraph_spellings
[token
->type
- CPP_FIRST_DIGRAPH
];
3326 spelling
= token_spellings
[token
->type
].spelling
;
3328 while ((c
= *spelling
++) != '\0')
3334 memcpy (buffer
, token
->val
.name
.text
, token
->val
.name
.len
);
3335 buffer
+= token
->val
.name
.len
;
3342 if (token
->type
== CPP_WSTRING
|| token
->type
== CPP_WCHAR
)
3345 if (token
->type
== CPP_STRING
|| token
->type
== CPP_WSTRING
)
3348 memcpy (buffer
, token
->val
.name
.text
, token
->val
.name
.len
);
3349 buffer
+= token
->val
.name
.len
;
3355 *buffer
++ = token
->aux
;
3359 cpp_ice (pfile
, "Unspellable token");
3366 /* Temporary function for illustrative purposes. */
3368 _cpp_lex_file (pfile
)
3373 init_trigraph_map ();
3374 list
= (cpp_toklist
*) xmalloc (sizeof (cpp_toklist
));
3375 _cpp_init_toklist (list
, DUMMY_TOKEN
);
3379 _cpp_lex_line (pfile
, list
);
3380 if (list
->tokens
[0].type
== CPP_EOF
)
3385 _cpp_handle_directive (pfile
, list
);
3388 _cpp_output_list (pfile
, list
);
3389 _cpp_clear_toklist (list
);
3393 /* Temporary function for illustrative purposes. */
3395 _cpp_output_list (pfile
, list
)
3401 for (i
= 0; i
< list
->tokens_used
; i
++)
3403 CPP_RESERVE (pfile
, TOKEN_LEN (&list
->tokens
[i
]));
3404 pfile
->limit
= spell_token (pfile
, &list
->tokens
[i
], pfile
->limit
, 1);