libcpp/internal.h

   1 /* Part of CPP library.
   2    Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2007,
   3    2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
   4
   5 This program is free software; you can redistribute it and/or modify it
   6 under the terms of the GNU General Public License as published by the
   7 Free Software Foundation; either version 3, or (at your option) any
   8 later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program; see the file COPYING3.  If not see
  17 <http://www.gnu.org/licenses/>.  */
  18
  19 /* This header defines all the internal data structures and functions
  20    that need to be visible across files.  It should not be used outside
  21    cpplib.  */
  22
  23 #ifndef LIBCPP_INTERNAL_H
  24 #define LIBCPP_INTERNAL_H
  25
  26 #include "symtab.h"
  27 #include "cpp-id-data.h"
  28
  29 #if HAVE_ICONV
  30 #include <iconv.h>
  31 #else
  32 #define HAVE_ICONV 0
  33 typedef int iconv_t;  /* dummy */
  34 #endif
  35
  36 #ifdef __cplusplus
  37 extern "C" {
  38 #endif
  39
  40 struct directive;               /* Deliberately incomplete.  */
  41 struct pending_option;
  42 struct op;
  43 struct _cpp_strbuf;
  44
  45 typedef bool (*convert_f) (iconv_t, const unsigned char *, size_t,
  46                            struct _cpp_strbuf *);
  47 struct cset_converter
  48 {
  49   convert_f func;
  50   iconv_t cd;
  51   int width;
  52 };
  53
  54 #define BITS_PER_CPPCHAR_T (CHAR_BIT * sizeof (cppchar_t))
  55
  56 /* Test if a sign is valid within a preprocessing number.  */
  57 #define VALID_SIGN(c, prevc) \
  58   (((c) == '+' || (c) == '-') && \
  59    ((prevc) == 'e' || (prevc) == 'E' \
  60     || (((prevc) == 'p' || (prevc) == 'P') \
  61         && CPP_OPTION (pfile, extended_numbers))))
  62
  63 #define CPP_OPTION(PFILE, OPTION) ((PFILE)->opts.OPTION)
  64 #define CPP_BUFFER(PFILE) ((PFILE)->buffer)
  65 #define CPP_BUF_COLUMN(BUF, CUR) ((CUR) - (BUF)->line_base)
  66 #define CPP_BUF_COL(BUF) CPP_BUF_COLUMN(BUF, (BUF)->cur)
  67
  68 #define CPP_INCREMENT_LINE(PFILE, COLS_HINT) do { \
  69     const struct line_maps *line_table = PFILE->line_table; \
  70     const struct line_map *map = \
  71       LINEMAPS_LAST_ORDINARY_MAP (line_table); \
  72     linenum_type line = SOURCE_LINE (map, line_table->highest_line); \
  73     linemap_line_start (PFILE->line_table, line + 1, COLS_HINT); \
  74   } while (0)
  75
  76 /* Maximum nesting of cpp_buffers.  We use a static limit, partly for
  77    efficiency, and partly to limit runaway recursion.  */
  78 #define CPP_STACK_MAX 200
  79
  80 /* Host alignment handling.  */
  81 struct dummy
  82 {
  83   char c;
  84   union
  85   {
  86     double d;
  87     int *p;
  88   } u;
  89 };
  90
  91 #define DEFAULT_ALIGNMENT offsetof (struct dummy, u)
  92 #define CPP_ALIGN2(size, align) (((size) + ((align) - 1)) & ~((align) - 1))
  93 #define CPP_ALIGN(size) CPP_ALIGN2 (size, DEFAULT_ALIGNMENT)
  94
  95 #define _cpp_mark_macro_used(NODE) do {                                 \
  96   if ((NODE)->type == NT_MACRO && !((NODE)->flags & NODE_BUILTIN))      \
  97     (NODE)->value.macro->used = 1; } while (0)
  98
  99 /* A generic memory buffer, and operations on it.  */
 100 typedef struct _cpp_buff _cpp_buff;
 101 struct _cpp_buff
 102 {
 103   struct _cpp_buff *next;
 104   unsigned char *base, *cur, *limit;
 105 };
 106
 107 extern _cpp_buff *_cpp_get_buff (cpp_reader *, size_t);
 108 extern void _cpp_release_buff (cpp_reader *, _cpp_buff *);
 109 extern void _cpp_extend_buff (cpp_reader *, _cpp_buff **, size_t);
 110 extern _cpp_buff *_cpp_append_extend_buff (cpp_reader *, _cpp_buff *, size_t);
 111 extern void _cpp_free_buff (_cpp_buff *);
 112 extern unsigned char *_cpp_aligned_alloc (cpp_reader *, size_t);
 113 extern unsigned char *_cpp_unaligned_alloc (cpp_reader *, size_t);
 114
 115 #define BUFF_ROOM(BUFF) (size_t) ((BUFF)->limit - (BUFF)->cur)
 116 #define BUFF_FRONT(BUFF) ((BUFF)->cur)
 117 #define BUFF_LIMIT(BUFF) ((BUFF)->limit)
 118
 119 /* #include types.  */
 120 enum include_type {IT_INCLUDE, IT_INCLUDE_NEXT, IT_IMPORT, IT_CMDLINE};
 121
 122 union utoken
 123 {
 124   const cpp_token *token;
 125   const cpp_token **ptoken;
 126 };
 127
 128 /* A "run" of tokens; part of a chain of runs.  */
 129 typedef struct tokenrun tokenrun;
 130 struct tokenrun
 131 {
 132   tokenrun *next, *prev;
 133   cpp_token *base, *limit;
 134 };
 135
 136 /* Accessor macros for struct cpp_context.  */
 137 #define FIRST(c) ((c)->u.iso.first)
 138 #define LAST(c) ((c)->u.iso.last)
 139 #define CUR(c) ((c)->u.trad.cur)
 140 #define RLIMIT(c) ((c)->u.trad.rlimit)
 141
 142 /* This describes some additional data that is added to the macro
 143    token context of type cpp_context, when -ftrack-macro-expansion is
 144    on.  */
 145 typedef struct
 146 {
 147   /* The node of the macro we are referring to.  */
 148   cpp_hashnode *macro_node;
 149   /* This buffer contains an array of virtual locations.  The virtual
 150      location at index 0 is the virtual location of the token at index
 151      0 in the current instance of cpp_context; similarly for all the
 152      other virtual locations.  */
 153   source_location *virt_locs;
 154   /* This is a pointer to the current virtual location.  This is used
 155      to iterate over the virtual locations while we iterate over the
 156      tokens they belong to.  */
 157   source_location *cur_virt_loc;
 158 } macro_context;
 159
 160 /* The kind of tokens carried by a cpp_context.  */
 161 enum context_tokens_kind {
 162   /* This is the value of cpp_context::tokens_kind if u.iso.first
 163      contains an instance of cpp_token **.  */
 164   TOKENS_KIND_INDIRECT,
 165   /* This is the value of cpp_context::tokens_kind if u.iso.first
 166      contains an instance of cpp_token *.  */
 167   TOKENS_KIND_DIRECT,
 168   /* This is the value of cpp_context::tokens_kind when the token
 169      context contains tokens resulting from macro expansion.  In that
 170      case struct cpp_context::macro points to an instance of struct
 171      macro_context.  This is used only when the
 172      -ftrack-macro-expansion flag is on.  */
 173   TOKENS_KIND_EXTENDED
 174 };
 175
 176 typedef struct cpp_context cpp_context;
 177 struct cpp_context
 178 {
 179   /* Doubly-linked list.  */
 180   cpp_context *next, *prev;
 181
 182   union
 183   {
 184     /* For ISO macro expansion.  Contexts other than the base context
 185        are contiguous tokens.  e.g. macro expansions, expanded
 186        argument tokens.  */
 187     struct
 188     {
 189       union utoken first;
 190       union utoken last;
 191     } iso;
 192
 193     /* For traditional macro expansion.  */
 194     struct
 195     {
 196       const unsigned char *cur;
 197       const unsigned char *rlimit;
 198     } trad;
 199   } u;
 200
 201   /* If non-NULL, a buffer used for storage related to this context.
 202      When the context is popped, the buffer is released.  */
 203   _cpp_buff *buff;
 204
 205   /* If tokens_kind is TOKEN_KIND_EXTENDED, then (as we thus are in a
 206      macro context) this is a pointer to an instance of macro_context.
 207      Otherwise if tokens_kind is *not* TOKEN_KIND_EXTENDED, then, if
 208      we are in a macro context, this is a pointer to an instance of
 209      cpp_hashnode, representing the name of the macro this context is
 210      for.  If we are not in a macro context, then this is just NULL.
 211      Note that when tokens_kind is TOKEN_KIND_EXTENDED, the memory
 212      used by the instance of macro_context pointed to by this member
 213      is de-allocated upon de-allocation of the instance of struct
 214      cpp_context.  */
 215   union
 216   {
 217     macro_context *mc;
 218     cpp_hashnode *macro;
 219   } c;
 220
 221   /* This determines the type of tokens held by this context.  */
 222   enum context_tokens_kind tokens_kind;
 223 };
 224
 225 struct lexer_state
 226 {
 227   /* Nonzero if first token on line is CPP_HASH.  */
 228   unsigned char in_directive;
 229
 230   /* Nonzero if in a directive that will handle padding tokens itself.
 231      #include needs this to avoid problems with computed include and
 232      spacing between tokens.  */
 233   unsigned char directive_wants_padding;
 234
 235   /* True if we are skipping a failed conditional group.  */
 236   unsigned char skipping;
 237
 238   /* Nonzero if in a directive that takes angle-bracketed headers.  */
 239   unsigned char angled_headers;
 240
 241   /* Nonzero if in a #if or #elif directive.  */
 242   unsigned char in_expression;
 243
 244   /* Nonzero to save comments.  Turned off if discard_comments, and in
 245      all directives apart from #define.  */
 246   unsigned char save_comments;
 247
 248   /* Nonzero if lexing __VA_ARGS__ is valid.  */
 249   unsigned char va_args_ok;
 250
 251   /* Nonzero if lexing poisoned identifiers is valid.  */
 252   unsigned char poisoned_ok;
 253
 254   /* Nonzero to prevent macro expansion.  */
 255   unsigned char prevent_expansion;
 256
 257   /* Nonzero when parsing arguments to a function-like macro.  */
 258   unsigned char parsing_args;
 259
 260   /* Nonzero if prevent_expansion is true only because output is
 261      being discarded.  */
 262   unsigned char discarding_output;
 263
 264   /* Nonzero to skip evaluating part of an expression.  */
 265   unsigned int skip_eval;
 266
 267   /* Nonzero when handling a deferred pragma.  */
 268   unsigned char in_deferred_pragma;
 269
 270   /* Nonzero if the deferred pragma being handled allows macro expansion.  */
 271   unsigned char pragma_allow_expansion;
 272 };
 273
 274 /* Special nodes - identifiers with predefined significance.  */
 275 struct spec_nodes
 276 {
 277   cpp_hashnode *n_defined;              /* defined operator */
 278   cpp_hashnode *n_true;                 /* C++ keyword true */
 279   cpp_hashnode *n_false;                /* C++ keyword false */
 280   cpp_hashnode *n__VA_ARGS__;           /* C99 vararg macros */
 281 };
 282
 283 typedef struct _cpp_line_note _cpp_line_note;
 284 struct _cpp_line_note
 285 {
 286   /* Location in the clean line the note refers to.  */
 287   const unsigned char *pos;
 288
 289   /* Type of note.  The 9 'from' trigraph characters represent those
 290      trigraphs, '\\' an escaped newline, ' ' an escaped newline with
 291      intervening space, 0 represents a note that has already been handled,
 292      and anything else is invalid.  */
 293   unsigned int type;
 294 };
 295
 296 /* Represents the contents of a file cpplib has read in.  */
 297 struct cpp_buffer
 298 {
 299   const unsigned char *cur;        /* Current location.  */
 300   const unsigned char *line_base;  /* Start of current physical line.  */
 301   const unsigned char *next_line;  /* Start of to-be-cleaned logical line.  */
 302
 303   const unsigned char *buf;        /* Entire character buffer.  */
 304   const unsigned char *rlimit;     /* Writable byte at end of file.  */
 305
 306   _cpp_line_note *notes;           /* Array of notes.  */
 307   unsigned int cur_note;           /* Next note to process.  */
 308   unsigned int notes_used;         /* Number of notes.  */
 309   unsigned int notes_cap;          /* Size of allocated array.  */
 310
 311   struct cpp_buffer *prev;
 312
 313   /* Pointer into the file table; non-NULL if this is a file buffer.
 314      Used for include_next and to record control macros.  */
 315   struct _cpp_file *file;
 316
 317   /* Saved value of __TIMESTAMP__ macro - date and time of last modification
 318      of the assotiated file.  */
 319   const unsigned char *timestamp;
 320
 321   /* Value of if_stack at start of this file.
 322      Used to prohibit unmatched #endif (etc) in an include file.  */
 323   struct if_stack *if_stack;
 324
 325   /* True if we need to get the next clean line.  */
 326   bool need_line;
 327
 328   /* True if we have already warned about C++ comments in this file.
 329      The warning happens only for C89 extended mode with -pedantic on,
 330      or for -Wtraditional, and only once per file (otherwise it would
 331      be far too noisy).  */
 332   unsigned int warned_cplusplus_comments : 1;
 333
 334   /* True if we don't process trigraphs and escaped newlines.  True
 335      for preprocessed input, command line directives, and _Pragma
 336      buffers.  */
 337   unsigned int from_stage3 : 1;
 338
 339   /* At EOF, a buffer is automatically popped.  If RETURN_AT_EOF is
 340      true, a CPP_EOF token is then returned.  Otherwise, the next
 341      token from the enclosing buffer is returned.  */
 342   unsigned int return_at_eof : 1;
 343
 344   /* One for a system header, two for a C system header file that therefore
 345      needs to be extern "C" protected in C++, and zero otherwise.  */
 346   unsigned char sysp;
 347
 348   /* The directory of the this buffer's file.  Its NAME member is not
 349      allocated, so we don't need to worry about freeing it.  */
 350   struct cpp_dir dir;
 351
 352   /* Descriptor for converting from the input character set to the
 353      source character set.  */
 354   struct cset_converter input_cset_desc;
 355 };
 356
 357 /* The list of saved macros by push_macro pragma.  */
 358 struct def_pragma_macro {
 359   /* Chain element to previous saved macro.  */
 360   struct def_pragma_macro *next;
 361   /* Name of the macro.  */
 362   char *name;
 363   /* The stored macro content.  */
 364   unsigned char *definition;
 365
 366   /* Definition line number.  */
 367   source_location line;
 368   /* If macro defined in system header.  */
 369   unsigned int syshdr   : 1;
 370   /* Nonzero if it has been expanded or had its existence tested.  */
 371   unsigned int used     : 1;
 372
 373   /* Mark if we save an undefined macro.  */
 374   unsigned int is_undef : 1;
 375 };
 376
 377 /* A cpp_reader encapsulates the "state" of a pre-processor run.
 378    Applying cpp_get_token repeatedly yields a stream of pre-processor
 379    tokens.  Usually, there is only one cpp_reader object active.  */
 380 struct cpp_reader
 381 {
 382   /* Top of buffer stack.  */
 383   cpp_buffer *buffer;
 384
 385   /* Overlaid buffer (can be different after processing #include).  */
 386   cpp_buffer *overlaid_buffer;
 387
 388   /* Lexer state.  */
 389   struct lexer_state state;
 390
 391   /* Source line tracking.  */
 392   struct line_maps *line_table;
 393
 394   /* The line of the '#' of the current directive.  */
 395   source_location directive_line;
 396
 397   /* Memory buffers.  */
 398   _cpp_buff *a_buff;            /* Aligned permanent storage.  */
 399   _cpp_buff *u_buff;            /* Unaligned permanent storage.  */
 400   _cpp_buff *free_buffs;        /* Free buffer chain.  */
 401
 402   /* Context stack.  */
 403   struct cpp_context base_context;
 404   struct cpp_context *context;
 405
 406   /* If in_directive, the directive if known.  */
 407   const struct directive *directive;
 408
 409   /* Token generated while handling a directive, if any. */
 410   cpp_token directive_result;
 411
 412   /* When expanding a macro at top-level, this is the location of the
 413      macro invocation.  */
 414   source_location invocation_location;
 415
 416   /* Nonzero if we are about to expand a macro.  Note that if we are
 417      really expanding a macro, the function macro_of_context returns
 418      the macro being expanded and this flag is set to false.  Client
 419      code should use the function in_macro_expansion_p to know if we
 420      are either about to expand a macro, or are actually expanding
 421      one.  */
 422   bool about_to_expand_macro_p;
 423
 424   /* Search paths for include files.  */
 425   struct cpp_dir *quote_include;        /* "" */
 426   struct cpp_dir *bracket_include;      /* <> */
 427   struct cpp_dir no_search_path;        /* No path.  */
 428
 429   /* Chain of all hashed _cpp_file instances.  */
 430   struct _cpp_file *all_files;
 431
 432   struct _cpp_file *main_file;
 433
 434   /* File and directory hash table.  */
 435   struct htab *file_hash;
 436   struct htab *dir_hash;
 437   struct file_hash_entry_pool *file_hash_entries;
 438
 439   /* Negative path lookup hash table.  */
 440   struct htab *nonexistent_file_hash;
 441   struct obstack nonexistent_file_ob;
 442
 443   /* Nonzero means don't look for #include "foo" the source-file
 444      directory.  */
 445   bool quote_ignores_source_dir;
 446
 447   /* Nonzero if any file has contained #pragma once or #import has
 448      been used.  */
 449   bool seen_once_only;
 450
 451   /* Multiple include optimization.  */
 452   const cpp_hashnode *mi_cmacro;
 453   const cpp_hashnode *mi_ind_cmacro;
 454   bool mi_valid;
 455
 456   /* Lexing.  */
 457   cpp_token *cur_token;
 458   tokenrun base_run, *cur_run;
 459   unsigned int lookaheads;
 460
 461   /* Nonzero prevents the lexer from re-using the token runs.  */
 462   unsigned int keep_tokens;
 463
 464   /* Buffer to hold macro definition string.  */
 465   unsigned char *macro_buffer;
 466   unsigned int macro_buffer_len;
 467
 468   /* Descriptor for converting from the source character set to the
 469      execution character set.  */
 470   struct cset_converter narrow_cset_desc;
 471
 472   /* Descriptor for converting from the source character set to the
 473      UTF-8 execution character set.  */
 474   struct cset_converter utf8_cset_desc;
 475
 476   /* Descriptor for converting from the source character set to the
 477      UTF-16 execution character set.  */
 478   struct cset_converter char16_cset_desc;
 479
 480   /* Descriptor for converting from the source character set to the
 481      UTF-32 execution character set.  */
 482   struct cset_converter char32_cset_desc;
 483
 484   /* Descriptor for converting from the source character set to the
 485      wide execution character set.  */
 486   struct cset_converter wide_cset_desc;
 487
 488   /* Date and time text.  Calculated together if either is requested.  */
 489   const unsigned char *date;
 490   const unsigned char *time;
 491
 492   /* EOF token, and a token forcing paste avoidance.  */
 493   cpp_token avoid_paste;
 494   cpp_token eof;
 495
 496   /* Opaque handle to the dependencies of mkdeps.c.  */
 497   struct deps *deps;
 498
 499   /* Obstack holding all macro hash nodes.  This never shrinks.
 500      See identifiers.c */
 501   struct obstack hash_ob;
 502
 503   /* Obstack holding buffer and conditional structures.  This is a
 504      real stack.  See directives.c.  */
 505   struct obstack buffer_ob;
 506
 507   /* Pragma table - dynamic, because a library user can add to the
 508      list of recognized pragmas.  */
 509   struct pragma_entry *pragmas;
 510
 511   /* Call backs to cpplib client.  */
 512   struct cpp_callbacks cb;
 513
 514   /* Identifier hash table.  */
 515   struct ht *hash_table;
 516
 517   /* Expression parser stack.  */
 518   struct op *op_stack, *op_limit;
 519
 520   /* User visible options.  */
 521   struct cpp_options opts;
 522
 523   /* Special nodes - identifiers with predefined significance to the
 524      preprocessor.  */
 525   struct spec_nodes spec_nodes;
 526
 527   /* Whether cpplib owns the hashtable.  */
 528   bool our_hashtable;
 529
 530   /* Traditional preprocessing output buffer (a logical line).  */
 531   struct
 532   {
 533     unsigned char *base;
 534     unsigned char *limit;
 535     unsigned char *cur;
 536     source_location first_line;
 537   } out;
 538
 539   /* Used for buffer overlays by traditional.c.  */
 540   const unsigned char *saved_cur, *saved_rlimit, *saved_line_base;
 541
 542   /* A saved list of the defined macros, for dependency checking
 543      of precompiled headers.  */
 544   struct cpp_savedstate *savedstate;
 545
 546   /* Next value of __COUNTER__ macro. */
 547   unsigned int counter;
 548
 549   /* Table of comments, when state.save_comments is true.  */
 550   cpp_comment_table comments;
 551
 552   /* List of saved macros by push_macro.  */
 553   struct def_pragma_macro *pushed_macros;
 554
 555   /* If non-null, the lexer will use this location for the next token
 556      instead of getting a location from the linemap.  */
 557   source_location *forced_token_location_p;
 558 };
 559
 560 /* Character classes.  Based on the more primitive macros in safe-ctype.h.
 561    If the definition of `numchar' looks odd to you, please look up the
 562    definition of a pp-number in the C standard [section 6.4.8 of C99].
 563
 564    In the unlikely event that characters other than \r and \n enter
 565    the set is_vspace, the macro handle_newline() in lex.c must be
 566    updated.  */
 567 #define _dollar_ok(x)   ((x) == '$' && CPP_OPTION (pfile, dollars_in_ident))
 568
 569 #define is_idchar(x)    (ISIDNUM(x) || _dollar_ok(x))
 570 #define is_numchar(x)   ISIDNUM(x)
 571 #define is_idstart(x)   (ISIDST(x) || _dollar_ok(x))
 572 #define is_numstart(x)  ISDIGIT(x)
 573 #define is_hspace(x)    ISBLANK(x)
 574 #define is_vspace(x)    IS_VSPACE(x)
 575 #define is_nvspace(x)   IS_NVSPACE(x)
 576 #define is_space(x)     IS_SPACE_OR_NUL(x)
 577
 578 /* This table is constant if it can be initialized at compile time,
 579    which is the case if cpp was compiled with GCC >=2.7, or another
 580    compiler that supports C99.  */
 581 #if HAVE_DESIGNATED_INITIALIZERS
 582 extern const unsigned char _cpp_trigraph_map[UCHAR_MAX + 1];
 583 #else
 584 extern unsigned char _cpp_trigraph_map[UCHAR_MAX + 1];
 585 #endif
 586
 587 /* Macros.  */
 588
 589 static inline int cpp_in_system_header (cpp_reader *);
 590 static inline int
 591 cpp_in_system_header (cpp_reader *pfile)
 592 {
 593   return pfile->buffer ? pfile->buffer->sysp : 0;
 594 }
 595 #define CPP_PEDANTIC(PF) CPP_OPTION (PF, cpp_pedantic)
 596 #define CPP_WTRADITIONAL(PF) CPP_OPTION (PF, cpp_warn_traditional)
 597
 598 static inline int cpp_in_primary_file (cpp_reader *);
 599 static inline int
 600 cpp_in_primary_file (cpp_reader *pfile)
 601 {
 602   return pfile->line_table->depth == 1;
 603 }
 604
 605 /* In macro.c */
 606 extern void _cpp_free_definition (cpp_hashnode *);
 607 extern bool _cpp_create_definition (cpp_reader *, cpp_hashnode *);
 608 extern void _cpp_pop_context (cpp_reader *);
 609 extern void _cpp_push_text_context (cpp_reader *, cpp_hashnode *,
 610                                     const unsigned char *, size_t);
 611 extern bool _cpp_save_parameter (cpp_reader *, cpp_macro *, cpp_hashnode *);
 612 extern bool _cpp_arguments_ok (cpp_reader *, cpp_macro *, const cpp_hashnode *,
 613                                unsigned int);
 614 extern const unsigned char *_cpp_builtin_macro_text (cpp_reader *,
 615                                                      cpp_hashnode *);
 616 extern int _cpp_warn_if_unused_macro (cpp_reader *, cpp_hashnode *, void *);
 617 extern void _cpp_push_token_context (cpp_reader *, cpp_hashnode *,
 618                                      const cpp_token *, unsigned int);
 619 extern void _cpp_backup_tokens_direct (cpp_reader *, unsigned int);
 620
 621 /* In identifiers.c */
 622 extern void _cpp_init_hashtable (cpp_reader *, hash_table *);
 623 extern void _cpp_destroy_hashtable (cpp_reader *);
 624
 625 /* In files.c */
 626 typedef struct _cpp_file _cpp_file;
 627 extern _cpp_file *_cpp_find_file (cpp_reader *, const char *, cpp_dir *,
 628                                   bool, int);
 629 extern bool _cpp_find_failed (_cpp_file *);
 630 extern void _cpp_mark_file_once_only (cpp_reader *, struct _cpp_file *);
 631 extern void _cpp_fake_include (cpp_reader *, const char *);
 632 extern bool _cpp_stack_file (cpp_reader *, _cpp_file*, bool);
 633 extern bool _cpp_stack_include (cpp_reader *, const char *, int,
 634                                 enum include_type);
 635 extern int _cpp_compare_file_date (cpp_reader *, const char *, int);
 636 extern void _cpp_report_missing_guards (cpp_reader *);
 637 extern void _cpp_init_files (cpp_reader *);
 638 extern void _cpp_cleanup_files (cpp_reader *);
 639 extern void _cpp_pop_file_buffer (cpp_reader *, struct _cpp_file *);
 640 extern bool _cpp_save_file_entries (cpp_reader *pfile, FILE *f);
 641 extern bool _cpp_read_file_entries (cpp_reader *, FILE *);
 642 extern const char *_cpp_get_file_name (_cpp_file *);
 643 extern struct stat *_cpp_get_file_stat (_cpp_file *);
 644
 645 /* In expr.c */
 646 extern bool _cpp_parse_expr (cpp_reader *, bool);
 647 extern struct op *_cpp_expand_op_stack (cpp_reader *);
 648
 649 /* In lex.c */
 650 extern void _cpp_process_line_notes (cpp_reader *, int);
 651 extern void _cpp_clean_line (cpp_reader *);
 652 extern bool _cpp_get_fresh_line (cpp_reader *);
 653 extern bool _cpp_skip_block_comment (cpp_reader *);
 654 extern cpp_token *_cpp_temp_token (cpp_reader *);
 655 extern const cpp_token *_cpp_lex_token (cpp_reader *);
 656 extern cpp_token *_cpp_lex_direct (cpp_reader *);
 657 extern int _cpp_equiv_tokens (const cpp_token *, const cpp_token *);
 658 extern void _cpp_init_tokenrun (tokenrun *, unsigned int);
 659 extern cpp_hashnode *_cpp_lex_identifier (cpp_reader *, const char *);
 660 extern int _cpp_remaining_tokens_num_in_context (cpp_context *);
 661 extern void _cpp_init_lexer (void);
 662
 663 /* In init.c.  */
 664 extern void _cpp_maybe_push_include_file (cpp_reader *);
 665 extern const char *cpp_named_operator2name (enum cpp_ttype type);
 666
 667 /* In directives.c */
 668 extern int _cpp_test_assertion (cpp_reader *, unsigned int *);
 669 extern int _cpp_handle_directive (cpp_reader *, int);
 670 extern void _cpp_define_builtin (cpp_reader *, const char *);
 671 extern char ** _cpp_save_pragma_names (cpp_reader *);
 672 extern void _cpp_restore_pragma_names (cpp_reader *, char **);
 673 extern int _cpp_do__Pragma (cpp_reader *);
 674 extern void _cpp_init_directives (cpp_reader *);
 675 extern void _cpp_init_internal_pragmas (cpp_reader *);
 676 extern void _cpp_do_file_change (cpp_reader *, enum lc_reason, const char *,
 677                                  linenum_type, unsigned int);
 678 extern void _cpp_pop_buffer (cpp_reader *);
 679
 680 /* In directives.c */
 681 struct _cpp_dir_only_callbacks
 682 {
 683   /* Called to print a block of lines. */
 684   void (*print_lines) (int, const void *, size_t);
 685   void (*maybe_print_line) (source_location);
 686 };
 687
 688 extern void _cpp_preprocess_dir_only (cpp_reader *,
 689                                       const struct _cpp_dir_only_callbacks *);
 690
 691 /* In traditional.c.  */
 692 extern bool _cpp_scan_out_logical_line (cpp_reader *, cpp_macro *);
 693 extern bool _cpp_read_logical_line_trad (cpp_reader *);
 694 extern void _cpp_overlay_buffer (cpp_reader *pfile, const unsigned char *,
 695                                  size_t);
 696 extern void _cpp_remove_overlay (cpp_reader *);
 697 extern bool _cpp_create_trad_definition (cpp_reader *, cpp_macro *);
 698 extern bool _cpp_expansions_different_trad (const cpp_macro *,
 699                                             const cpp_macro *);
 700 extern unsigned char *_cpp_copy_replacement_text (const cpp_macro *,
 701                                                   unsigned char *);
 702 extern size_t _cpp_replacement_text_len (const cpp_macro *);
 703
 704 /* In charset.c.  */
 705
 706 /* The normalization state at this point in the sequence.
 707    It starts initialized to all zeros, and at the end
 708    'level' is the normalization level of the sequence.  */
 709
 710 struct normalize_state
 711 {
 712   /* The previous character.  */
 713   cppchar_t previous;
 714   /* The combining class of the previous character.  */
 715   unsigned char prev_class;
 716   /* The lowest normalization level so far.  */
 717   enum cpp_normalize_level level;
 718 };
 719 #define INITIAL_NORMALIZE_STATE { 0, 0, normalized_KC }
 720 #define NORMALIZE_STATE_RESULT(st) ((st)->level)
 721
 722 /* We saw a character that matches ISIDNUM(), update a
 723    normalize_state appropriately.  */
 724 #define NORMALIZE_STATE_UPDATE_IDNUM(st) \
 725   ((st)->previous = 0, (st)->prev_class = 0)
 726
 727 extern cppchar_t _cpp_valid_ucn (cpp_reader *, const unsigned char **,
 728                                  const unsigned char *, int,
 729                                  struct normalize_state *state);
 730 extern void _cpp_destroy_iconv (cpp_reader *);
 731 extern unsigned char *_cpp_convert_input (cpp_reader *, const char *,
 732                                           unsigned char *, size_t, size_t,
 733                                           const unsigned char **, off_t *);
 734 extern const char *_cpp_default_encoding (void);
 735 extern cpp_hashnode * _cpp_interpret_identifier (cpp_reader *pfile,
 736                                                  const unsigned char *id,
 737                                                  size_t len);
 738
 739 /* Utility routines and macros.  */
 740 #define DSC(str) (const unsigned char *)str, sizeof str - 1
 741
 742 /* These are inline functions instead of macros so we can get type
 743    checking.  */
 744 static inline int ustrcmp (const unsigned char *, const unsigned char *);
 745 static inline int ustrncmp (const unsigned char *, const unsigned char *,
 746                             size_t);
 747 static inline size_t ustrlen (const unsigned char *);
 748 static inline const unsigned char *uxstrdup (const unsigned char *);
 749 static inline const unsigned char *ustrchr (const unsigned char *, int);
 750 static inline int ufputs (const unsigned char *, FILE *);
 751
 752 /* Use a const char for the second parameter since it is usually a literal.  */
 753 static inline int ustrcspn (const unsigned char *, const char *);
 754
 755 static inline int
 756 ustrcmp (const unsigned char *s1, const unsigned char *s2)
 757 {
 758   return strcmp ((const char *)s1, (const char *)s2);
 759 }
 760
 761 static inline int
 762 ustrncmp (const unsigned char *s1, const unsigned char *s2, size_t n)
 763 {
 764   return strncmp ((const char *)s1, (const char *)s2, n);
 765 }
 766
 767 static inline int
 768 ustrcspn (const unsigned char *s1, const char *s2)
 769 {
 770   return strcspn ((const char *)s1, s2);
 771 }
 772
 773 static inline size_t
 774 ustrlen (const unsigned char *s1)
 775 {
 776   return strlen ((const char *)s1);
 777 }
 778
 779 static inline const unsigned char *
 780 uxstrdup (const unsigned char *s1)
 781 {
 782   return (const unsigned char *) xstrdup ((const char *)s1);
 783 }
 784
 785 static inline const unsigned char *
 786 ustrchr (const unsigned char *s1, int c)
 787 {
 788   return (const unsigned char *) strchr ((const char *)s1, c);
 789 }
 790
 791 static inline int
 792 ufputs (const unsigned char *s, FILE *f)
 793 {
 794   return fputs ((const char *)s, f);
 795 }
 796
 797   /* In line-map.c.  */
 798
 799 /* Create a macro map.  A macro map encodes source locations of tokens
 800    that are part of a macro replacement-list, at a macro expansion
 801    point. See the extensive comments of struct line_map and struct
 802    line_map_macro, in line-map.h.
 803
 804    This map shall be created when the macro is expanded. The map
 805    encodes the source location of the expansion point of the macro as
 806    well as the "original" source location of each token that is part
 807    of the macro replacement-list. If a macro is defined but never
 808    expanded, it has no macro map.  SET is the set of maps the macro
 809    map should be part of.  MACRO_NODE is the macro which the new macro
 810    map should encode source locations for.  EXPANSION is the location
 811    of the expansion point of MACRO. For function-like macros
 812    invocations, it's best to make it point to the closing parenthesis
 813    of the macro, rather than the the location of the first character
 814    of the macro.  NUM_TOKENS is the number of tokens that are part of
 815    the replacement-list of MACRO.  */
 816 const struct line_map *linemap_enter_macro (struct line_maps *,
 817                                             struct cpp_hashnode*,
 818                                             source_location,
 819                                             unsigned int);
 820
 821 /* Create and return a virtual location for a token that is part of a
 822    macro expansion-list at a macro expansion point.  See the comment
 823    inside struct line_map_macro to see what an expansion-list exactly
 824    is.
 825
 826    A call to this function must come after a call to
 827    linemap_enter_macro.
 828
 829    MAP is the map into which the source location is created.  TOKEN_NO
 830    is the index of the token in the macro replacement-list, starting
 831    at number 0.
 832
 833    ORIG_LOC is the location of the token outside of this macro
 834    expansion.  If the token comes originally from the macro
 835    definition, it is the locus in the macro definition; otherwise it
 836    is a location in the context of the caller of this macro expansion
 837    (which is a virtual location or a source location if the caller is
 838    itself a macro expansion or not).
 839
 840    MACRO_DEFINITION_LOC is the location in the macro definition,
 841    either of the token itself or of a macro parameter that it
 842    replaces.  */
 843 source_location linemap_add_macro_token (const struct line_map *,
 844                                          unsigned int,
 845                                          source_location,
 846                                          source_location);
 847
 848 /* Return the source line number corresponding to source location
 849    LOCATION.  SET is the line map set LOCATION comes from.  If
 850    LOCATION is the location of token that is part of the
 851    expansion-list of a macro expansion return the line number of the
 852    macro expansion point.  */
 853 int linemap_get_expansion_line (struct line_maps *,
 854                                 source_location);
 855
 856 /* Return the path of the file corresponding to source code location
 857    LOCATION.
 858
 859    If LOCATION is the location of a token that is part of the
 860    replacement-list of a macro expansion return the file path of the
 861    macro expansion point.
 862
 863    SET is the line map set LOCATION comes from.  */
 864 const char* linemap_get_expansion_filename (struct line_maps *,
 865                                             source_location);
 866
 867 #ifdef __cplusplus
 868 }
 869 #endif
 870
 871 #endif /* ! LIBCPP_INTERNAL_H */