/* CPP Library - lexical analysis.
- Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2007, 2008, 2009, 2010,
- 2011 Free Software Foundation, Inc.
+ Copyright (C) 2000-2016 Free Software Foundation, Inc.
Contributed by Per Bothner, 1994-95.
Based on CCCP program by Paul Rubin, June 1986
Adapted to ANSI C, Richard Stallman, Jan 1987
}
}
-/* Disable on Solaris 2/x86 until the following problems can be properly
+/* Disable on Solaris 2/x86 until the following problem can be properly
autoconfed:
- The Solaris 8 assembler cannot assemble SSE2/SSE4.2 insns.
- The Solaris 9 assembler cannot assemble SSE4.2 insns.
- Before Solaris 9 Update 6, SSE insns cannot be executed.
The Solaris 10+ assembler tags objects with the instruction set
extensions used, so SSE4.2 executables cannot run on machines that
don't support that extension. */
-#if (GCC_VERSION >= 4005) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
+#if (GCC_VERSION >= 4005) && (__GNUC__ >= 5 || !defined(__PIC__)) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
/* Replicated character data to be shared between implementations.
Recall that outside of a context with vector support we can't
/* Check for unaligned input. */
if (si & 15)
{
+ v16qi sv;
+
if (__builtin_expect (end - s < 16, 0)
&& __builtin_expect ((si & 0xfff) > 0xff0, 0))
{
/* ??? The builtin doesn't understand that the PCMPESTRI read from
memory need not be aligned. */
- __asm ("%vpcmpestri $0, (%1), %2"
- : "=c"(index) : "r"(s), "x"(search), "a"(4), "d"(16));
+ sv = __builtin_ia32_loaddqu ((const char *) s);
+ index = __builtin_ia32_pcmpestri128 (search, 4, sv, 16, 0);
+
if (__builtin_expect (index < 16, 0))
goto found;
/* Advance the pointer to an aligned address. We will re-scan a
few bytes, but we no longer need care for reading past the
end of a page, since we're guaranteed a match. */
- s = (const uchar *)((si + 16) & -16);
+ s = (const uchar *)((si + 15) & -16);
}
- /* Main loop, processing 16 bytes at a time. By doing the whole loop
- in inline assembly, we can make proper use of the flags set. */
- __asm ( "sub $16, %1\n"
- " .balign 16\n"
+ /* Main loop, processing 16 bytes at a time. */
+#ifdef __GCC_ASM_FLAG_OUTPUTS__
+ while (1)
+ {
+ char f;
+
+ /* By using inline assembly instead of the builtin,
+ we can use the result, as well as the flags set. */
+ __asm ("%vpcmpestri\t$0, %2, %3"
+ : "=c"(index), "=@ccc"(f)
+ : "m"(*s), "x"(search), "a"(4), "d"(16));
+ if (f)
+ break;
+
+ s += 16;
+ }
+#else
+ s -= 16;
+ /* By doing the whole loop in inline assembly,
+ we can make proper use of the flags set. */
+ __asm ( ".balign 16\n"
"0: add $16, %1\n"
- " %vpcmpestri $0, (%1), %2\n"
+ " %vpcmpestri\t$0, (%1), %2\n"
" jnc 0b"
: "=&c"(index), "+r"(s)
: "x"(search), "a"(4), "d"(16));
+#endif
found:
return s + index;
typedef const uchar * (*search_line_fast_type) (const uchar *, const uchar *);
static search_line_fast_type search_line_fast;
-static void __attribute__((constructor))
+#define HAVE_init_vectorized_lexer 1
+static inline void
init_vectorized_lexer (void)
{
unsigned dummy, ecx = 0, edx = 0;
search_line_fast = impl;
}
-#elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__)
+#elif defined(_ARCH_PWR8) && defined(__ALTIVEC__)
+
+/* A vection of the fast scanner using AltiVec vectorized byte compares
+ and VSX unaligned loads (when VSX is available). This is otherwise
+ the same as the pre-GCC 5 version. */
+
+ATTRIBUTE_NO_SANITIZE_UNDEFINED
+static const uchar *
+search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
+{
+ typedef __attribute__((altivec(vector))) unsigned char vc;
+
+ const vc repl_nl = {
+ '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
+ '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
+ };
+ const vc repl_cr = {
+ '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
+ '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
+ };
+ const vc repl_bs = {
+ '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
+ '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
+ };
+ const vc repl_qm = {
+ '?', '?', '?', '?', '?', '?', '?', '?',
+ '?', '?', '?', '?', '?', '?', '?', '?',
+ };
+ const vc zero = { 0 };
+
+ vc data, t;
+
+ /* Main loop processing 16 bytes at a time. */
+ do
+ {
+ vc m_nl, m_cr, m_bs, m_qm;
+
+ data = *((const vc *)s);
+ s += 16;
+
+ m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl);
+ m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr);
+ m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs);
+ m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm);
+ t = (m_nl | m_cr) | (m_bs | m_qm);
+
+ /* T now contains 0xff in bytes for which we matched one of the relevant
+ characters. We want to exit the loop if any byte in T is non-zero.
+ Below is the expansion of vec_any_ne(t, zero). */
+ }
+ while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero));
+
+ /* Restore s to to point to the 16 bytes we just processed. */
+ s -= 16;
+
+ {
+#define N (sizeof(vc) / sizeof(long))
+
+ union {
+ vc v;
+ /* Statically assert that N is 2 or 4. */
+ unsigned long l[(N == 2 || N == 4) ? N : -1];
+ } u;
+ unsigned long l, i = 0;
+
+ u.v = t;
+
+ /* Find the first word of T that is non-zero. */
+ switch (N)
+ {
+ case 4:
+ l = u.l[i++];
+ if (l != 0)
+ break;
+ s += sizeof(unsigned long);
+ l = u.l[i++];
+ if (l != 0)
+ break;
+ s += sizeof(unsigned long);
+ case 2:
+ l = u.l[i++];
+ if (l != 0)
+ break;
+ s += sizeof(unsigned long);
+ l = u.l[i];
+ }
+
+ /* L now contains 0xff in bytes for which we matched one of the
+ relevant characters. We can find the byte index by finding
+ its bit index and dividing by 8. */
+#ifdef __BIG_ENDIAN__
+ l = __builtin_clzl(l) >> 3;
+#else
+ l = __builtin_ctzl(l) >> 3;
+#endif
+ return s + l;
+
+#undef N
+ }
+}
+
+#elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__) && defined (__BIG_ENDIAN__)
-/* A vection of the fast scanner using AltiVec vectorized byte compares. */
+/* A vection of the fast scanner using AltiVec vectorized byte compares.
+ This cannot be used for little endian because vec_lvsl/lvsr are
+ deprecated for little endian and the code won't work properly. */
/* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
so we can't compile this function without -maltivec on the command line
(or implied by some other switch). */
{
#define N (sizeof(vc) / sizeof(long))
- typedef char check_count[(N == 2 || N == 4) * 2 - 1];
union {
vc v;
- unsigned long l[N];
+ /* Statically assert that N is 2 or 4. */
+ unsigned long l[(N == 2 || N == 4) ? N : -1];
} u;
unsigned long l, i = 0;
}
}
+#elif defined (__ARM_NEON)
+#include "arm_neon.h"
+
+static const uchar *
+search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
+{
+ const uint8x16_t repl_nl = vdupq_n_u8 ('\n');
+ const uint8x16_t repl_cr = vdupq_n_u8 ('\r');
+ const uint8x16_t repl_bs = vdupq_n_u8 ('\\');
+ const uint8x16_t repl_qm = vdupq_n_u8 ('?');
+ const uint8x16_t xmask = (uint8x16_t) vdupq_n_u64 (0x8040201008040201ULL);
+
+ unsigned int misalign, found, mask;
+ const uint8_t *p;
+ uint8x16_t data;
+
+ /* Align the source pointer. */
+ misalign = (uintptr_t)s & 15;
+ p = (const uint8_t *)((uintptr_t)s & -16);
+ data = vld1q_u8 (p);
+
+ /* Create a mask for the bytes that are valid within the first
+ 16-byte block. The Idea here is that the AND with the mask
+ within the loop is "free", since we need some AND or TEST
+ insn in order to set the flags for the branch anyway. */
+ mask = (-1u << misalign) & 0xffff;
+
+ /* Main loop, processing 16 bytes at a time. */
+ goto start;
+
+ do
+ {
+ uint8x8_t l;
+ uint16x4_t m;
+ uint32x2_t n;
+ uint8x16_t t, u, v, w;
+
+ p += 16;
+ data = vld1q_u8 (p);
+ mask = 0xffff;
+
+ start:
+ t = vceqq_u8 (data, repl_nl);
+ u = vceqq_u8 (data, repl_cr);
+ v = vorrq_u8 (t, vceqq_u8 (data, repl_bs));
+ w = vorrq_u8 (u, vceqq_u8 (data, repl_qm));
+ t = vandq_u8 (vorrq_u8 (v, w), xmask);
+ l = vpadd_u8 (vget_low_u8 (t), vget_high_u8 (t));
+ m = vpaddl_u8 (l);
+ n = vpaddl_u16 (m);
+
+ found = vget_lane_u32 ((uint32x2_t) vorr_u64 ((uint64x1_t) n,
+ vshr_n_u64 ((uint64x1_t) n, 24)), 0);
+ found &= mask;
+ }
+ while (!found);
+
+ /* FOUND contains 1 in bits for which we matched a relevant
+ character. Conversion to the byte index is trivial. */
+ found = __builtin_ctz (found);
+ return (const uchar *)p + found;
+}
+
#else
/* We only have one accellerated alternative. Use a direct call so that
#endif
+/* Initialize the lexer if needed. */
+
+void
+_cpp_init_lexer (void)
+{
+#ifdef HAVE_init_vectorized_lexer
+ init_vectorized_lexer ();
+#endif
+}
+
/* Returns with a logical line that contains no escaped newlines or
trigraphs. This is a time-critical inner loop. */
void
else
cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
"`%.*s' is not in NFC", (int) sz, buf);
+ free (buf);
}
}
&& *buffer->cur == '\\'
&& (buffer->cur[1] == 'u' || buffer->cur[1] == 'U'))
{
+ cppchar_t s;
buffer->cur += 2;
if (_cpp_valid_ucn (pfile, &buffer->cur, buffer->rlimit, 1 + !first,
- state))
+ state, &s))
return true;
buffer->cur -= 2;
}
replacement list of a variadic macro. */
if (result == pfile->spec_nodes.n__VA_ARGS__
&& !pfile->state.va_args_ok)
- cpp_error (pfile, CPP_DL_PEDWARN,
- "__VA_ARGS__ can only appear in the expansion"
- " of a C99 variadic macro");
+ {
+ if (CPP_OPTION (pfile, cplusplus))
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "__VA_ARGS__ can only appear in the expansion"
+ " of a C++11 variadic macro");
+ else
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "__VA_ARGS__ can only appear in the expansion"
+ " of a C99 variadic macro");
+ }
/* For -Wc++-compat, warn about use of C++ named operators. */
if (result->flags & NODE_WARN_OPERATOR)
/* Lex an identifier starting at BUFFER->CUR - 1. */
static cpp_hashnode *
lex_identifier (cpp_reader *pfile, const uchar *base, bool starts_ucn,
- struct normalize_state *nst)
+ struct normalize_state *nst, cpp_hashnode **spelling)
{
cpp_hashnode *result;
const uchar *cur;
cur = pfile->buffer->cur;
if (! starts_ucn)
- while (ISIDNUM (*cur))
- {
- hash = HT_HASHSTEP (hash, *cur);
- cur++;
- }
+ {
+ while (ISIDNUM (*cur))
+ {
+ hash = HT_HASHSTEP (hash, *cur);
+ cur++;
+ }
+ NORMALIZE_STATE_UPDATE_IDNUM (nst, *(cur - 1));
+ }
pfile->buffer->cur = cur;
if (starts_ucn || forms_identifier_p (pfile, false, nst))
{
do {
while (ISIDNUM (*pfile->buffer->cur))
{
+ NORMALIZE_STATE_UPDATE_IDNUM (nst, *pfile->buffer->cur);
pfile->buffer->cur++;
- NORMALIZE_STATE_UPDATE_IDNUM (nst);
}
} while (forms_identifier_p (pfile, false, nst));
result = _cpp_interpret_identifier (pfile, base,
pfile->buffer->cur - base);
+ *spelling = cpp_lookup (pfile, base, pfile->buffer->cur - base);
}
else
{
result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
base, len, hash, HT_ALLOC));
+ *spelling = result;
}
/* Rarely, identifiers require diagnostics when lexed. */
replacement list of a variadic macro. */
if (result == pfile->spec_nodes.n__VA_ARGS__
&& !pfile->state.va_args_ok)
- cpp_error (pfile, CPP_DL_PEDWARN,
- "__VA_ARGS__ can only appear in the expansion"
- " of a C99 variadic macro");
+ {
+ if (CPP_OPTION (pfile, cplusplus))
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "__VA_ARGS__ can only appear in the expansion"
+ " of a C++11 variadic macro");
+ else
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "__VA_ARGS__ can only appear in the expansion"
+ " of a C99 variadic macro");
+ }
/* For -Wc++-compat, warn about use of C++ named operators. */
if (result->flags & NODE_WARN_OPERATOR)
cur = pfile->buffer->cur;
/* N.B. ISIDNUM does not include $. */
- while (ISIDNUM (*cur) || *cur == '.' || VALID_SIGN (*cur, cur[-1]))
+ while (ISIDNUM (*cur) || *cur == '.' || DIGIT_SEP (*cur)
+ || VALID_SIGN (*cur, cur[-1]))
{
+ NORMALIZE_STATE_UPDATE_IDNUM (nst, *cur);
cur++;
- NORMALIZE_STATE_UPDATE_IDNUM (nst);
}
+ /* A number can't end with a digit separator. */
+ while (cur > pfile->buffer->cur && DIGIT_SEP (cur[-1]))
+ --cur;
pfile->buffer->cur = cur;
}
*last_buff_p = last_buff;
}
+
+/* Returns true if a macro has been defined.
+ This might not work if compile with -save-temps,
+ or preprocess separately from compilation. */
+
+static bool
+is_macro(cpp_reader *pfile, const uchar *base)
+{
+ const uchar *cur = base;
+ if (! ISIDST (*cur))
+ return false;
+ unsigned int hash = HT_HASHSTEP (0, *cur);
+ ++cur;
+ while (ISIDNUM (*cur))
+ {
+ hash = HT_HASHSTEP (hash, *cur);
+ ++cur;
+ }
+ hash = HT_HASHFINISH (hash, cur - base);
+
+ cpp_hashnode *result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
+ base, cur - base, hash, HT_NO_INSERT));
+
+ return !result ? false : (result->type == NT_MACRO);
+}
+
+
/* Lexes a raw string. The stored string contains the spelling, including
double quotes, delimiter string, '(' and ')', any leading
'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
lex_raw_string (cpp_reader *pfile, cpp_token *token, const uchar *base,
const uchar *cur)
{
- const uchar *raw_prefix;
- unsigned int raw_prefix_len = 0;
+ uchar raw_prefix[17];
+ uchar temp_buffer[18];
+ const uchar *orig_base;
+ unsigned int raw_prefix_len = 0, raw_suffix_len = 0;
+ enum raw_str_phase { RAW_STR_PREFIX, RAW_STR, RAW_STR_SUFFIX };
+ raw_str_phase phase = RAW_STR_PREFIX;
enum cpp_ttype type;
size_t total_len = 0;
+ /* Index into temp_buffer during phases other than RAW_STR,
+ during RAW_STR phase 17 to tell BUF_APPEND that nothing should
+ be appended to temp_buffer. */
+ size_t temp_buffer_len = 0;
_cpp_buff *first_buff = NULL, *last_buff = NULL;
+ size_t raw_prefix_start;
_cpp_line_note *note = &pfile->buffer->notes[pfile->buffer->cur_note];
type = (*base == 'L' ? CPP_WSTRING :
*base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
: CPP_STRING);
- raw_prefix = cur + 1;
- while (raw_prefix_len < 16)
- {
- switch (raw_prefix[raw_prefix_len])
- {
- case ' ': case '(': case ')': case '\\': case '\t':
- case '\v': case '\f': case '\n': default:
- break;
- /* Basic source charset except the above chars. */
- case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
- case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
- case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
- case 's': case 't': case 'u': case 'v': case 'w': case 'x':
- case 'y': case 'z':
- case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
- case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
- case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
- case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
- case 'Y': case 'Z':
- case '0': case '1': case '2': case '3': case '4': case '5':
- case '6': case '7': case '8': case '9':
- case '_': case '{': case '}': case '#': case '[': case ']':
- case '<': case '>': case '%': case ':': case ';': case '.':
- case '?': case '*': case '+': case '-': case '/': case '^':
- case '&': case '|': case '~': case '!': case '=': case ',':
- case '"': case '\'':
- raw_prefix_len++;
- continue;
- }
- break;
- }
-
- if (raw_prefix[raw_prefix_len] != '(')
- {
- int col = CPP_BUF_COLUMN (pfile->buffer, raw_prefix + raw_prefix_len)
- + 1;
- if (raw_prefix_len == 16)
- cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, col,
- "raw string delimiter longer than 16 characters");
- else
- cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, col,
- "invalid character '%c' in raw string delimiter",
- (int) raw_prefix[raw_prefix_len]);
- pfile->buffer->cur = raw_prefix - 1;
- create_literal (pfile, token, base, raw_prefix - 1 - base, CPP_OTHER);
- return;
- }
-
- cur = raw_prefix + raw_prefix_len + 1;
- for (;;)
- {
#define BUF_APPEND(STR,LEN) \
do { \
bufring_append (pfile, (const uchar *)(STR), (LEN), \
&first_buff, &last_buff); \
total_len += (LEN); \
+ if (__builtin_expect (temp_buffer_len < 17, 0) \
+ && (const uchar *)(STR) != base \
+ && (LEN) <= 2) \
+ { \
+ memcpy (temp_buffer + temp_buffer_len, \
+ (const uchar *)(STR), (LEN)); \
+ temp_buffer_len += (LEN); \
+ } \
} while (0);
+ orig_base = base;
+ ++cur;
+ raw_prefix_start = cur - base;
+ for (;;)
+ {
cppchar_t c;
/* If we previously performed any trigraph or line splicing
- transformations, undo them within the body of the raw string. */
+ transformations, undo them in between the opening and closing
+ double quote. */
while (note->pos < cur)
++note;
for (; note->pos == cur; ++note)
++note;
goto after_backslash;
}
- /* The ) from ??) could be part of the suffix. */
- else if (type == ')'
- && strncmp ((const char *) cur+1,
- (const char *) raw_prefix,
- raw_prefix_len) == 0
- && cur[raw_prefix_len+1] == '"')
- {
- BUF_APPEND (")", 1);
- base++;
- cur += raw_prefix_len + 2;
- goto break_outer_loop;
- }
else
{
/* Skip the replacement character. */
base = ++cur;
BUF_APPEND (&type, 1);
+ c = type;
+ goto check_c;
}
}
else
}
}
c = *cur++;
+ if (__builtin_expect (temp_buffer_len < 17, 0))
+ temp_buffer[temp_buffer_len++] = c;
- if (c == ')'
- && strncmp ((const char *) cur, (const char *) raw_prefix,
- raw_prefix_len) == 0
- && cur[raw_prefix_len] == '"')
+ check_c:
+ if (phase == RAW_STR_PREFIX)
{
- cur += raw_prefix_len + 1;
- break;
+ while (raw_prefix_len < temp_buffer_len)
+ {
+ raw_prefix[raw_prefix_len] = temp_buffer[raw_prefix_len];
+ switch (raw_prefix[raw_prefix_len])
+ {
+ case ' ': case '(': case ')': case '\\': case '\t':
+ case '\v': case '\f': case '\n': default:
+ break;
+ /* Basic source charset except the above chars. */
+ case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
+ case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
+ case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
+ case 's': case 't': case 'u': case 'v': case 'w': case 'x':
+ case 'y': case 'z':
+ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
+ case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
+ case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
+ case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
+ case 'Y': case 'Z':
+ case '0': case '1': case '2': case '3': case '4': case '5':
+ case '6': case '7': case '8': case '9':
+ case '_': case '{': case '}': case '#': case '[': case ']':
+ case '<': case '>': case '%': case ':': case ';': case '.':
+ case '?': case '*': case '+': case '-': case '/': case '^':
+ case '&': case '|': case '~': case '!': case '=': case ',':
+ case '"': case '\'':
+ if (raw_prefix_len < 16)
+ {
+ raw_prefix_len++;
+ continue;
+ }
+ break;
+ }
+
+ if (raw_prefix[raw_prefix_len] != '(')
+ {
+ int col = CPP_BUF_COLUMN (pfile->buffer, cur) + 1;
+ if (raw_prefix_len == 16)
+ cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc,
+ col, "raw string delimiter longer "
+ "than 16 characters");
+ else if (raw_prefix[raw_prefix_len] == '\n')
+ cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc,
+ col, "invalid new-line in raw "
+ "string delimiter");
+ else
+ cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc,
+ col, "invalid character '%c' in "
+ "raw string delimiter",
+ (int) raw_prefix[raw_prefix_len]);
+ pfile->buffer->cur = orig_base + raw_prefix_start - 1;
+ create_literal (pfile, token, orig_base,
+ raw_prefix_start - 1, CPP_OTHER);
+ if (first_buff)
+ _cpp_release_buff (pfile, first_buff);
+ return;
+ }
+ raw_prefix[raw_prefix_len] = '"';
+ phase = RAW_STR;
+ /* Nothing should be appended to temp_buffer during
+ RAW_STR phase. */
+ temp_buffer_len = 17;
+ break;
+ }
+ continue;
+ }
+ else if (phase == RAW_STR_SUFFIX)
+ {
+ while (raw_suffix_len <= raw_prefix_len
+ && raw_suffix_len < temp_buffer_len
+ && temp_buffer[raw_suffix_len] == raw_prefix[raw_suffix_len])
+ raw_suffix_len++;
+ if (raw_suffix_len > raw_prefix_len)
+ break;
+ if (raw_suffix_len == temp_buffer_len)
+ continue;
+ phase = RAW_STR;
+ /* Nothing should be appended to temp_buffer during
+ RAW_STR phase. */
+ temp_buffer_len = 17;
+ }
+ if (c == ')')
+ {
+ phase = RAW_STR_SUFFIX;
+ raw_suffix_len = 0;
+ temp_buffer_len = 0;
}
else if (c == '\n')
{
if (pfile->state.in_directive
- || pfile->state.parsing_args
- || pfile->state.in_deferred_pragma)
+ || (pfile->state.parsing_args
+ && pfile->buffer->next_line >= pfile->buffer->rlimit))
{
cur--;
type = CPP_OTHER;
note = &pfile->buffer->notes[pfile->buffer->cur_note];
}
}
- break_outer_loop:
+
+ if (CPP_OPTION (pfile, user_literals))
+ {
+ /* If a string format macro, say from inttypes.h, is placed touching
+ a string literal it could be parsed as a C++11 user-defined string
+ literal thus breaking the program.
+ Try to identify macros with is_macro. A warning is issued. */
+ if (is_macro (pfile, cur))
+ {
+ /* Raise a warning, but do not consume subsequent tokens. */
+ if (CPP_OPTION (pfile, warn_literal_suffix) && !pfile->state.skipping)
+ cpp_warning_with_line (pfile, CPP_W_LITERAL_SUFFIX,
+ token->src_loc, 0,
+ "invalid suffix on literal; C++11 requires "
+ "a space between literal and string macro");
+ }
+ /* Grab user defined literal suffix. */
+ else if (ISIDST (*cur))
+ {
+ type = cpp_userdef_string_add_type (type);
+ ++cur;
+
+ while (ISIDNUM (*cur))
+ ++cur;
+ }
+ }
pfile->buffer->cur = cur;
if (first_buff == NULL)
else if (terminator == '\'')
type = (*base == 'L' ? CPP_WCHAR :
*base == 'U' ? CPP_CHAR32 :
- *base == 'u' ? CPP_CHAR16 : CPP_CHAR);
+ *base == 'u' ? (base[1] == '8' ? CPP_UTF8CHAR : CPP_CHAR16)
+ : CPP_CHAR);
else
terminator = '>', type = CPP_HEADER_NAME;
cpp_error (pfile, CPP_DL_PEDWARN, "missing terminating %c character",
(int) terminator);
+ if (CPP_OPTION (pfile, user_literals))
+ {
+ /* If a string format macro, say from inttypes.h, is placed touching
+ a string literal it could be parsed as a C++11 user-defined string
+ literal thus breaking the program.
+ Try to identify macros with is_macro. A warning is issued. */
+ if (is_macro (pfile, cur))
+ {
+ /* Raise a warning, but do not consume subsequent tokens. */
+ if (CPP_OPTION (pfile, warn_literal_suffix) && !pfile->state.skipping)
+ cpp_warning_with_line (pfile, CPP_W_LITERAL_SUFFIX,
+ token->src_loc, 0,
+ "invalid suffix on literal; C++11 requires "
+ "a space between literal and string macro");
+ }
+ /* Grab user defined literal suffix. */
+ else if (ISIDST (*cur))
+ {
+ type = cpp_userdef_char_add_type (type);
+ type = cpp_userdef_string_add_type (type);
+ ++cur;
+
+ while (ISIDNUM (*cur))
+ ++cur;
+ }
+ }
+ else if (CPP_OPTION (pfile, cpp_warn_cxx11_compat)
+ && is_macro (pfile, cur)
+ && !pfile->state.skipping)
+ cpp_warning_with_line (pfile, CPP_W_CXX11_COMPAT,
+ token->src_loc, 0, "C++11 requires a space "
+ "between string literal and macro");
+
pfile->buffer->cur = cur;
create_literal (pfile, token, base, cur - base, type);
}
return run->next;
}
-/* Return the number of not yet processed token in the the current
+/* Return the number of not yet processed token in a given
context. */
int
-_cpp_remaining_tokens_num_in_context (cpp_reader *pfile)
+_cpp_remaining_tokens_num_in_context (cpp_context *context)
{
- cpp_context *context = pfile->context;
if (context->tokens_kind == TOKENS_KIND_DIRECT)
return (LAST (context).token - FIRST (context).token);
else if (context->tokens_kind == TOKENS_KIND_INDIRECT
abort ();
}
-/* Returns the token present at index INDEX in the current context.
- If INDEX is zero, the next token to be processed is returned. */
+/* Returns the token present at index INDEX in a given context. If
+ INDEX is zero, the next token to be processed is returned. */
static const cpp_token*
-_cpp_token_from_context_at (cpp_reader *pfile, int index)
+_cpp_token_from_context_at (cpp_context *context, int index)
{
- cpp_context *context = pfile->context;
if (context->tokens_kind == TOKENS_KIND_DIRECT)
return &(FIRST (context).token[index]);
else if (context->tokens_kind == TOKENS_KIND_INDIRECT
/* First, scan through any pending cpp_context objects. */
while (context->prev)
{
- ptrdiff_t sz = _cpp_remaining_tokens_num_in_context (pfile);
+ ptrdiff_t sz = _cpp_remaining_tokens_num_in_context (context);
if (index < (int) sz)
- return _cpp_token_from_context_at (pfile, index);
+ return _cpp_token_from_context_at (context, index);
index -= (int) sz;
context = context->prev;
}
count = index;
pfile->keep_tokens++;
+ /* For peeked tokens temporarily disable line_change reporting,
+ until the tokens are parsed for real. */
+ void (*line_change) (cpp_reader *, const cpp_token *, int)
+ = pfile->cb.line_change;
+ pfile->cb.line_change = NULL;
+
do
{
peektok = _cpp_lex_token (pfile);
if (peektok->type == CPP_EOF)
- return peektok;
+ {
+ index--;
+ break;
+ }
}
while (index--);
- _cpp_backup_tokens_direct (pfile, count + 1);
+ _cpp_backup_tokens_direct (pfile, count - index);
pfile->keep_tokens--;
+ pfile->cb.line_change = line_change;
return peektok;
}
&& CPP_OPTION (pfile, rliterals))
|| (*buffer->cur == '8'
&& c == 'u'
- && (buffer->cur[1] == '"'
+ && ((buffer->cur[1] == '"' || (buffer->cur[1] == '\''
+ && CPP_OPTION (pfile, utf8_char_literals)))
|| (buffer->cur[1] == 'R' && buffer->cur[2] == '"'
&& CPP_OPTION (pfile, rliterals)))))
{
{
struct normalize_state nst = INITIAL_NORMALIZE_STATE;
result->val.node.node = lex_identifier (pfile, buffer->cur - 1, false,
- &nst);
+ &nst,
+ &result->val.node.spelling);
warn_about_normalization (pfile, result, &nst);
}
if (_cpp_skip_block_comment (pfile))
cpp_error (pfile, CPP_DL_ERROR, "unterminated comment");
}
- else if (c == '/' && (CPP_OPTION (pfile, cplusplus_comments)
- || cpp_in_system_header (pfile)))
+ else if (c == '/' && ! CPP_OPTION (pfile, traditional))
{
- /* Warn about comments only if pedantically GNUC89, and not
+ /* Don't warn for system headers. */
+ if (cpp_in_system_header (pfile))
+ ;
+ /* Warn about comments if pedantically GNUC89, and not
in system headers. */
- if (CPP_OPTION (pfile, lang) == CLK_GNUC89 && CPP_PEDANTIC (pfile)
- && ! buffer->warned_cplusplus_comments)
+ else if (CPP_OPTION (pfile, lang) == CLK_GNUC89
+ && CPP_PEDANTIC (pfile)
+ && ! buffer->warned_cplusplus_comments)
{
cpp_error (pfile, CPP_DL_PEDWARN,
"C++ style comments are not allowed in ISO C90");
"(this will be reported only once per input file)");
buffer->warned_cplusplus_comments = 1;
}
-
+ /* Or if specifically desired via -Wc90-c99-compat. */
+ else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
+ && ! CPP_OPTION (pfile, cplusplus)
+ && ! buffer->warned_cplusplus_comments)
+ {
+ cpp_error (pfile, CPP_DL_WARNING,
+ "C++ style comments are incompatible with C90");
+ cpp_error (pfile, CPP_DL_WARNING,
+ "(this will be reported only once per input file)");
+ buffer->warned_cplusplus_comments = 1;
+ }
+ /* In C89/C94, C++ style comments are forbidden. */
+ else if ((CPP_OPTION (pfile, lang) == CLK_STDC89
+ || CPP_OPTION (pfile, lang) == CLK_STDC94))
+ {
+ /* But don't be confused about valid code such as
+ - // immediately followed by *,
+ - // in a preprocessing directive,
+ - // in an #if 0 block. */
+ if (buffer->cur[1] == '*'
+ || pfile->state.in_directive
+ || pfile->state.skipping)
+ {
+ result->type = CPP_DIV;
+ break;
+ }
+ else if (! buffer->warned_cplusplus_comments)
+ {
+ cpp_error (pfile, CPP_DL_ERROR,
+ "C++ style comments are not allowed in ISO C90");
+ cpp_error (pfile, CPP_DL_ERROR,
+ "(this will be reported only once per input "
+ "file)");
+ buffer->warned_cplusplus_comments = 1;
+ }
+ }
if (skip_line_comment (pfile) && CPP_OPTION (pfile, warn_comments))
cpp_warning (pfile, CPP_W_COMMENTS, "multi-line comment");
}
{
if (*buffer->cur == ':')
{
+ /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next
+ three characters are <:: and the subsequent character
+ is neither : nor >, the < is treated as a preprocessor
+ token by itself". */
+ if (CPP_OPTION (pfile, cplusplus)
+ && CPP_OPTION (pfile, lang) != CLK_CXX98
+ && CPP_OPTION (pfile, lang) != CLK_GNUCXX
+ && buffer->cur[1] == ':'
+ && buffer->cur[2] != ':' && buffer->cur[2] != '>')
+ break;
+
buffer->cur++;
result->flags |= DIGRAPH;
result->type = CPP_OPEN_SQUARE;
if (forms_identifier_p (pfile, true, &nst))
{
result->type = CPP_NAME;
- result->val.node.node = lex_identifier (pfile, base, true, &nst);
+ result->val.node.node = lex_identifier (pfile, base, true, &nst,
+ &result->val.node.spelling);
warn_about_normalization (pfile, result, &nst);
break;
}
break;
}
+ source_range tok_range;
+ tok_range.m_start = result->src_loc;
+ if (result->src_loc >= RESERVED_LOCATION_COUNT)
+ tok_range.m_finish
+ = linemap_position_for_column (pfile->line_table,
+ CPP_BUF_COLUMN (buffer, buffer->cur));
+ else
+ tok_range.m_finish = tok_range.m_start;
+
+ result->src_loc = COMBINE_LOCATION_DATA (pfile->line_table,
+ result->src_loc,
+ tok_range, NULL);
+
return result;
}
return digraph_spellings[(int) type - (int) CPP_FIRST_DIGRAPH];
}
+/* Write the spelling of an identifier IDENT, using UCNs, to BUFFER.
+ The buffer must already contain the enough space to hold the
+ token's spelling. Returns a pointer to the character after the
+ last character written. */
+unsigned char *
+_cpp_spell_ident_ucns (unsigned char *buffer, cpp_hashnode *ident)
+{
+ size_t i;
+ const unsigned char *name = NODE_NAME (ident);
+
+ for (i = 0; i < NODE_LEN (ident); i++)
+ if (name[i] & ~0x7F)
+ {
+ i += utf8_to_ucn (buffer, name + i) - 1;
+ buffer += 10;
+ }
+ else
+ *buffer++ = name[i];
+
+ return buffer;
+}
+
/* Write the spelling of a token TOKEN to BUFFER. The buffer must
already contain the enough space to hold the token's spelling.
Returns a pointer to the character after the last character written.
FORSTRING is true if this is to be the spelling after translation
- phase 1 (this is different for UCNs).
+ phase 1 (with the original spelling of extended identifiers), false
+ if extended identifiers should always be written using UCNs (there is
+ no option for always writing them in the internal UTF-8 form).
FIXME: Would be nice if we didn't need the PFILE argument. */
unsigned char *
cpp_spell_token (cpp_reader *pfile, const cpp_token *token,
case SPELL_IDENT:
if (forstring)
{
- memcpy (buffer, NODE_NAME (token->val.node.node),
- NODE_LEN (token->val.node.node));
- buffer += NODE_LEN (token->val.node.node);
+ memcpy (buffer, NODE_NAME (token->val.node.spelling),
+ NODE_LEN (token->val.node.spelling));
+ buffer += NODE_LEN (token->val.node.spelling);
}
else
- {
- size_t i;
- const unsigned char * name = NODE_NAME (token->val.node.node);
-
- for (i = 0; i < NODE_LEN (token->val.node.node); i++)
- if (name[i] & ~0x7F)
- {
- i += utf8_to_ucn (buffer, name + i) - 1;
- buffer += 10;
- }
- else
- *buffer++ = NODE_NAME (token->val.node.node)[i];
- }
+ buffer = _cpp_spell_ident_ucns (buffer, token->val.node.node);
break;
case SPELL_LITERAL:
return (a->type != CPP_PASTE || a->val.token_no == b->val.token_no);
case SPELL_NONE:
return (a->type != CPP_MACRO_ARG
- || a->val.macro_arg.arg_no == b->val.macro_arg.arg_no);
+ || (a->val.macro_arg.arg_no == b->val.macro_arg.arg_no
+ && a->val.macro_arg.spelling == b->val.macro_arg.spelling));
case SPELL_IDENT:
- return a->val.node.node == b->val.node.node;
+ return (a->val.node.node == b->val.node.node
+ && a->val.node.spelling == b->val.node.spelling);
case SPELL_LITERAL:
return (a->val.str.len == b->val.str.len
&& !memcmp (a->val.str.text, b->val.str.text,
|| (CPP_OPTION (pfile, objc)
&& token1->val.str.text[0] == '@'
&& (b == CPP_NAME || b == CPP_STRING)));
+ case CPP_STRING:
+ case CPP_WSTRING:
+ case CPP_UTF8STRING:
+ case CPP_STRING16:
+ case CPP_STRING32: return (CPP_OPTION (pfile, user_literals)
+ && (b == CPP_NAME
+ || (TOKEN_SPELL (token2) == SPELL_LITERAL
+ && ISIDST (token2->val.str.text[0]))));
+
default: break;
}
len = MIN_BUFF_SIZE;
len = CPP_ALIGN (len);
+#ifdef ENABLE_VALGRIND_CHECKING
+ /* Valgrind warns about uses of interior pointers, so put _cpp_buff
+ struct first. */
+ size_t slen = CPP_ALIGN2 (sizeof (_cpp_buff), 2 * DEFAULT_ALIGNMENT);
+ base = XNEWVEC (unsigned char, len + slen);
+ result = (_cpp_buff *) base;
+ base += slen;
+#else
base = XNEWVEC (unsigned char, len + sizeof (_cpp_buff));
result = (_cpp_buff *) (base + len);
+#endif
result->base = base;
result->cur = base;
result->limit = base + len;
for (; buff; buff = next)
{
next = buff->next;
+#ifdef ENABLE_VALGRIND_CHECKING
+ free (buff);
+#else
free (buff->base);
+#endif
}
}
/* Say which field of TOK is in use. */
enum cpp_token_fld_kind
-cpp_token_val_index (cpp_token *tok)
+cpp_token_val_index (const cpp_token *tok)
{
switch (TOKEN_SPELL (tok))
{
projects / gcc.git / blobdiff