/* CPP Library - charsets
- Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004
- Free Software Foundation, Inc.
+ Copyright (C) 1998-2020 Free Software Foundation, Inc.
Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
-Free Software Foundation; either version 2, or (at your option) any
+Free Software Foundation; either version 3, or (at your option) any
later version.
This program is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
+along with this program; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
#include "config.h"
#include "system.h"
one_utf8_to_cppchar (const uchar **inbufp, size_t *inbytesleftp,
cppchar_t *cp)
{
- static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x02, 0x01 };
+ static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
cppchar_t c;
return EILSEQ;
}
- if (s < 0xFFFF)
+ if (s <= 0xFFFF)
{
if (*outbytesleftp < 2)
{
outbytesleft += OUTBUF_BLOCK_SIZE;
to->asize += OUTBUF_BLOCK_SIZE;
- to->text = xrealloc (to->text, to->asize);
+ to->text = XRESIZEVEC (uchar, to->text, to->asize);
outbuf = to->text + to->asize - outbytesleft;
}
}
if (to->len + flen > to->asize)
{
to->asize = to->len + flen;
- to->text = xrealloc (to->text, to->asize);
+ to->asize += to->asize / 4;
+ to->text = XRESIZEVEC (uchar, to->text, to->asize);
}
memcpy (to->text + to->len, from, flen);
to->len += flen;
/* And this one uses the system iconv primitive. It's a little
different, since iconv's interface is a little different. */
#if HAVE_ICONV
+
+#define CONVERT_ICONV_GROW_BUFFER \
+ do { \
+ outbytesleft += OUTBUF_BLOCK_SIZE; \
+ to->asize += OUTBUF_BLOCK_SIZE; \
+ to->text = XRESIZEVEC (uchar, to->text, to->asize); \
+ outbuf = (char *)to->text + to->asize - outbytesleft; \
+ } while (0)
+
static bool
convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
struct _cpp_strbuf *to)
iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
if (__builtin_expect (inbytesleft == 0, 1))
{
+ /* Close out any shift states, returning to the initial state. */
+ if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
+ {
+ if (errno != E2BIG)
+ return false;
+
+ CONVERT_ICONV_GROW_BUFFER;
+ if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
+ return false;
+ }
+
to->len = to->asize - outbytesleft;
return true;
}
if (errno != E2BIG)
return false;
- outbytesleft += OUTBUF_BLOCK_SIZE;
- to->asize += OUTBUF_BLOCK_SIZE;
- to->text = xrealloc (to->text, to->asize);
- outbuf = (char *)to->text + to->asize - outbytesleft;
+ CONVERT_ICONV_GROW_BUFFER;
}
}
#else
#define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
-struct conversion
+struct cpp_conversion
{
const char *pair;
convert_f func;
iconv_t fake_cd;
};
-static const struct conversion conversion_tab[] = {
+static const struct cpp_conversion conversion_tab[] = {
{ "UTF-8/UTF-32LE", convert_utf8_utf32, (iconv_t)0 },
{ "UTF-8/UTF-32BE", convert_utf8_utf32, (iconv_t)1 },
{ "UTF-8/UTF-16LE", convert_utf8_utf16, (iconv_t)0 },
char *pair;
size_t i;
+ ret.to = to;
+ ret.from = from;
+
if (!strcasecmp (to, from))
{
ret.func = convert_no_conversion;
ret.cd = (iconv_t) -1;
+ ret.width = -1;
return ret;
}
- pair = alloca(strlen(to) + strlen(from) + 2);
+ pair = (char *) alloca(strlen(to) + strlen(from) + 2);
strcpy(pair, from);
strcat(pair, "/");
{
ret.func = conversion_tab[i].func;
ret.cd = conversion_tab[i].fake_cd;
+ ret.width = -1;
return ret;
}
{
ret.func = convert_using_iconv;
ret.cd = iconv_open (to, from);
+ ret.width = -1;
if (ret.cd == (iconv_t) -1)
{
from, to);
ret.func = convert_no_conversion;
ret.cd = (iconv_t) -1;
+ ret.width = -1;
}
return ret;
}
wcset = default_wcset;
pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET);
+ pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
+ pfile->utf8_cset_desc = init_iconv_desc (pfile, "UTF-8", SOURCE_CHARSET);
+ pfile->utf8_cset_desc.width = CPP_OPTION (pfile, char_precision);
+ pfile->char16_cset_desc = init_iconv_desc (pfile,
+ be ? "UTF-16BE" : "UTF-16LE",
+ SOURCE_CHARSET);
+ pfile->char16_cset_desc.width = 16;
+ pfile->char32_cset_desc = init_iconv_desc (pfile,
+ be ? "UTF-32BE" : "UTF-32LE",
+ SOURCE_CHARSET);
+ pfile->char32_cset_desc.width = 32;
pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET);
+ pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision);
}
/* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
{
if (pfile->narrow_cset_desc.func == convert_using_iconv)
iconv_close (pfile->narrow_cset_desc.cd);
+ if (pfile->utf8_cset_desc.func == convert_using_iconv)
+ iconv_close (pfile->utf8_cset_desc.cd);
+ if (pfile->char16_cset_desc.func == convert_using_iconv)
+ iconv_close (pfile->char16_cset_desc.cd);
+ if (pfile->char32_cset_desc.func == convert_using_iconv)
+ iconv_close (pfile->char32_cset_desc.cd);
if (pfile->wide_cset_desc.func == convert_using_iconv)
iconv_close (pfile->wide_cset_desc.cd);
}
/* This should never need to reallocate, but just in case... */
tbuf.asize = 1;
- tbuf.text = xmalloc (tbuf.asize);
+ tbuf.text = XNEWVEC (uchar, tbuf.asize);
tbuf.len = 0;
if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf))
\f
+/* cpp_substring_ranges's constructor. */
+
+cpp_substring_ranges::cpp_substring_ranges () :
+ m_ranges (NULL),
+ m_num_ranges (0),
+ m_alloc_ranges (8)
+{
+ m_ranges = XNEWVEC (source_range, m_alloc_ranges);
+}
+
+/* cpp_substring_ranges's destructor. */
+
+cpp_substring_ranges::~cpp_substring_ranges ()
+{
+ free (m_ranges);
+}
+
+/* Add RANGE to the vector of source_range information. */
+
+void
+cpp_substring_ranges::add_range (source_range range)
+{
+ if (m_num_ranges >= m_alloc_ranges)
+ {
+ m_alloc_ranges *= 2;
+ m_ranges
+ = (source_range *)xrealloc (m_ranges,
+ sizeof (source_range) * m_alloc_ranges);
+ }
+ m_ranges[m_num_ranges++] = range;
+}
+
+/* Read NUM ranges from LOC_READER, adding them to the vector of source_range
+ information. */
+
+void
+cpp_substring_ranges::add_n_ranges (int num,
+ cpp_string_location_reader &loc_reader)
+{
+ for (int i = 0; i < num; i++)
+ add_range (loc_reader.get_next ());
+}
+
+\f
+
/* Utility routine that computes a mask of the form 0000...111... with
WIDTH 1-bits. */
static inline size_t
/* Valid in a C99 identifier? */
C99 = 1,
/* Valid in a C99 identifier, but not as the first character? */
- DIG = 2,
+ N99 = 2,
/* Valid in a C++ identifier? */
CXX = 4,
+ /* Valid in a C11/C++11 identifier? */
+ C11 = 8,
+ /* Valid in a C11/C++11 identifier, but not as the first character? */
+ N11 = 16,
/* NFC representation is not valid in an identifier? */
- CID = 8,
+ CID = 32,
/* Might be valid NFC form? */
- NFC = 16,
+ NFC = 64,
/* Might be valid NFKC form? */
- NKC = 32,
+ NKC = 128,
/* Certain preceding characters might make it not valid NFC/NKFC form? */
- CTX = 64
+ CTX = 256
};
-static const struct {
+struct ucnrange {
/* Bitmap of flags above. */
- unsigned char flags;
+ unsigned short flags;
/* Combining class of the character. */
unsigned char combine;
/* Last character in the range described by this entry. */
- unsigned short end;
-} ucnranges[] = {
-#include "ucnid.h"
+ unsigned int end;
};
+#include "ucnid.h"
+
+/* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */
+#define UCS_LIMIT 0x10FFFF
/* Returns 1 if C is valid in an identifier, 2 if C is valid except at
the start of an identifier, and 0 if C is not valid in an
struct normalize_state *nst)
{
int mn, mx, md;
+ unsigned short valid_flags, invalid_start_flags;
- if (c > 0xFFFF)
+ if (c > UCS_LIMIT)
return 0;
mn = 0;
/* When -pedantic, we require the character to have been listed by
the standard for the current language. Otherwise, we accept the
- union of the acceptable sets for C++98 and C99. */
- if (! (ucnranges[mn].flags & (C99 | CXX)))
+ union of the acceptable sets for all supported language versions. */
+ valid_flags = C99 | CXX | C11;
+ if (CPP_PEDANTIC (pfile))
+ {
+ if (CPP_OPTION (pfile, c11_identifiers))
+ valid_flags = C11;
+ else if (CPP_OPTION (pfile, c99))
+ valid_flags = C99;
+ else if (CPP_OPTION (pfile, cplusplus))
+ valid_flags = CXX;
+ }
+ if (! (ucnranges[mn].flags & valid_flags))
return 0;
-
- if (CPP_PEDANTIC (pfile)
- && ((CPP_OPTION (pfile, c99) && !(ucnranges[mn].flags & C99))
- || (CPP_OPTION (pfile, cplusplus)
- && !(ucnranges[mn].flags & CXX))))
- return 0;
+ if (CPP_OPTION (pfile, c11_identifiers))
+ invalid_start_flags = N11;
+ else if (CPP_OPTION (pfile, c99))
+ invalid_start_flags = N99;
+ else
+ invalid_start_flags = 0;
/* Update NST. */
if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class)
bool safe;
cppchar_t p = nst->previous;
- /* Easy cases from Bengali, Oriya, Tamil, Jannada, and Malayalam. */
- if (c == 0x09BE)
- safe = p != 0x09C7; /* Use 09CB instead of 09C7 09BE. */
- else if (c == 0x0B3E)
- safe = p != 0x0B47; /* Use 0B4B instead of 0B47 0B3E. */
- else if (c == 0x0BBE)
- safe = p != 0x0BC6 && p != 0x0BC7; /* Use 0BCA/0BCB instead. */
- else if (c == 0x0CC2)
- safe = p != 0x0CC6; /* Use 0CCA instead of 0CC6 0CC2. */
- else if (c == 0x0D3E)
- safe = p != 0x0D46 && p != 0x0D47; /* Use 0D4A/0D4B instead. */
/* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
and are combined algorithmically from a sequence of the form
1100-1112 1161-1175 11A8-11C2
really a valid character).
Unfortunately, C99 allows (only) the NFC form, but C++ allows
only the combining characters. */
- else if (c >= 0x1161 && c <= 0x1175)
+ if (c >= 0x1161 && c <= 0x1175)
safe = p < 0x1100 || p > 0x1112;
else if (c >= 0x11A8 && c <= 0x11C2)
safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0);
else
+ safe = check_nfc (pfile, c, p);
+ if (!safe)
{
- /* Uh-oh, someone updated ucnid.h without updating this code. */
- cpp_error (pfile, CPP_DL_ICE, "Character %x might not be NFKC", c);
- safe = true;
+ if ((c >= 0x1161 && c <= 0x1175) || (c >= 0x11A8 && c <= 0x11C2))
+ nst->level = MAX (nst->level, normalized_identifier_C);
+ else
+ nst->level = normalized_none;
}
- if (!safe && c < 0x1161)
- nst->level = normalized_none;
- else if (!safe)
- nst->level = MAX (nst->level, normalized_identifier_C);
}
else if (ucnranges[mn].flags & NKC)
;
nst->level = MAX (nst->level, normalized_identifier_C);
else
nst->level = normalized_none;
- nst->previous = c;
+ if (ucnranges[mn].combine == 0)
+ nst->previous = c;
nst->prev_class = ucnranges[mn].combine;
- /* In C99, UCN digits may not begin identifiers. */
- if (CPP_OPTION (pfile, c99) && (ucnranges[mn].flags & DIG))
+ /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
+ UCN combining characters may not begin identifiers. */
+ if (ucnranges[mn].flags & invalid_start_flags)
return 2;
return 1;
ISO/IEC 10646 is NNNNNNNN; the character designated by the
universal character name \uNNNN is that character whose character
short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
- for a universal character name is less than 0x20 or in the range
- 0x7F-0x9F (inclusive), or if the universal character name
- designates a character in the basic source character set, then the
- program is ill-formed.
+ for a universal character name corresponds to a surrogate code point
+ (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
+ Additionally, if the hexadecimal value for a universal-character-name
+ outside a character or string literal corresponds to a control character
+ (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
+ character in the basic source character set, the program is ill-formed.
+
+ C99 6.4.3: A universal character name shall not specify a character
+ whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
+ or 0060 (`), nor one in the range D800 through DFFF inclusive.
+
+ If the hexadecimal value is larger than the upper bound of the UCS
+ codespace specified in ISO/IEC 10646, a pedantic warning is issued
+ in all versions of C and in the C++20 or later versions of C++.
*PSTR must be preceded by "\u" or "\U"; it is assumed that the
- buffer end is delimited by a non-hex digit. Returns zero if the
- UCN has not been consumed.
+ buffer end is delimited by a non-hex digit. Returns false if the
+ UCN has not been consumed, true otherwise.
- Otherwise the nonzero value of the UCN, whether valid or invalid,
- is returned. Diagnostics are emitted for invalid values. PSTR
- is updated to point one beyond the UCN, or to the syntactically
- invalid character.
+ The value of the UCN, whether valid or invalid, is returned in *CP.
+ Diagnostics are emitted for invalid values. PSTR is updated to point
+ one beyond the UCN, or to the syntactically invalid character.
IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
- an identifier, or 2 otherwise. */
+ an identifier, or 2 otherwise.
-cppchar_t
+ If LOC_READER is non-NULL, then position information is
+ read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
+
+bool
_cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
const uchar *limit, int identifier_pos,
- struct normalize_state *nst)
+ struct normalize_state *nst, cppchar_t *cp,
+ source_range *char_range,
+ cpp_string_location_reader *loc_reader)
{
cppchar_t result, c;
unsigned int length;
if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
cpp_error (pfile, CPP_DL_WARNING,
"universal character names are only valid in C++ and C99");
- else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
+ else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
+ && !CPP_OPTION (pfile, cplusplus))
cpp_error (pfile, CPP_DL_WARNING,
- "the meaning of '\\%c' is different in traditional C",
- (int) str[-1]);
+ "C99's universal character names are incompatible with C90");
+ else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
+ cpp_warning (pfile, CPP_W_TRADITIONAL,
+ "the meaning of '\\%c' is different in traditional C",
+ (int) str[-1]);
if (str[-1] == 'u')
length = 4;
if (!ISXDIGIT (c))
break;
str++;
+ if (loc_reader)
+ {
+ gcc_assert (char_range);
+ char_range->m_finish = loc_reader->get_next ().m_finish;
+ }
result = (result << 4) + hex_value (c);
}
while (--length && str < limit);
multiple tokens in identifiers, so we can't give a helpful
error message in that case. */
if (length && identifier_pos)
- return 0;
-
+ {
+ *cp = 0;
+ return false;
+ }
+
*pstr = str;
if (length)
{
(int) (str - base), base);
result = 1;
}
- /* The standard permits $, @ and ` to be specified as UCNs. We use
- hex escapes so that this also works with EBCDIC hosts. */
+ /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
+ hex escapes so that this also works with EBCDIC hosts.
+ C++0x permits everything below 0xa0 within literals;
+ ucn_valid_in_identifier will complain about identifiers. */
else if ((result < 0xa0
+ && !CPP_OPTION (pfile, cplusplus)
&& (result != 0x24 && result != 0x40 && result != 0x60))
|| (result & 0x80000000)
|| (result >= 0xD800 && result <= 0xDFFF))
CPP_OPTION (pfile, warn_dollars) = 0;
cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
}
- NORMALIZE_STATE_UPDATE_IDNUM (nst);
+ NORMALIZE_STATE_UPDATE_IDNUM (nst, result);
}
else if (identifier_pos)
{
"universal character %.*s is not valid at the start of an identifier",
(int) (str - base), base);
}
-
- if (result == 0)
- result = 1;
-
- return result;
+ else if (result > UCS_LIMIT
+ && (!CPP_OPTION (pfile, cplusplus)
+ || CPP_OPTION (pfile, lang) > CLK_CXX17))
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "%.*s is outside the UCS codespace",
+ (int) (str - base), base);
+
+ *cp = result;
+ return true;
}
/* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
- it to the execution character set and write the result into TBUF.
- An advanced pointer is returned. Issues all relevant diagnostics. */
+ it to the execution character set and write the result into TBUF,
+ if TBUF is non-NULL.
+ An advanced pointer is returned. Issues all relevant diagnostics.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+ contains the location of the character so far: location information
+ is read from *LOC_READER, and *RANGES is updated accordingly. */
static const uchar *
convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ source_range char_range,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
cppchar_t ucn;
uchar buf[6];
uchar *bufp = buf;
size_t bytesleft = 6;
int rval;
- struct cset_converter cvt
- = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc;
struct normalize_state nst = INITIAL_NORMALIZE_STATE;
+ /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
from++; /* Skip u/U. */
- ucn = _cpp_valid_ucn (pfile, &from, limit, 0, &nst);
+
+ if (loc_reader)
+ /* The u/U is part of the spelling of this character. */
+ char_range.m_finish = loc_reader->get_next ().m_finish;
+
+ _cpp_valid_ucn (pfile, &from, limit, 0, &nst,
+ &ucn, &char_range, loc_reader);
rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft);
if (rval)
cpp_errno (pfile, CPP_DL_ERROR,
"converting UCN to source character set");
}
- else if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
- cpp_errno (pfile, CPP_DL_ERROR,
- "converting UCN to execution character set");
+ else
+ {
+ if (tbuf)
+ if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
+ cpp_errno (pfile, CPP_DL_ERROR,
+ "converting UCN to execution character set");
+
+ if (loc_reader)
+ {
+ int num_encoded_bytes = 6 - bytesleft;
+ for (int i = 0; i < num_encoded_bytes; i++)
+ ranges->add_range (char_range);
+ }
+ }
return from;
}
+/* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded
+ extended characters rather than UCNs. If the return value is TRUE, then a
+ character was successfully decoded and stored in *CP; *PSTR has been
+ updated to point one past the valid UTF-8 sequence. Diagnostics may have
+ been emitted if the character parsed is not allowed in the current context.
+ If the return value is FALSE, then *PSTR has not been modified and *CP may
+ equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it
+ may, when processing an identifier in C mode, equal a codepoint that was
+ validly encoded but is not allowed to appear in an identifier. In either
+ case, no diagnostic is emitted, and the return value of FALSE should cause
+ a new token to be formed.
+
+ Unlike _cpp_valid_ucn, this will never be called when lexing a string; only
+ a potential identifier, or a CPP_OTHER token. NST is unused in the latter
+ case.
+
+ As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for
+ the start of an identifier, or 2 otherwise. */
+
+extern bool
+_cpp_valid_utf8 (cpp_reader *pfile,
+ const uchar **pstr,
+ const uchar *limit,
+ int identifier_pos,
+ struct normalize_state *nst,
+ cppchar_t *cp)
+{
+ const uchar *base = *pstr;
+ size_t inbytesleft = limit - base;
+ if (one_utf8_to_cppchar (pstr, &inbytesleft, cp))
+ {
+ /* No diagnostic here as this byte will rather become a
+ new token. */
+ *cp = 0;
+ return false;
+ }
+
+ if (identifier_pos)
+ {
+ switch (ucn_valid_in_identifier (pfile, *cp, nst))
+ {
+
+ case 0:
+ /* In C++, this is an error for invalid character in an identifier
+ because logically, the UTF-8 was converted to a UCN during
+ translation phase 1 (even though we don't physically do it that
+ way). In C, this byte rather becomes grammatically a separate
+ token. */
+
+ if (CPP_OPTION (pfile, cplusplus))
+ cpp_error (pfile, CPP_DL_ERROR,
+ "extended character %.*s is not valid in an identifier",
+ (int) (*pstr - base), base);
+ else
+ {
+ *pstr = base;
+ return false;
+ }
+
+ break;
+
+ case 2:
+ if (identifier_pos == 1)
+ {
+ /* This is treated the same way in C++ or C99 -- lexed as an
+ identifier which is then invalid because an identifier is
+ not allowed to start with this character. */
+ cpp_error (pfile, CPP_DL_ERROR,
+ "extended character %.*s is not valid at the start of an identifier",
+ (int) (*pstr - base), base);
+ }
+ break;
+ }
+ }
+
+ return true;
+}
+
/* Subroutine of convert_hex and convert_oct. N is the representation
in the execution character set of a numeric escape; write it into the
string buffer TBUF and update the end-of-string pointer therein. WIDE
function issues no diagnostics and never fails. */
static void
emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt)
{
- if (wide)
+ size_t width = cvt.width;
+
+ if (width != CPP_OPTION (pfile, char_precision))
{
/* We have to render this into the target byte order, which may not
be our byte order. */
bool bigend = CPP_OPTION (pfile, bytes_big_endian);
- size_t width = CPP_OPTION (pfile, wchar_precision);
size_t cwidth = CPP_OPTION (pfile, char_precision);
size_t cmask = width_to_mask (cwidth);
size_t nbwc = width / cwidth;
if (tbuf->len + nbwc > tbuf->asize)
{
tbuf->asize += OUTBUF_BLOCK_SIZE;
- tbuf->text = xrealloc (tbuf->text, tbuf->asize);
+ tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
}
for (i = 0; i < nbwc; i++)
if (tbuf->len + 1 > tbuf->asize)
{
tbuf->asize += OUTBUF_BLOCK_SIZE;
- tbuf->text = xrealloc (tbuf->text, tbuf->asize);
+ tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
}
tbuf->text[tbuf->len++] = n;
}
}
/* Convert a hexadecimal escape, pointed to by FROM, to the execution
- character set and write it into the string buffer TBUF. Returns an
- advanced pointer, and issues diagnostics as necessary.
+ character set and write it into the string buffer TBUF (if non-NULL).
+ Returns an advanced pointer, and issues diagnostics as necessary.
No character set translation occurs; this routine always produces the
execution-set character with numeric value equal to the given hex
- number. You can, e.g. generate surrogate pairs this way. */
+ number. You can, e.g. generate surrogate pairs this way.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+ contains the location of the character so far: location information
+ is read from *LOC_READER, and *RANGES is updated accordingly. */
static const uchar *
convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ source_range char_range,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
cppchar_t c, n = 0, overflow = 0;
int digits_found = 0;
- size_t width = (wide ? CPP_OPTION (pfile, wchar_precision)
- : CPP_OPTION (pfile, char_precision));
+ size_t width = cvt.width;
size_t mask = width_to_mask (width);
+ /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
if (CPP_WTRADITIONAL (pfile))
- cpp_error (pfile, CPP_DL_WARNING,
- "the meaning of '\\x' is different in traditional C");
+ cpp_warning (pfile, CPP_W_TRADITIONAL,
+ "the meaning of '\\x' is different in traditional C");
+
+ /* Skip 'x'. */
+ from++;
+
+ /* The 'x' is part of the spelling of this character. */
+ if (loc_reader)
+ char_range.m_finish = loc_reader->get_next ().m_finish;
- from++; /* Skip 'x'. */
while (from < limit)
{
c = *from;
if (! hex_p (c))
break;
from++;
+ if (loc_reader)
+ char_range.m_finish = loc_reader->get_next ().m_finish;
overflow |= n ^ (n << 4 >> 4);
n = (n << 4) + hex_value (c);
digits_found = 1;
n &= mask;
}
- emit_numeric_escape (pfile, n, tbuf, wide);
+ if (tbuf)
+ emit_numeric_escape (pfile, n, tbuf, cvt);
+ if (ranges)
+ ranges->add_range (char_range);
return from;
}
advanced pointer, and issues diagnostics as necessary.
No character set translation occurs; this routine always produces the
execution-set character with numeric value equal to the given octal
- number. */
+ number.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+ contains the location of the character so far: location information
+ is read from *LOC_READER, and *RANGES is updated accordingly. */
static const uchar *
convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ source_range char_range,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
size_t count = 0;
cppchar_t c, n = 0;
- size_t width = (wide ? CPP_OPTION (pfile, wchar_precision)
- : CPP_OPTION (pfile, char_precision));
+ size_t width = cvt.width;
size_t mask = width_to_mask (width);
bool overflow = false;
+ /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
while (from < limit && count++ < 3)
{
c = *from;
if (c < '0' || c > '7')
break;
from++;
+ if (loc_reader)
+ char_range.m_finish = loc_reader->get_next ().m_finish;
overflow |= n ^ (n << 3 >> 3);
n = (n << 3) + c - '0';
}
n &= mask;
}
- emit_numeric_escape (pfile, n, tbuf, wide);
+ if (tbuf)
+ emit_numeric_escape (pfile, n, tbuf, cvt);
+ if (ranges)
+ ranges->add_range (char_range);
return from;
}
/* Convert an escape sequence (pointed to by FROM) to its value on
the target, and to the execution character set. Do not scan past
- LIMIT. Write the converted value into TBUF. Returns an advanced
- pointer. Handles all relevant diagnostics. */
+ LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
+ Returns an advanced pointer. Handles all relevant diagnostics.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL: location
+ information is read from *LOC_READER, and *RANGES is updated
+ accordingly. */
static const uchar *
convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
/* Values of \a \b \e \f \n \r \t \v respectively. */
#if HOST_CHARSET == HOST_CHARSET_ASCII
#endif
uchar c;
- struct cset_converter cvt
- = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc;
+
+ /* Record the location of the backslash. */
+ source_range char_range;
+ if (loc_reader)
+ char_range = loc_reader->get_next ();
c = *from;
switch (c)
{
/* UCNs, hex escapes, and octal escapes are processed separately. */
case 'u': case 'U':
- return convert_ucn (pfile, from, limit, tbuf, wide);
+ return convert_ucn (pfile, from, limit, tbuf, cvt,
+ char_range, loc_reader, ranges);
case 'x':
- return convert_hex (pfile, from, limit, tbuf, wide);
+ return convert_hex (pfile, from, limit, tbuf, cvt,
+ char_range, loc_reader, ranges);
break;
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7':
- return convert_oct (pfile, from, limit, tbuf, wide);
+ return convert_oct (pfile, from, limit, tbuf, cvt,
+ char_range, loc_reader, ranges);
/* Various letter escapes. Get the appropriate host-charset
value into C. */
case 'a':
if (CPP_WTRADITIONAL (pfile))
- cpp_error (pfile, CPP_DL_WARNING,
- "the meaning of '\\a' is different in traditional C");
+ cpp_warning (pfile, CPP_W_TRADITIONAL,
+ "the meaning of '\\a' is different in traditional C");
c = charconsts[0];
break;
unknown:
if (ISGRAPH (c))
cpp_error (pfile, CPP_DL_PEDWARN,
- "unknown escape sequence '\\%c'", (int) c);
+ "unknown escape sequence: '\\%c'", (int) c);
else
- cpp_error (pfile, CPP_DL_PEDWARN,
- "unknown escape sequence: '\\%03o'", (int) c);
+ {
+ /* diagnostic.c does not support "%03o". When it does, this
+ code can use %03o directly in the diagnostic again. */
+ char buf[32];
+ sprintf(buf, "%03o", (int) c);
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "unknown escape sequence: '\\%s'", buf);
+ }
}
- /* Now convert what we have to the execution character set. */
- if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
- cpp_errno (pfile, CPP_DL_ERROR,
- "converting escape sequence to execution character set");
+ if (tbuf)
+ /* Now convert what we have to the execution character set. */
+ if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
+ cpp_errno (pfile, CPP_DL_ERROR,
+ "converting escape sequence to execution character set");
+
+ if (loc_reader)
+ {
+ char_range.m_finish = loc_reader->get_next ().m_finish;
+ ranges->add_range (char_range);
+ }
return from + 1;
}
\f
+/* TYPE is a token type. The return value is the conversion needed to
+ convert from source to execution character set for the given type. */
+static struct cset_converter
+converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
+{
+ switch (type)
+ {
+ default:
+ return pfile->narrow_cset_desc;
+ case CPP_UTF8CHAR:
+ case CPP_UTF8STRING:
+ return pfile->utf8_cset_desc;
+ case CPP_CHAR16:
+ case CPP_STRING16:
+ return pfile->char16_cset_desc;
+ case CPP_CHAR32:
+ case CPP_STRING32:
+ return pfile->char32_cset_desc;
+ case CPP_WCHAR:
+ case CPP_WSTRING:
+ return pfile->wide_cset_desc;
+ }
+}
+
/* FROM is an array of cpp_string structures of length COUNT. These
are to be converted from the source to the execution character set,
escape sequences translated, and finally all are to be
concatenated. WIDE indicates whether or not to produce a wide
- string. The result is written into TO. Returns true for success,
- false for failure. */
-bool
-cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
- cpp_string *to, bool wide)
+ string. If TO is non-NULL, the result is written into TO.
+ If LOC_READERS and OUT are non-NULL, then location information
+ is read from LOC_READERS (which must be an array of length COUNT),
+ and location information is written to *RANGES.
+
+ Returns true for success, false for failure. */
+
+static bool
+cpp_interpret_string_1 (cpp_reader *pfile, const cpp_string *from, size_t count,
+ cpp_string *to, enum cpp_ttype type,
+ cpp_string_location_reader *loc_readers,
+ cpp_substring_ranges *out)
{
struct _cpp_strbuf tbuf;
const uchar *p, *base, *limit;
size_t i;
- struct cset_converter cvt
- = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc;
+ struct cset_converter cvt = converter_for_type (pfile, type);
- tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
- tbuf.text = xmalloc (tbuf.asize);
- tbuf.len = 0;
+ /* loc_readers and out must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_readers != NULL) == (out != NULL));
+ if (to)
+ {
+ tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
+ tbuf.text = XNEWVEC (uchar, tbuf.asize);
+ tbuf.len = 0;
+ }
+
+ cpp_string_location_reader *loc_reader = NULL;
for (i = 0; i < count; i++)
{
+ if (loc_readers)
+ loc_reader = &loc_readers[i];
+
p = from[i].text;
- if (*p == 'L') p++;
- p++; /* Skip leading quote. */
+ if (*p == 'u')
+ {
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ if (*p == '8')
+ {
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ }
+ }
+ else if (*p == 'L' || *p == 'U') p++;
+ if (*p == 'R')
+ {
+ const uchar *prefix;
+
+ /* Skip over 'R"'. */
+ p += 2;
+ if (loc_reader)
+ {
+ loc_reader->get_next ();
+ loc_reader->get_next ();
+ }
+ prefix = p;
+ while (*p != '(')
+ {
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ }
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ limit = from[i].text + from[i].len;
+ if (limit >= p + (p - prefix) + 1)
+ limit -= (p - prefix) + 1;
+
+ /* Raw strings are all normal characters; these can be fed
+ directly to convert_cset. */
+ if (to)
+ if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
+ goto fail;
+
+ if (loc_reader)
+ {
+ /* If generating source ranges, assume we have a 1:1
+ correspondence between bytes in the source encoding and bytes
+ in the execution encoding (e.g. if we have a UTF-8 to UTF-8
+ conversion), so that this run of bytes in the source file
+ corresponds to a run of bytes in the execution string.
+ This requirement is guaranteed by an early-reject in
+ cpp_interpret_string_ranges. */
+ gcc_assert (cvt.func == convert_no_conversion);
+ out->add_n_ranges (limit - p, *loc_reader);
+ }
+
+ continue;
+ }
+
+ /* If we don't now have a leading quote, something has gone wrong.
+ This can occur if cpp_interpret_string_ranges is handling a
+ stringified macro argument, but should not be possible otherwise. */
+ if (*p != '"' && *p != '\'')
+ {
+ gcc_assert (out != NULL);
+ cpp_error (pfile, CPP_DL_ERROR, "missing open quote");
+ if (to)
+ free (tbuf.text);
+ return false;
+ }
+
+ /* Skip leading quote. */
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+
limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
for (;;)
{
/* We have a run of normal characters; these can be fed
directly to convert_cset. */
- if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
- goto fail;
+ if (to)
+ if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
+ goto fail;
+ /* Similar to above: assumes we have a 1:1 correspondence
+ between bytes in the source encoding and bytes in the
+ execution encoding. */
+ if (loc_reader)
+ {
+ gcc_assert (cvt.func == convert_no_conversion);
+ out->add_n_ranges (p - base, *loc_reader);
+ }
}
- if (p == limit)
+ if (p >= limit)
break;
- p = convert_escape (pfile, p + 1, limit, &tbuf, wide);
+ struct _cpp_strbuf *tbuf_ptr = to ? &tbuf : NULL;
+ p = convert_escape (pfile, p + 1, limit, tbuf_ptr, cvt,
+ loc_reader, out);
}
}
- /* NUL-terminate the 'to' buffer and translate it to a cpp_string
- structure. */
- emit_numeric_escape (pfile, 0, &tbuf, wide);
- tbuf.text = xrealloc (tbuf.text, tbuf.len);
- to->text = tbuf.text;
- to->len = tbuf.len;
+
+ if (to)
+ {
+ /* NUL-terminate the 'to' buffer and translate it to a cpp_string
+ structure. */
+ emit_numeric_escape (pfile, 0, &tbuf, cvt);
+ tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
+ to->text = tbuf.text;
+ to->len = tbuf.len;
+ }
+ /* Use the location of the trailing quote as the location of the
+ NUL-terminator. */
+ if (loc_reader)
+ {
+ source_range range = loc_reader->get_next ();
+ out->add_range (range);
+ }
+
return true;
fail:
cpp_errno (pfile, CPP_DL_ERROR, "converting to execution character set");
- free (tbuf.text);
+ if (to)
+ free (tbuf.text);
return false;
}
+/* FROM is an array of cpp_string structures of length COUNT. These
+ are to be converted from the source to the execution character set,
+ escape sequences translated, and finally all are to be
+ concatenated. WIDE indicates whether or not to produce a wide
+ string. The result is written into TO. Returns true for success,
+ false for failure. */
+bool
+cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
+ cpp_string *to, enum cpp_ttype type)
+{
+ return cpp_interpret_string_1 (pfile, from, count, to, type, NULL, NULL);
+}
+
+/* A "do nothing" diagnostic-handling callback for use by
+ cpp_interpret_string_ranges, so that it can temporarily suppress
+ diagnostic-handling. */
+
+static bool
+noop_diagnostic_cb (cpp_reader *, enum cpp_diagnostic_level,
+ enum cpp_warning_reason, rich_location *,
+ const char *, va_list *)
+{
+ /* no-op. */
+ return true;
+}
+
+/* This function mimics the behavior of cpp_interpret_string, but
+ rather than generating a string in the execution character set,
+ *OUT is written to with the source code ranges of the characters
+ in such a string.
+ FROM and LOC_READERS should both be arrays of length COUNT.
+ Returns NULL for success, or an error message for failure. */
+
+const char *
+cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from,
+ cpp_string_location_reader *loc_readers,
+ size_t count,
+ cpp_substring_ranges *out,
+ enum cpp_ttype type)
+{
+ /* There are a couple of cases in the range-handling in
+ cpp_interpret_string_1 that rely on there being a 1:1 correspondence
+ between bytes in the source encoding and bytes in the execution
+ encoding, so that each byte in the execution string can correspond
+ to the location of a byte in the source string.
+
+ This holds for the typical case of a UTF-8 to UTF-8 conversion.
+ Enforce this requirement by only attempting to track substring
+ locations if we have source encoding == execution encoding.
+
+ This is a stronger condition than we need, since we could e.g.
+ have ASCII to EBCDIC (with 1 byte per character before and after),
+ but it seems to be a reasonable restriction. */
+ struct cset_converter cvt = converter_for_type (pfile, type);
+ if (cvt.func != convert_no_conversion)
+ return "execution character set != source character set";
+
+ /* For on-demand strings we have already lexed the strings, so there
+ should be no diagnostics. However, if we have bogus source location
+ data (or stringified macro arguments), the attempt to lex the
+ strings could fail with an diagnostic. Temporarily install an
+ diagnostic-handler to catch the diagnostic, so that it can lead to this call
+ failing, rather than being emitted as a user-visible diagnostic.
+ If an diagnostic does occur, we should see it via the return value of
+ cpp_interpret_string_1. */
+ bool (*saved_diagnostic_handler) (cpp_reader *, enum cpp_diagnostic_level,
+ enum cpp_warning_reason, rich_location *,
+ const char *, va_list *)
+ ATTRIBUTE_FPTR_PRINTF(5,0);
+
+ saved_diagnostic_handler = pfile->cb.diagnostic;
+ pfile->cb.diagnostic = noop_diagnostic_cb;
+
+ bool result = cpp_interpret_string_1 (pfile, from, count, NULL, type,
+ loc_readers, out);
+
+ /* Restore the saved diagnostic-handler. */
+ pfile->cb.diagnostic = saved_diagnostic_handler;
+
+ if (!result)
+ return "cpp_interpret_string_1 failed";
+
+ /* Success. */
+ return NULL;
+}
+
/* Subroutine of do_line and do_linemarker. Convert escape sequences
in a string, but do not perform character set conversion. */
bool
cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from,
- size_t count, cpp_string *to, bool wide)
+ size_t count, cpp_string *to,
+ enum cpp_ttype type ATTRIBUTE_UNUSED)
{
struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc;
bool retval;
pfile->narrow_cset_desc.func = convert_no_conversion;
pfile->narrow_cset_desc.cd = (iconv_t) -1;
+ pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
- retval = cpp_interpret_string (pfile, from, count, to, wide);
+ retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING);
pfile->narrow_cset_desc = save_narrow_cset_desc;
return retval;
/* Subroutine of cpp_interpret_charconst which performs the conversion
to a number, for narrow strings. STR is the string structure returned
by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
- cpp_interpret_charconst. */
+ cpp_interpret_charconst. TYPE is the token type. */
static cppchar_t
narrow_str_to_charconst (cpp_reader *pfile, cpp_string str,
- unsigned int *pchars_seen, int *unsignedp)
+ unsigned int *pchars_seen, int *unsignedp,
+ enum cpp_ttype type)
{
size_t width = CPP_OPTION (pfile, char_precision);
size_t max_chars = CPP_OPTION (pfile, int_precision) / width;
result = c;
}
+ if (type == CPP_UTF8CHAR)
+ max_chars = 1;
if (i > max_chars)
{
i = max_chars;
- cpp_error (pfile, CPP_DL_WARNING,
+ cpp_error (pfile, type == CPP_UTF8CHAR ? CPP_DL_ERROR : CPP_DL_WARNING,
"character constant too long for its type");
}
else if (i > 1 && CPP_OPTION (pfile, warn_multichar))
- cpp_error (pfile, CPP_DL_WARNING, "multi-character character constant");
+ cpp_warning (pfile, CPP_W_MULTICHAR, "multi-character character constant");
/* Multichar constants are of type int and therefore signed. */
if (i > 1)
unsigned_p = 0;
+ else if (type == CPP_UTF8CHAR && !CPP_OPTION (pfile, cplusplus))
+ unsigned_p = 1;
else
unsigned_p = CPP_OPTION (pfile, unsigned_char);
/* Subroutine of cpp_interpret_charconst which performs the conversion
to a number, for wide strings. STR is the string structure returned
by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
- cpp_interpret_charconst. */
+ cpp_interpret_charconst. TYPE is the token type. */
static cppchar_t
wide_str_to_charconst (cpp_reader *pfile, cpp_string str,
- unsigned int *pchars_seen, int *unsignedp)
+ unsigned int *pchars_seen, int *unsignedp,
+ enum cpp_ttype type)
{
bool bigend = CPP_OPTION (pfile, bytes_big_endian);
- size_t width = CPP_OPTION (pfile, wchar_precision);
+ size_t width = converter_for_type (pfile, type).width;
size_t cwidth = CPP_OPTION (pfile, char_precision);
size_t mask = width_to_mask (width);
size_t cmask = width_to_mask (cwidth);
size_t off, i;
cppchar_t result = 0, c;
+ if (str.len <= nbwc)
+ {
+ /* Error recovery, if no errors have been diagnosed previously,
+ there should be at least two wide characters. Empty literals
+ are diagnosed earlier and we can get just the zero terminator
+ only if there were errors diagnosed during conversion. */
+ *pchars_seen = 0;
+ *unsignedp = 0;
+ return 0;
+ }
+
/* This is finicky because the string is in the target's byte order,
which may not be our byte order. Only the last character, ignoring
the NUL terminator, is relevant. */
/* Wide character constants have type wchar_t, and a single
character exactly fills a wchar_t, so a multi-character wide
character constant is guaranteed to overflow. */
- if (off > 0)
- cpp_error (pfile, CPP_DL_WARNING,
+ if (str.len > nbwc * 2)
+ cpp_error (pfile, (CPP_OPTION (pfile, cplusplus)
+ && (type == CPP_CHAR16 || type == CPP_CHAR32))
+ ? CPP_DL_ERROR : CPP_DL_WARNING,
"character constant too long for its type");
/* Truncate the constant to its natural width, and simultaneously
sign- or zero-extend to the full width of cppchar_t. */
if (width < BITS_PER_CPPCHAR_T)
{
- if (CPP_OPTION (pfile, unsigned_wchar) || !(result & (1 << (width - 1))))
+ if (type == CPP_CHAR16 || type == CPP_CHAR32
+ || CPP_OPTION (pfile, unsigned_wchar)
+ || !(result & (1 << (width - 1))))
result &= mask;
else
result |= ~mask;
}
- *unsignedp = CPP_OPTION (pfile, unsigned_wchar);
+ if (type == CPP_CHAR16 || type == CPP_CHAR32
+ || CPP_OPTION (pfile, unsigned_wchar))
+ *unsignedp = 1;
+ else
+ *unsignedp = 0;
+
*pchars_seen = 1;
return result;
}
unsigned int *pchars_seen, int *unsignedp)
{
cpp_string str = { 0, 0 };
- bool wide = (token->type == CPP_WCHAR);
+ bool wide = (token->type != CPP_CHAR && token->type != CPP_UTF8CHAR);
+ int u8 = 2 * int(token->type == CPP_UTF8CHAR);
cppchar_t result;
- /* an empty constant will appear as L'' or '' */
- if (token->val.str.len == (size_t) (2 + wide))
+ /* An empty constant will appear as L'', u'', U'', u8'', or '' */
+ if (token->val.str.len == (size_t) (2 + wide + u8))
{
cpp_error (pfile, CPP_DL_ERROR, "empty character constant");
+ *pchars_seen = 0;
+ *unsignedp = 0;
+ return 0;
+ }
+ else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str,
+ token->type))
+ {
+ *pchars_seen = 0;
+ *unsignedp = 0;
return 0;
}
- else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, wide))
- return 0;
if (wide)
- result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp);
+ result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
+ token->type);
else
- result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp);
+ result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp,
+ token->type);
if (str.text != token->val.str.text)
free ((void *)str.text);
}
\f
/* Convert an identifier denoted by ID and LEN, which might contain
- UCN escapes, to the source character set, either UTF-8 or
- UTF-EBCDIC. Assumes that the identifier is actually a valid identifier. */
+ UCN escapes or UTF-8 multibyte chars, to the source character set,
+ either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually
+ a valid identifier. */
cpp_hashnode *
_cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len)
{
/* It turns out that a UCN escape always turns into fewer characters
than the escape itself, so we can allocate a temporary in advance. */
- uchar * buf = alloca (len + 1);
+ uchar * buf = (uchar *) alloca (len + 1);
uchar * bufp = buf;
size_t idp;
source file) from INPUT_CHARSET to the source character set. INPUT
points to the input buffer, SIZE is its allocated size, and LEN is
the length of the meaningful data within the buffer. The
- translated buffer is returned, and *ST_SIZE is set to the length of
- the meaningful data within the translated buffer.
-
- INPUT is expected to have been allocated with xmalloc. This function
- will either return INPUT, or free it and return a pointer to another
- xmalloc-allocated block of memory. */
+ translated buffer is returned, *ST_SIZE is set to the length of
+ the meaningful data within the translated buffer, and *BUFFER_START
+ is set to the start of the returned buffer. *BUFFER_START may
+ differ from the return value in the case of a BOM or other ignored
+ marker information.
+
+ INPUT is expected to have been allocated with xmalloc. This
+ function will either set *BUFFER_START to INPUT, or free it and set
+ *BUFFER_START to a pointer to another xmalloc-allocated block of
+ memory. */
uchar *
_cpp_convert_input (cpp_reader *pfile, const char *input_charset,
- uchar *input, size_t size, size_t len, off_t *st_size)
+ uchar *input, size_t size, size_t len,
+ const unsigned char **buffer_start, off_t *st_size)
{
struct cset_converter input_cset;
struct _cpp_strbuf to;
+ unsigned char *buffer;
input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset);
if (input_cset.func == convert_no_conversion)
else
{
to.asize = MAX (65536, len);
- to.text = xmalloc (to.asize);
+ to.text = XNEWVEC (uchar, to.asize);
to.len = 0;
if (!APPLY_CONVERSION (input_cset, input, len, &to))
iconv_close (input_cset.cd);
/* Resize buffer if we allocated substantially too much, or if we
- haven't enough space for the \n-terminator. */
- if (to.len + 4096 < to.asize || to.len >= to.asize)
- to.text = xrealloc (to.text, to.len + 1);
+ haven't enough space for the \n-terminator or following
+ 15 bytes of padding (used to quiet warnings from valgrind or
+ Address Sanitizer, when the optimized lexer accesses aligned
+ 16-byte memory chunks, including the bytes after the malloced,
+ area, and stops lexing on '\n'). */
+ if (to.len + 4096 < to.asize || to.len + 16 > to.asize)
+ to.text = XRESIZEVEC (uchar, to.text, to.len + 16);
+
+ memset (to.text + to.len, '\0', 16);
/* If the file is using old-school Mac line endings (\r only),
terminate with another \r, not an \n, so that we do not mistake
the \r\n sequence for a single DOS line ending and erroneously
issue the "No newline at end of file" diagnostic. */
- if (to.text[to.len - 1] == '\r')
+ if (to.len && to.text[to.len - 1] == '\r')
to.text[to.len] = '\r';
else
to.text[to.len] = '\n';
+ buffer = to.text;
*st_size = to.len;
- return to.text;
+#if HOST_CHARSET == HOST_CHARSET_ASCII
+ /* The HOST_CHARSET test just above ensures that the source charset
+ is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
+ glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
+ BOM -- however, even if it did, we would still need this code due
+ to the 'convert_no_conversion' case. */
+ if (to.len >= 3 && to.text[0] == 0xef && to.text[1] == 0xbb
+ && to.text[2] == 0xbf)
+ {
+ *st_size -= 3;
+ buffer += 3;
+ }
+#endif
+
+ *buffer_start = to.text;
+ return buffer;
}
/* Decide on the default encoding to assume for input files. */
return current_encoding;
}
+
+/* Implementation of class cpp_string_location_reader. */
+
+/* Constructor for cpp_string_location_reader. */
+
+cpp_string_location_reader::
+cpp_string_location_reader (location_t src_loc,
+ line_maps *line_table)
+{
+ src_loc = get_range_from_loc (line_table, src_loc).m_start;
+
+ /* SRC_LOC might be a macro location. It only makes sense to do
+ column-by-column calculations on ordinary maps, so get the
+ corresponding location in an ordinary map. */
+ m_loc
+ = linemap_resolve_location (line_table, src_loc,
+ LRK_SPELLING_LOCATION, NULL);
+
+ const line_map_ordinary *map
+ = linemap_check_ordinary (linemap_lookup (line_table, m_loc));
+ m_offset_per_column = (1 << map->m_range_bits);
+}
+
+/* Get the range of the next source byte. */
+
+source_range
+cpp_string_location_reader::get_next ()
+{
+ source_range result;
+ result.m_start = m_loc;
+ result.m_finish = m_loc;
+ if (m_loc <= LINE_MAP_MAX_LOCATION_WITH_COLS)
+ m_loc += m_offset_per_column;
+ return result;
+}
+
+cpp_display_width_computation::
+cpp_display_width_computation (const char *data, int data_length, int tabstop) :
+ m_begin (data),
+ m_next (m_begin),
+ m_bytes_left (data_length),
+ m_tabstop (tabstop),
+ m_display_cols (0)
+{
+ gcc_assert (m_tabstop > 0);
+}
+
+
+/* The main implementation function for class cpp_display_width_computation.
+ m_next points on entry to the start of the UTF-8 encoding of the next
+ character, and is updated to point just after the last byte of the encoding.
+ m_bytes_left contains on entry the remaining size of the buffer into which
+ m_next points, and this is also updated accordingly. If m_next does not
+ point to a valid UTF-8-encoded sequence, then it will be treated as a single
+ byte with display width 1. m_cur_display_col is the current display column,
+ relative to which tab stops should be expanded. Returns the display width of
+ the codepoint just processed. */
+
+int
+cpp_display_width_computation::process_next_codepoint ()
+{
+ cppchar_t c;
+ int next_width;
+
+ if (*m_next == '\t')
+ {
+ ++m_next;
+ --m_bytes_left;
+ next_width = m_tabstop - (m_display_cols % m_tabstop);
+ }
+ else if (one_utf8_to_cppchar ((const uchar **) &m_next, &m_bytes_left, &c)
+ != 0)
+ {
+ /* Input is not convertible to UTF-8. This could be fine, e.g. in a
+ string literal, so don't complain. Just treat it as if it has a width
+ of one. */
+ ++m_next;
+ --m_bytes_left;
+ next_width = 1;
+ }
+ else
+ {
+ /* one_utf8_to_cppchar() has updated m_next and m_bytes_left for us. */
+ next_width = cpp_wcwidth (c);
+ }
+
+ m_display_cols += next_width;
+ return next_width;
+}
+
+/* Utility to advance the byte stream by the minimum amount needed to consume
+ N display columns. Returns the number of display columns that were
+ actually skipped. This could be less than N, if there was not enough data,
+ or more than N, if the last character to be skipped had a sufficiently large
+ display width. */
+int
+cpp_display_width_computation::advance_display_cols (int n)
+{
+ const int start = m_display_cols;
+ const int target = start + n;
+ while (m_display_cols < target && !done ())
+ process_next_codepoint ();
+ return m_display_cols - start;
+}
+
+/* For the string of length DATA_LENGTH bytes that begins at DATA, compute
+ how many display columns are occupied by the first COLUMN bytes. COLUMN
+ may exceed DATA_LENGTH, in which case the phantom bytes at the end are
+ treated as if they have display width 1. Tabs are expanded to the next tab
+ stop, relative to the start of DATA. */
+
+int
+cpp_byte_column_to_display_column (const char *data, int data_length,
+ int column, int tabstop)
+{
+ const int offset = MAX (0, column - data_length);
+ cpp_display_width_computation dw (data, column - offset, tabstop);
+ while (!dw.done ())
+ dw.process_next_codepoint ();
+ return dw.display_cols_processed () + offset;
+}
+
+/* For the string of length DATA_LENGTH bytes that begins at DATA, compute
+ the least number of bytes that will result in at least DISPLAY_COL display
+ columns. The return value may exceed DATA_LENGTH if the entire string does
+ not occupy enough display columns. */
+
+int
+cpp_display_column_to_byte_column (const char *data, int data_length,
+ int display_col, int tabstop)
+{
+ cpp_display_width_computation dw (data, data_length, tabstop);
+ const int avail_display = dw.advance_display_cols (display_col);
+ return dw.bytes_processed () + MAX (0, display_col - avail_display);
+}
+
+/* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc,
+ because that will inspect the user's locale, and in particular in an ASCII
+ locale, it will not return anything useful for extended characters. But GCC
+ in other respects (see e.g. _cpp_default_encoding()) behaves as if
+ everything is UTF-8. We also make some tweaks that are useful for the way
+ GCC needs to use this data, e.g. tabs and other control characters should be
+ treated as having width 1. The lookup tables are generated from
+ contrib/unicode/gen_wcwidth.py and were made by simply calling glibc
+ wcwidth() on all codepoints, then applying the small tweaks. These tables
+ are not highly optimized, but for the present purpose of outputting
+ diagnostics, they are sufficient. */
+
+#include "generated_cpp_wcwidth.h"
+int cpp_wcwidth (cppchar_t c)
+{
+ if (__builtin_expect (c <= wcwidth_range_ends[0], true))
+ return wcwidth_widths[0];
+
+ /* Binary search the tables. */
+ int begin = 1;
+ static const int end
+ = sizeof wcwidth_range_ends / sizeof (*wcwidth_range_ends);
+ int len = end - begin;
+ do
+ {
+ int half = len/2;
+ int middle = begin + half;
+ if (c > wcwidth_range_ends[middle])
+ {
+ begin = middle + 1;
+ len -= half + 1;
+ }
+ else
+ len = half;
+ } while (len);
+
+ if (__builtin_expect (begin != end, true))
+ return wcwidth_widths[begin];
+ return 1;
+}
projects / gcc.git / blobdiff