1 /* CPP Library - charsets
2 Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003
3 Free Software Foundation, Inc.
5 Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
7 This program is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by the
9 Free Software Foundation; either version 2, or (at your option) any
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
23 #include "coretypes.h"
29 /* Character set handling for C-family languages.
31 Terminological note: In what follows, "charset" or "character set"
32 will be taken to mean both an abstract set of characters and an
33 encoding for that set.
35 The C99 standard discusses two character sets: source and execution.
36 The source character set is used for internal processing in translation
37 phases 1 through 4; the execution character set is used thereafter.
38 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
39 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
40 of these terms). Furthermore, the "basic character set" (listed in
41 5.2.1p3) is to be encoded in each with values one byte wide, and is
42 to appear in the initial shift state.
44 It is not explicitly mentioned, but there is also a "wide execution
45 character set" used to encode wide character constants and wide
46 string literals; this is supposed to be the result of applying the
47 standard library function mbstowcs() to an equivalent narrow string
48 (6.4.5p5). However, the behavior of hexadecimal and octal
49 \-escapes is at odds with this; they are supposed to be translated
50 directly to wchar_t values (6.4.4.4p5,6).
52 The source character set is not necessarily the character set used
53 to encode physical source files on disk; translation phase 1 converts
54 from whatever that encoding is to the source character set.
56 The presence of universal character names in C99 (6.4.3 et seq.)
57 forces the source character set to be isomorphic to ISO 10646,
58 that is, Unicode. There is no such constraint on the execution
59 character set; note also that the conversion from source to
60 execution character set does not occur for identifiers (5.1.1.2p1#5).
62 For convenience of implementation, the source character set's
63 encoding of the basic character set should be identical to the
64 execution character set OF THE HOST SYSTEM's encoding of the basic
65 character set, and it should not be a state-dependent encoding.
67 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
68 depending on whether the host is based on ASCII or EBCDIC (see
69 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
70 Technical Report #16). It relies on the system library's iconv()
71 primitive to do charset conversion (specified in SUSv2). If this
72 primitive is not present, the source and execution character sets
73 must be identical and are limited to the basic ASCII or EBCDIC
74 range, and wide characters are implemented by padding narrow
75 characters to the size of wchar_t. */
78 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
79 below, which are guarded only by if statements with compile-time
80 constant conditions, do not cause link errors. */
81 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
82 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
83 #define iconv_close(x) 0
87 #if HOST_CHARSET == HOST_CHARSET_ASCII
88 #define SOURCE_CHARSET "UTF-8"
89 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
90 #define SOURCE_CHARSET "UTF-EBCDIC"
92 #error "Unrecognized basic host character set"
95 /* This structure is used for a resizable string buffer throughout. */
103 /* This is enough to hold any string that fits on a single 80-column
104 line, even if iconv quadruples its size (e.g. conversion from
105 ASCII to UTF-32) rounded up to a power of two. */
106 #define OUTBUF_BLOCK_SIZE 256
108 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
109 logic. This is because a depressing number of systems lack iconv,
110 or have have iconv libraries that do not do these conversions, so
111 we need a fallback implementation for them. To ensure the fallback
112 doesn't break due to neglect, it is used on all systems.
114 UTF-32 encoding is nice and simple: a four-byte binary number,
115 constrained to the range 00000000-7FFFFFFF to avoid questions of
116 signedness. We do have to cope with big- and little-endian
119 UTF-16 encoding uses two-byte binary numbers, again in big- and
120 little-endian variants, for all values in the 00000000-0000FFFF
121 range. Values in the 00010000-0010FFFF range are encoded as pairs
122 of two-byte numbers, called "surrogate pairs": given a number S in
123 this range, it is mapped to a pair (H, L) as follows:
125 H = (S - 0x10000) / 0x400 + 0xD800
126 L = (S - 0x10000) % 0x400 + 0xDC00
128 Two-byte values in the D800...DFFF range are ill-formed except as a
129 component of a surrogate pair. Even if the encoding within a
130 two-byte value is little-endian, the H member of the surrogate pair
133 There is no way to encode values in the 00110000-7FFFFFFF range,
134 which is not currently a problem as there are no assigned code
135 points in that range; however, the author expects that it will
136 eventually become necessary to abandon UTF-16 due to this
137 limitation. Note also that, because of these pairs, UTF-16 does
138 not meet the requirements of the C standard for a wide character
139 encoding (see 3.7.3 and 6.4.4.4p11).
141 UTF-8 encoding looks like this:
143 value range encoded as
144 00000000-0000007F 0xxxxxxx
145 00000080-000007FF 110xxxxx 10xxxxxx
146 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
147 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
148 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
149 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
151 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
152 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
153 never occur. Note also that any value that can be encoded by a
154 given row of the table can also be encoded by all successive rows,
155 but this is not done; only the shortest possible encoding for any
156 given value is valid. For instance, the character 07C0 could be
157 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
158 FC 80 80 80 9F 80. Only the first is valid.
160 An implementation note: the transformation from UTF-16 to UTF-8, or
161 vice versa, is easiest done by using UTF-32 as an intermediary. */
163 /* Internal primitives which go from an UTF-8 byte stream to native-endian
164 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
165 operation in several places below. */
167 one_utf8_to_cppchar (const uchar
**inbufp
, size_t *inbytesleftp
,
170 static const uchar masks
[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x02, 0x01 };
171 static const uchar patns
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
174 const uchar
*inbuf
= *inbufp
;
177 if (*inbytesleftp
< 1)
189 /* The number of leading 1-bits in the first byte indicates how many
191 for (nbytes
= 2; nbytes
< 7; nbytes
++)
192 if ((c
& ~masks
[nbytes
-1]) == patns
[nbytes
-1])
197 if (*inbytesleftp
< nbytes
)
200 c
= (c
& masks
[nbytes
-1]);
202 for (i
= 1; i
< nbytes
; i
++)
204 cppchar_t n
= *inbuf
++;
205 if ((n
& 0xC0) != 0x80)
207 c
= ((c
<< 6) + (n
& 0x3F));
210 /* Make sure the shortest possible encoding was used. */
211 if (c
<= 0x7F && nbytes
> 1) return EILSEQ
;
212 if (c
<= 0x7FF && nbytes
> 2) return EILSEQ
;
213 if (c
<= 0xFFFF && nbytes
> 3) return EILSEQ
;
214 if (c
<= 0x1FFFFF && nbytes
> 4) return EILSEQ
;
215 if (c
<= 0x3FFFFFF && nbytes
> 5) return EILSEQ
;
217 /* Make sure the character is valid. */
218 if (c
> 0x7FFFFFFF || (c
>= 0xD800 && c
<= 0xDFFF)) return EILSEQ
;
222 *inbytesleftp
-= nbytes
;
227 one_cppchar_to_utf8 (cppchar_t c
, uchar
**outbufp
, size_t *outbytesleftp
)
229 static const uchar masks
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
230 static const uchar limits
[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
232 uchar buf
[6], *p
= &buf
[6];
233 uchar
*outbuf
= *outbufp
;
242 *--p
= ((c
& 0x3F) | 0x80);
246 while (c
>= 0x3F || (c
& limits
[nbytes
-1]));
247 *--p
= (c
| masks
[nbytes
-1]);
250 if (*outbytesleftp
< nbytes
)
255 *outbytesleftp
-= nbytes
;
260 /* The following four functions transform one character between the two
261 encodings named in the function name. All have the signature
262 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
263 uchar **outbufp, size_t *outbytesleftp)
265 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
266 interpreted as a boolean indicating whether big-endian or
267 little-endian encoding is to be used for the member of the pair
270 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
273 The return value is either 0 for success, or an errno value for
274 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
275 input sequence), ir EINVAL (incomplete input sequence). */
278 one_utf8_to_utf32 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
279 uchar
**outbufp
, size_t *outbytesleftp
)
285 /* Check for space first, since we know exactly how much we need. */
286 if (*outbytesleftp
< 4)
289 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
294 outbuf
[bigend
? 3 : 0] = (s
& 0x000000FF);
295 outbuf
[bigend
? 2 : 1] = (s
& 0x0000FF00) >> 8;
296 outbuf
[bigend
? 1 : 2] = (s
& 0x00FF0000) >> 16;
297 outbuf
[bigend
? 0 : 3] = (s
& 0xFF000000) >> 24;
305 one_utf32_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
306 uchar
**outbufp
, size_t *outbytesleftp
)
312 if (*inbytesleftp
< 4)
317 s
= inbuf
[bigend
? 0 : 3] << 24;
318 s
+= inbuf
[bigend
? 1 : 2] << 16;
319 s
+= inbuf
[bigend
? 2 : 1] << 8;
320 s
+= inbuf
[bigend
? 3 : 0];
322 if (s
>= 0x7FFFFFFF || (s
>= 0xD800 && s
<= 0xDFFF))
325 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
335 one_utf8_to_utf16 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
336 uchar
**outbufp
, size_t *outbytesleftp
)
340 const uchar
*save_inbuf
= *inbufp
;
341 size_t save_inbytesleft
= *inbytesleftp
;
342 uchar
*outbuf
= *outbufp
;
344 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
350 *inbufp
= save_inbuf
;
351 *inbytesleftp
= save_inbytesleft
;
357 if (*outbytesleftp
< 2)
359 *inbufp
= save_inbuf
;
360 *inbytesleftp
= save_inbytesleft
;
363 outbuf
[bigend
? 1 : 0] = (s
& 0x00FF);
364 outbuf
[bigend
? 0 : 1] = (s
& 0xFF00) >> 8;
374 if (*outbytesleftp
< 4)
376 *inbufp
= save_inbuf
;
377 *inbytesleftp
= save_inbytesleft
;
381 hi
= (s
- 0x10000) / 0x400 + 0xD800;
382 lo
= (s
- 0x10000) % 0x400 + 0xDC00;
384 /* Even if we are little-endian, put the high surrogate first.
385 ??? Matches practice? */
386 outbuf
[bigend
? 1 : 0] = (hi
& 0x00FF);
387 outbuf
[bigend
? 0 : 1] = (hi
& 0xFF00) >> 8;
388 outbuf
[bigend
? 3 : 2] = (lo
& 0x00FF);
389 outbuf
[bigend
? 2 : 3] = (lo
& 0xFF00) >> 8;
398 one_utf16_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
399 uchar
**outbufp
, size_t *outbytesleftp
)
402 const uchar
*inbuf
= *inbufp
;
405 if (*inbytesleftp
< 2)
407 s
= inbuf
[bigend
? 0 : 1] << 8;
408 s
+= inbuf
[bigend
? 1 : 0];
410 /* Low surrogate without immediately preceding high surrogate is invalid. */
411 if (s
>= 0xDC00 && s
<= 0xDFFF)
413 /* High surrogate must have a following low surrogate. */
414 else if (s
>= 0xD800 && s
<= 0xDBFF)
416 cppchar_t hi
= s
, lo
;
417 if (*inbytesleftp
< 4)
420 lo
= inbuf
[bigend
? 2 : 3] << 8;
421 lo
+= inbuf
[bigend
? 3 : 2];
423 if (lo
< 0xDC00 || lo
> 0xDFFF)
426 s
= (hi
- 0xD800) * 0x400 + (lo
- 0xDC00) + 0x10000;
429 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
433 /* Success - update the input pointers (one_cppchar_to_utf8 has done
434 the output pointers for us). */
448 /* Helper routine for the next few functions. The 'const' on
449 one_conversion means that we promise not to modify what function is
450 pointed to, which lets the inliner see through it. */
453 conversion_loop (int (*const one_conversion
)(iconv_t
, const uchar
**, size_t *,
455 iconv_t cd
, const uchar
*from
, size_t flen
, struct strbuf
*to
)
459 size_t inbytesleft
, outbytesleft
;
464 outbuf
= to
->text
+ to
->len
;
465 outbytesleft
= to
->asize
- to
->len
;
470 rval
= one_conversion (cd
, &inbuf
, &inbytesleft
,
471 &outbuf
, &outbytesleft
);
472 while (inbytesleft
&& !rval
);
474 if (__builtin_expect (inbytesleft
== 0, 1))
476 to
->len
= to
->asize
- outbytesleft
;
485 outbytesleft
+= OUTBUF_BLOCK_SIZE
;
486 to
->asize
+= OUTBUF_BLOCK_SIZE
;
487 to
->text
= xrealloc (to
->text
, to
->asize
);
488 outbuf
= to
->text
+ to
->asize
- outbytesleft
;
493 /* These functions convert entire strings between character sets.
494 They all have the signature
496 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct strbuf *to);
498 The input string FROM is converted as specified by the function
499 name plus the iconv descriptor CD (which may be fake), and the
500 result appended to TO. On any error, false is returned, otherwise true. */
502 /* These four use the custom conversion code above. */
504 convert_utf8_utf16 (iconv_t cd
, const uchar
*from
, size_t flen
,
507 return conversion_loop (one_utf8_to_utf16
, cd
, from
, flen
, to
);
511 convert_utf8_utf32 (iconv_t cd
, const uchar
*from
, size_t flen
,
514 return conversion_loop (one_utf8_to_utf32
, cd
, from
, flen
, to
);
518 convert_utf16_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
521 return conversion_loop (one_utf16_to_utf8
, cd
, from
, flen
, to
);
525 convert_utf32_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
528 return conversion_loop (one_utf32_to_utf8
, cd
, from
, flen
, to
);
531 /* Identity conversion, used when we have no alternative. */
533 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED
,
534 const uchar
*from
, size_t flen
, struct strbuf
*to
)
536 if (to
->len
+ flen
> to
->asize
)
538 to
->asize
= to
->len
+ flen
;
539 to
->text
= xrealloc (to
->text
, to
->asize
);
541 memcpy (to
->text
+ to
->len
, from
, flen
);
546 /* And this one uses the system iconv primitive. It's a little
547 different, since iconv's interface is a little different. */
550 convert_using_iconv (iconv_t cd
, const uchar
*from
, size_t flen
,
553 ICONV_CONST
char *inbuf
;
555 size_t inbytesleft
, outbytesleft
;
557 /* Reset conversion descriptor and check that it is valid. */
558 if (iconv (cd
, 0, 0, 0, 0) == (size_t)-1)
561 inbuf
= (ICONV_CONST
char *)from
;
563 outbuf
= (char *)to
->text
+ to
->len
;
564 outbytesleft
= to
->asize
- to
->len
;
568 iconv (cd
, &inbuf
, &inbytesleft
, &outbuf
, &outbytesleft
);
569 if (__builtin_expect (inbytesleft
== 0, 1))
571 to
->len
= to
->asize
- outbytesleft
;
577 outbytesleft
+= OUTBUF_BLOCK_SIZE
;
578 to
->asize
+= OUTBUF_BLOCK_SIZE
;
579 to
->text
= xrealloc (to
->text
, to
->asize
);
580 outbuf
= (char *)to
->text
+ to
->asize
- outbytesleft
;
584 /* Arrange for the above custom conversion logic to be used automatically
585 when conversion between a suitable pair of character sets is requested. */
587 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
588 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
596 static const struct conversion conversion_tab
[] = {
597 { "UTF-8/UTF-32LE", convert_utf8_utf32
, (iconv_t
)0 },
598 { "UTF-8/UTF-32BE", convert_utf8_utf32
, (iconv_t
)1 },
599 { "UTF-8/UTF-16LE", convert_utf8_utf16
, (iconv_t
)0 },
600 { "UTF-8/UTF-16BE", convert_utf8_utf16
, (iconv_t
)1 },
601 { "UTF-32LE/UTF-8", convert_utf32_utf8
, (iconv_t
)0 },
602 { "UTF-32BE/UTF-8", convert_utf32_utf8
, (iconv_t
)1 },
603 { "UTF-16LE/UTF-8", convert_utf16_utf8
, (iconv_t
)0 },
604 { "UTF-16BE/UTF-8", convert_utf16_utf8
, (iconv_t
)1 },
607 /* Subroutine of cpp_init_iconv: initialize and return a
608 cset_converter structure for conversion from FROM to TO. If
609 iconv_open() fails, issue an error and return an identity
610 converter. Silently return an identity converter if FROM and TO
612 static struct cset_converter
613 init_iconv_desc (cpp_reader
*pfile
, const char *to
, const char *from
)
615 struct cset_converter ret
;
619 if (!strcasecmp (to
, from
))
621 ret
.func
= convert_no_conversion
;
622 ret
.cd
= (iconv_t
) -1;
626 pair
= alloca(strlen(to
) + strlen(from
) + 2);
631 for (i
= 0; i
< ARRAY_SIZE (conversion_tab
); i
++)
632 if (!strcasecmp (pair
, conversion_tab
[i
].pair
))
634 ret
.func
= conversion_tab
[i
].func
;
635 ret
.cd
= conversion_tab
[i
].fake_cd
;
639 /* No custom converter - try iconv. */
640 ret
.func
= convert_using_iconv
;
641 ret
.cd
= iconv_open (to
, from
);
643 if (ret
.cd
== (iconv_t
) -1)
646 cpp_error (pfile
, DL_ERROR
, /* XXX should be DL_SORRY */
647 "conversion from %s to %s not supported by iconv",
650 cpp_errno (pfile
, DL_ERROR
, "iconv_open");
652 ret
.func
= convert_no_conversion
;
657 /* If charset conversion is requested, initialize iconv(3) descriptors
658 for conversion from the source character set to the execution
659 character sets. If iconv is not present in the C library, and
660 conversion is requested, issue an error. */
663 cpp_init_iconv (cpp_reader
*pfile
)
665 const char *ncset
= CPP_OPTION (pfile
, narrow_charset
);
666 const char *wcset
= CPP_OPTION (pfile
, wide_charset
);
667 const char *default_wcset
;
669 bool be
= CPP_OPTION (pfile
, bytes_big_endian
);
671 if (CPP_OPTION (pfile
, wchar_precision
) >= 32)
672 default_wcset
= be
? "UTF-32BE" : "UTF-32LE";
673 else if (CPP_OPTION (pfile
, wchar_precision
) >= 16)
674 default_wcset
= be
? "UTF-16BE" : "UTF-16LE";
676 /* This effectively means that wide strings are not supported,
677 so don't do any conversion at all. */
678 default_wcset
= SOURCE_CHARSET
;
682 if (ncset
&& strcmp (ncset
, SOURCE_CHARSET
))
683 cpp_error (pfile
, DL_ERROR
, /* XXX should be DL_SORRY */
684 "no iconv implementation, cannot convert to %s", ncset
);
686 if (wcset
&& strcmp (wcset
, default_wcset
))
687 cpp_error (pfile
, DL_ERROR
, /* XXX should be DL_SORRY */
688 "no iconv implementation, cannot convert to %s", wcset
);
693 ncset
= SOURCE_CHARSET
;
695 wcset
= default_wcset
;
697 pfile
->narrow_cset_desc
= init_iconv_desc (pfile
, ncset
, SOURCE_CHARSET
);
698 pfile
->wide_cset_desc
= init_iconv_desc (pfile
, wcset
, SOURCE_CHARSET
);
703 _cpp_destroy_iconv (cpp_reader
*pfile
)
707 if (pfile
->narrow_cset_desc
.func
== convert_using_iconv
)
708 iconv_close (pfile
->narrow_cset_desc
.cd
);
709 if (pfile
->wide_cset_desc
.func
== convert_using_iconv
)
710 iconv_close (pfile
->wide_cset_desc
.cd
);
715 /* Utility routine that computes a mask of the form 0000...111... with
718 width_to_mask (size_t width
)
720 width
= MIN (width
, BITS_PER_CPPCHAR_T
);
721 if (width
>= CHAR_BIT
* sizeof (size_t))
724 return ((size_t) 1 << width
) - 1;
729 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
730 the start of an identifier, and 0 if C is not valid in an
731 identifier. We assume C has already gone through the checks of
732 _cpp_valid_ucn. The algorithm is a simple binary search on the
733 table defined in cppucnid.h. */
736 ucn_valid_in_identifier (cpp_reader
*pfile
, cppchar_t c
)
741 mx
= ARRAY_SIZE (ucnranges
);
745 if (c
< ucnranges
[md
].lo
)
747 else if (c
> ucnranges
[md
].hi
)
755 /* When -pedantic, we require the character to have been listed by
756 the standard for the current language. Otherwise, we accept the
757 union of the acceptable sets for C++98 and C99. */
758 if (CPP_PEDANTIC (pfile
)
759 && ((CPP_OPTION (pfile
, c99
) && !(ucnranges
[md
].flags
& C99
))
760 || (CPP_OPTION (pfile
, cplusplus
)
761 && !(ucnranges
[md
].flags
& CXX
))))
764 /* In C99, UCN digits may not begin identifiers. */
765 if (CPP_OPTION (pfile
, c99
) && (ucnranges
[md
].flags
& DIG
))
771 /* [lex.charset]: The character designated by the universal character
772 name \UNNNNNNNN is that character whose character short name in
773 ISO/IEC 10646 is NNNNNNNN; the character designated by the
774 universal character name \uNNNN is that character whose character
775 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
776 for a universal character name is less than 0x20 or in the range
777 0x7F-0x9F (inclusive), or if the universal character name
778 designates a character in the basic source character set, then the
779 program is ill-formed.
781 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
782 buffer end is delimited by a non-hex digit. Returns zero if UCNs
783 are not part of the relevant standard, or if the string beginning
784 at *PSTR doesn't syntactically match the form 'NNNN' or 'NNNNNNNN'.
786 Otherwise the nonzero value of the UCN, whether valid or invalid,
787 is returned. Diagnostics are emitted for invalid values. PSTR
788 is updated to point one beyond the UCN, or to the syntactically
791 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
792 an identifier, or 2 otherwise.
796 _cpp_valid_ucn (cpp_reader
*pfile
, const uchar
**pstr
,
797 const uchar
*limit
, int identifier_pos
)
801 const uchar
*str
= *pstr
;
802 const uchar
*base
= str
- 2;
804 if (!CPP_OPTION (pfile
, cplusplus
) && !CPP_OPTION (pfile
, c99
))
805 cpp_error (pfile
, DL_WARNING
,
806 "universal character names are only valid in C++ and C99");
807 else if (CPP_WTRADITIONAL (pfile
) && identifier_pos
== 0)
808 cpp_error (pfile
, DL_WARNING
,
809 "the meaning of '\\%c' is different in traditional C",
814 else if (str
[-1] == 'U')
826 result
= (result
<< 4) + hex_value (c
);
828 while (--length
&& str
< limit
);
833 /* We'll error when we try it out as the start of an identifier. */
834 cpp_error (pfile
, DL_ERROR
, "incomplete universal character name %.*s",
835 (int) (str
- base
), base
);
838 /* The standard permits $, @ and ` to be specified as UCNs. We use
839 hex escapes so that this also works with EBCDIC hosts. */
840 else if ((result
< 0xa0
841 && (result
!= 0x24 && result
!= 0x40 && result
!= 0x60))
842 || (result
& 0x80000000)
843 || (result
>= 0xD800 && result
<= 0xDFFF))
845 cpp_error (pfile
, DL_ERROR
, "%.*s is not a valid universal character",
846 (int) (str
- base
), base
);
849 else if (identifier_pos
)
851 int validity
= ucn_valid_in_identifier (pfile
, result
);
854 cpp_error (pfile
, DL_ERROR
,
855 "universal character %.*s is not valid in an identifier",
856 (int) (str
- base
), base
);
857 else if (validity
== 2 && identifier_pos
== 1)
858 cpp_error (pfile
, DL_ERROR
,
859 "universal character %.*s is not valid at the start of an identifier",
860 (int) (str
- base
), base
);
869 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
870 it to the execution character set and write the result into TBUF.
871 An advanced pointer is returned. Issues all relevant diagnostics. */
875 convert_ucn (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
876 struct strbuf
*tbuf
, bool wide
)
881 size_t bytesleft
= 6;
883 struct cset_converter cvt
884 = wide
? pfile
->wide_cset_desc
: pfile
->narrow_cset_desc
;
886 from
++; /* skip u/U */
887 ucn
= _cpp_valid_ucn (pfile
, &from
, limit
, 0);
889 rval
= one_cppchar_to_utf8 (ucn
, &bufp
, &bytesleft
);
893 cpp_errno (pfile
, DL_ERROR
, "converting UCN to source character set");
895 else if (!APPLY_CONVERSION (cvt
, buf
, 6 - bytesleft
, tbuf
))
896 cpp_errno (pfile
, DL_ERROR
, "converting UCN to execution character set");
902 emit_numeric_escape (cpp_reader
*pfile
, cppchar_t n
,
903 struct strbuf
*tbuf
, bool wide
)
907 /* We have to render this into the target byte order, which may not
908 be our byte order. */
909 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
910 size_t width
= CPP_OPTION (pfile
, wchar_precision
);
911 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
912 size_t cmask
= width_to_mask (cwidth
);
913 size_t nbwc
= width
/ cwidth
;
915 size_t off
= tbuf
->len
;
918 if (tbuf
->len
+ nbwc
> tbuf
->asize
)
920 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
921 tbuf
->text
= xrealloc (tbuf
->text
, tbuf
->asize
);
924 for (i
= 0; i
< nbwc
; i
++)
928 tbuf
->text
[off
+ (bigend
? nbwc
- i
- 1 : i
)] = c
;
934 if (tbuf
->len
+ 1 > tbuf
->asize
)
936 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
937 tbuf
->text
= xrealloc (tbuf
->text
, tbuf
->asize
);
939 tbuf
->text
[tbuf
->len
++] = n
;
943 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
944 character set and write it into the string buffer TBUF. Returns an
945 advanced pointer, and issues diagnostics as necessary.
946 No character set translation occurs; this routine always produces the
947 execution-set character with numeric value equal to the given hex
948 number. You can, e.g. generate surrogate pairs this way. */
950 convert_hex (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
951 struct strbuf
*tbuf
, bool wide
)
953 cppchar_t c
, n
= 0, overflow
= 0;
954 int digits_found
= 0;
955 size_t width
= (wide
? CPP_OPTION (pfile
, wchar_precision
)
956 : CPP_OPTION (pfile
, char_precision
));
957 size_t mask
= width_to_mask (width
);
959 if (CPP_WTRADITIONAL (pfile
))
960 cpp_error (pfile
, DL_WARNING
,
961 "the meaning of '\\x' is different in traditional C");
963 from
++; /* skip 'x' */
970 overflow
|= n
^ (n
<< 4 >> 4);
971 n
= (n
<< 4) + hex_value (c
);
977 cpp_error (pfile
, DL_ERROR
,
978 "\\x used with no following hex digits");
982 if (overflow
| (n
!= (n
& mask
)))
984 cpp_error (pfile
, DL_PEDWARN
,
985 "hex escape sequence out of range");
989 emit_numeric_escape (pfile
, n
, tbuf
, wide
);
994 /* Convert an octal escape, pointed to by FROM, to the execution
995 character set and write it into the string buffer TBUF. Returns an
996 advanced pointer, and issues diagnostics as necessary.
997 No character set translation occurs; this routine always produces the
998 execution-set character with numeric value equal to the given octal
1000 static const uchar
*
1001 convert_oct (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1002 struct strbuf
*tbuf
, bool wide
)
1006 size_t width
= (wide
? CPP_OPTION (pfile
, wchar_precision
)
1007 : CPP_OPTION (pfile
, char_precision
));
1008 size_t mask
= width_to_mask (width
);
1009 bool overflow
= false;
1011 while (from
< limit
&& count
++ < 3)
1014 if (c
< '0' || c
> '7')
1017 overflow
|= n
^ (n
<< 3 >> 3);
1018 n
= (n
<< 3) + c
- '0';
1021 if (n
!= (n
& mask
))
1023 cpp_error (pfile
, DL_PEDWARN
,
1024 "octal escape sequence out of range");
1028 emit_numeric_escape (pfile
, n
, tbuf
, wide
);
1033 /* Convert an escape sequence (pointed to by FROM) to its value on
1034 the target, and to the execution character set. Do not scan past
1035 LIMIT. Write the converted value into TBUF. Returns an advanced
1036 pointer. Handles all relevant diagnostics. */
1037 static const uchar
*
1038 convert_escape (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1039 struct strbuf
*tbuf
, bool wide
)
1041 /* Values of \a \b \e \f \n \r \t \v respectively. */
1042 #if HOST_CHARSET == HOST_CHARSET_ASCII
1043 static const uchar charconsts
[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
1044 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
1045 static const uchar charconsts
[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
1047 #error "unknown host character set"
1051 struct cset_converter cvt
1052 = wide
? pfile
->wide_cset_desc
: pfile
->narrow_cset_desc
;
1057 /* UCNs, hex escapes, and octal escapes are processed separately. */
1059 return convert_ucn (pfile
, from
, limit
, tbuf
, wide
);
1062 return convert_hex (pfile
, from
, limit
, tbuf
, wide
);
1065 case '0': case '1': case '2': case '3':
1066 case '4': case '5': case '6': case '7':
1067 return convert_oct (pfile
, from
, limit
, tbuf
, wide
);
1069 /* Various letter escapes. Get the appropriate host-charset
1071 case '\\': case '\'': case '"': case '?': break;
1073 case '(': case '{': case '[': case '%':
1074 /* '\(', etc, can be used at the beginning of a line in a long
1075 string split onto multiple lines with \-newline, to prevent
1076 Emacs or other text editors from getting confused. '\%' can
1077 be used to prevent SCCS from mangling printf format strings. */
1078 if (CPP_PEDANTIC (pfile
))
1082 case 'b': c
= charconsts
[1]; break;
1083 case 'f': c
= charconsts
[3]; break;
1084 case 'n': c
= charconsts
[4]; break;
1085 case 'r': c
= charconsts
[5]; break;
1086 case 't': c
= charconsts
[6]; break;
1087 case 'v': c
= charconsts
[7]; break;
1090 if (CPP_WTRADITIONAL (pfile
))
1091 cpp_error (pfile
, DL_WARNING
,
1092 "the meaning of '\\a' is different in traditional C");
1097 if (CPP_PEDANTIC (pfile
))
1098 cpp_error (pfile
, DL_PEDWARN
,
1099 "non-ISO-standard escape sequence, '\\%c'", (int) c
);
1106 cpp_error (pfile
, DL_PEDWARN
,
1107 "unknown escape sequence '\\%c'", (int) c
);
1109 cpp_error (pfile
, DL_PEDWARN
,
1110 "unknown escape sequence: '\\%03o'", (int) c
);
1113 /* Now convert what we have to the execution character set. */
1114 if (!APPLY_CONVERSION (cvt
, &c
, 1, tbuf
))
1115 cpp_errno (pfile
, DL_ERROR
,
1116 "converting escape sequence to execution character set");
1121 /* FROM is an array of cpp_string structures of length COUNT. These
1122 are to be converted from the source to the execution character set,
1123 escape sequences translated, and finally all are to be
1124 concatenated. WIDE indicates whether or not to produce a wide
1125 string. The result is written into TO. Returns true for success,
1126 false for failure. */
1128 cpp_interpret_string (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
1129 cpp_string
*to
, bool wide
)
1132 const uchar
*p
, *base
, *limit
;
1134 struct cset_converter cvt
1135 = wide
? pfile
->wide_cset_desc
: pfile
->narrow_cset_desc
;
1137 tbuf
.asize
= MAX (OUTBUF_BLOCK_SIZE
, from
->len
);
1138 tbuf
.text
= xmalloc (tbuf
.asize
);
1141 for (i
= 0; i
< count
; i
++)
1145 p
++; /* skip leading quote */
1146 limit
= from
[i
].text
+ from
[i
].len
- 1; /* skip trailing quote */
1151 while (p
< limit
&& *p
!= '\\')
1155 /* We have a run of normal characters; these can be fed
1156 directly to convert_cset. */
1157 if (!APPLY_CONVERSION (cvt
, base
, p
- base
, &tbuf
))
1163 p
= convert_escape (pfile
, p
+ 1, limit
, &tbuf
, wide
);
1166 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
1168 emit_numeric_escape (pfile
, 0, &tbuf
, wide
);
1169 tbuf
.text
= xrealloc (tbuf
.text
, tbuf
.len
);
1170 to
->text
= tbuf
.text
;
1175 cpp_errno (pfile
, DL_ERROR
, "converting to execution character set");
1180 /* Subroutine of do_line and do_linemarker. Convert escape sequences
1181 in a string, but do not perform character set conversion. */
1183 _cpp_interpret_string_notranslate (cpp_reader
*pfile
, const cpp_string
*in
,
1186 struct cset_converter save_narrow_cset_desc
= pfile
->narrow_cset_desc
;
1189 pfile
->narrow_cset_desc
.func
= convert_no_conversion
;
1190 pfile
->narrow_cset_desc
.cd
= (iconv_t
) -1;
1192 retval
= cpp_interpret_string (pfile
, in
, 1, out
, false);
1194 pfile
->narrow_cset_desc
= save_narrow_cset_desc
;
1199 /* Subroutine of cpp_interpret_charconst which performs the conversion
1200 to a number, for narrow strings. STR is the string structure returned
1201 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1202 cpp_interpret_charconst. */
1204 narrow_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
1205 unsigned int *pchars_seen
, int *unsignedp
)
1207 size_t width
= CPP_OPTION (pfile
, char_precision
);
1208 size_t max_chars
= CPP_OPTION (pfile
, int_precision
) / width
;
1209 size_t mask
= width_to_mask (width
);
1211 cppchar_t result
, c
;
1214 /* The value of a multi-character character constant, or a
1215 single-character character constant whose representation in the
1216 execution character set is more than one byte long, is
1217 implementation defined. This implementation defines it to be the
1218 number formed by interpreting the byte sequence in memory as a
1219 big-endian binary number. If overflow occurs, the high bytes are
1220 lost, and a warning is issued.
1222 We don't want to process the NUL terminator handed back by
1223 cpp_interpret_string. */
1225 for (i
= 0; i
< str
.len
- 1; i
++)
1227 c
= str
.text
[i
] & mask
;
1228 if (width
< BITS_PER_CPPCHAR_T
)
1229 result
= (result
<< width
) | c
;
1237 cpp_error (pfile
, DL_WARNING
, "character constant too long for its type");
1239 else if (i
> 1 && CPP_OPTION (pfile
, warn_multichar
))
1240 cpp_error (pfile
, DL_WARNING
, "multi-character character constant");
1242 /* Multichar constants are of type int and therefore signed. */
1246 unsigned_p
= CPP_OPTION (pfile
, unsigned_char
);
1248 /* Truncate the constant to its natural width, and simultaneously
1249 sign- or zero-extend to the full width of cppchar_t.
1250 For single-character constants, the value is WIDTH bits wide.
1251 For multi-character constants, the value is INT_PRECISION bits wide. */
1253 width
= CPP_OPTION (pfile
, int_precision
);
1254 if (width
< BITS_PER_CPPCHAR_T
)
1256 mask
= ((cppchar_t
) 1 << width
) - 1;
1257 if (unsigned_p
|| !(result
& (1 << (width
- 1))))
1263 *unsignedp
= unsigned_p
;
1267 /* Subroutine of cpp_interpret_charconst which performs the conversion
1268 to a number, for wide strings. STR is the string structure returned
1269 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1270 cpp_interpret_charconst. */
1272 wide_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
1273 unsigned int *pchars_seen
, int *unsignedp
)
1275 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
1276 size_t width
= CPP_OPTION (pfile
, wchar_precision
);
1277 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
1278 size_t mask
= width_to_mask (width
);
1279 size_t cmask
= width_to_mask (cwidth
);
1280 size_t nbwc
= width
/ cwidth
;
1282 cppchar_t result
= 0, c
;
1284 /* This is finicky because the string is in the target's byte order,
1285 which may not be our byte order. Only the last character, ignoring
1286 the NUL terminator, is relevant. */
1287 off
= str
.len
- (nbwc
* 2);
1289 for (i
= 0; i
< nbwc
; i
++)
1291 c
= bigend
? str
.text
[off
+ i
] : str
.text
[off
+ nbwc
- i
- 1];
1292 result
= (result
<< cwidth
) | (c
& cmask
);
1295 /* Wide character constants have type wchar_t, and a single
1296 character exactly fills a wchar_t, so a multi-character wide
1297 character constant is guaranteed to overflow. */
1299 cpp_error (pfile
, DL_WARNING
, "character constant too long for its type");
1301 /* Truncate the constant to its natural width, and simultaneously
1302 sign- or zero-extend to the full width of cppchar_t. */
1303 if (width
< BITS_PER_CPPCHAR_T
)
1305 if (CPP_OPTION (pfile
, unsigned_wchar
) || !(result
& (1 << (width
- 1))))
1311 *unsignedp
= CPP_OPTION (pfile
, unsigned_wchar
);
1316 /* Interpret a (possibly wide) character constant in TOKEN.
1317 PCHARS_SEEN points to a variable that is filled in with the number
1318 of characters seen, and UNSIGNEDP to a variable that indicates
1319 whether the result has signed type. */
1321 cpp_interpret_charconst (cpp_reader
*pfile
, const cpp_token
*token
,
1322 unsigned int *pchars_seen
, int *unsignedp
)
1324 cpp_string str
= { 0, 0 };
1325 bool wide
= (token
->type
== CPP_WCHAR
);
1328 /* an empty constant will appear as L'' or '' */
1329 if (token
->val
.str
.len
== (size_t) (2 + wide
))
1331 cpp_error (pfile
, DL_ERROR
, "empty character constant");
1334 else if (!cpp_interpret_string (pfile
, &token
->val
.str
, 1, &str
, wide
))
1338 result
= wide_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
);
1340 result
= narrow_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
);
1342 if (str
.text
!= token
->val
.str
.text
)
1343 free ((void *)str
.text
);