cpplex.c: Remove conditional #undef of MULTIBYTE_CHARS.

2002-04-24  Matt Hiller  <hiller@redhat.com>

	* cpplex.c: Remove conditional #undef of MULTIBYTE_CHARS.
	* c-lex.c: Ditto.

	* cpplex.c (skip_line_comment): Process comment one multibyte
	character at a time rather than one char at a time, if
	appropriate.
	(parse_string): Process string one multibyte character at a time
	rather than one char at a time, if appropriate.
	* c-lex.c (lex_string): Lex and copy multibyte strings
	appropriately.
	* cpplib.h (cppchar_t): Change to unsigned.

From-SVN: r52737

diff --git a/gcc/c-lex.c b/gcc/c-lex.c
index 8a76dd35c315760702c461f32795136adb81fd1b..f2f8c8c7317daf626a8bfbe2b418fa64833969e2 100644 (file)
--- a/gcc/c-lex.c
+++ b/gcc/c-lex.c
@@ -40,13 +40,6 @@ Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include "splay-tree.h"
 #include "debug.h"
 
-/* MULTIBYTE_CHARS support only works for native compilers.
-   ??? Ideally what we want is to model widechar support after
-   the current floating point support.  */
-#ifdef CROSS_COMPILE
-#undef MULTIBYTE_CHARS
-#endif
-
 #ifdef MULTIBYTE_CHARS
 #include "mbchar.h"
 #include <locale.h>
@@ -1282,8 +1275,8 @@ lex_string (str, len, wide)
          c = cpp_parse_escape (parse_in, &p, limit, mask);
        }
        
-      /* Add this single character into the buffer either as a wchar_t
-        or as a single byte.  */
+      /* Add this single character into the buffer either as a wchar_t,
+        a multibyte sequence, or as a single byte.  */
       if (wide)
        {
          unsigned charwidth = TYPE_PRECISION (char_type_node);
@@ -1304,6 +1297,16 @@ lex_string (str, len, wide)
            }
          q += WCHAR_BYTES;
        }
+#ifdef MULTIBYTE_CHARS
+      else if (char_len > 1)
+       {
+         /* We're dealing with a multibyte character. */
+         for ( ; char_len >0; --char_len)
+           {
+             *q++ = *(p - char_len);
+           }
+       }
+#endif
       else
        {
          *q++ = c;

diff --git a/gcc/cpplex.c b/gcc/cpplex.c
index eed9ffda2115f59b982f38c90bf62b2b0d554432..6a44b86d15f944b115c903ff4a49516daece6c75 100644 (file)
--- a/gcc/cpplex.c
+++ b/gcc/cpplex.c
@@ -25,13 +25,6 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
 #include "cpplib.h"
 #include "cpphash.h"
 
-/* MULTIBYTE_CHARS support only works for native compilers.
-   ??? Ideally what we want is to model widechar support after
-   the current floating point support.  */
-#ifdef CROSS_COMPILE
-#undef MULTIBYTE_CHARS
-#endif
-
 #ifdef MULTIBYTE_CHARS
 #include "mbchar.h"
 #include <locale.h>
@@ -312,14 +305,39 @@ skip_line_comment (pfile)
   cpp_buffer *buffer = pfile->buffer;
   unsigned int orig_line = pfile->line;
   cppchar_t c;
+#ifdef MULTIBYTE_CHARS
+  wchar_t wc;
+  int char_len;
+#endif
 
   pfile->state.lexing_comment = 1;
+#ifdef MULTIBYTE_CHARS
+  /* Reset multibyte conversion state.  */
+  (void) local_mbtowc (NULL, NULL, 0);
+#endif
   do
     {
       if (buffer->cur == buffer->rlimit)
        goto at_eof;
 
+#ifdef MULTIBYTE_CHARS
+      char_len = local_mbtowc (&wc, (const char *) buffer->cur,
+                              buffer->rlimit - buffer->cur);
+      if (char_len == -1)
+       {
+         cpp_error (pfile, DL_WARNING,
+                    "ignoring invalid multibyte character");
+         char_len = 1;
+         c = *buffer->cur++;
+       }
+      else
+       {
+         buffer->cur += char_len;
+         c = wc;
+       }
+#else
       c = *buffer->cur++;
+#endif
       if (c == '?' || c == '\\')
        c = skip_escaped_newlines (pfile);
     }
@@ -617,10 +635,18 @@ parse_string (pfile, token, terminator)
   unsigned char *dest, *limit;
   cppchar_t c;
   bool warned_nulls = false;
+#ifdef MULTIBYTE_CHARS
+  wchar_t wc;
+  int char_len;
+#endif
 
   dest = BUFF_FRONT (pfile->u_buff);
   limit = BUFF_LIMIT (pfile->u_buff);
 
+#ifdef MULTIBYTE_CHARS
+  /* Reset multibyte conversion state.  */
+  (void) local_mbtowc (NULL, NULL, 0);
+#endif
   for (;;)
     {
       /* We need room for another char, possibly the terminating NUL.  */
@@ -632,8 +658,26 @@ parse_string (pfile, token, terminator)
          limit = BUFF_LIMIT (pfile->u_buff);
        }
 
-      /* Handle trigraphs, escaped newlines etc.  */
+#ifdef MULTIBYTE_CHARS
+      char_len = local_mbtowc (&wc, (const char *) buffer->cur,
+                              buffer->rlimit - buffer->cur);
+      if (char_len == -1)
+       {
+         cpp_error (pfile, DL_WARNING,
+                      "ignoring invalid multibyte character");
+         char_len = 1;
+         c = *buffer->cur++;
+       }
+      else
+       {
+         buffer->cur += char_len;
+         c = wc;
+       }
+#else
       c = *buffer->cur++;
+#endif
+
+      /* Handle trigraphs, escaped newlines etc.  */
       if (c == '?' || c == '\\')
        c = skip_escaped_newlines (pfile);
 
@@ -666,8 +710,15 @@ parse_string (pfile, token, terminator)
                         "null character(s) preserved in literal");
            }
        }
-
-      *dest++ = c;
+#ifdef MULTIBYTE_CHARS
+      if (char_len > 1)
+       {
+         for ( ; char_len > 0; --char_len)
+           *dest++ = (*buffer->cur - char_len);
+       }
+      else
+#endif
+       *dest++ = c;
     }
 
   *dest = '\0';

diff --git a/gcc/cpplib.h b/gcc/cpplib.h
index 359326e0138384e2ce21e734f01e43c85a37e9ea..765d65a3869485788bb36e452aa052e6f6892f18 100644 (file)
--- a/gcc/cpplib.h
+++ b/gcc/cpplib.h
@@ -187,9 +187,9 @@ struct cpp_token
   } val;
 };
 
-/* A standalone character.  We may want to make it unsigned for the
-   same reason we use unsigned char - to avoid signedness issues.  */
-typedef int cppchar_t;
+/* A standalone character.  It is unsigned for the same reason we use
+   unsigned char - to avoid signedness issues.  */
+typedef unsigned int cppchar_t;
 
 /* Values for opts.dump_macros.
   dump_only means inhibit output of the preprocessed text