Collapse multiple spaces in input down to a single space.

This is what gcc does, and it's actually less work to do
this. Previously we were having to save the contents of space tokens
as a string, but we don't need to do that now.

We extend test #0 to exercise this feature here.

diff --git a/glcpp-lex.l b/glcpp-lex.l
index f6d0c8b7d672c51647a55b0f3c53d1eba2fdebe3..516f42dee3254a8ab73d20c96c8a10357601c539 100644 (file)
--- a/glcpp-lex.l
+++ b/glcpp-lex.l
@@ -119,7 +119,6 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
 
 {HSPACE}+ {
        if (yyextra->space_tokens) {
-               yylval.str = xtalloc_strdup (yyextra, yytext);
                return SPACE;
        }
 }

diff --git a/glcpp-parse.y b/glcpp-parse.y
index a1981995fd0f897509ee39f1ffe1b75c674b85cb..0460f71f74688a314084fd632709bd3893de7d7d 100644 (file)
--- a/glcpp-parse.y
+++ b/glcpp-parse.y
@@ -132,8 +132,8 @@ glcpp_parser_lex (glcpp_parser_t *parser);
 
 %token HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_UNDEF IDENTIFIER NEWLINE OTHER SPACE
 %token LEFT_SHIFT RIGHT_SHIFT LESS_OR_EQUAL GREATER_OR_EQUAL EQUAL NOT_EQUAL AND OR PASTE
-%type <ival> punctuator
-%type <str> IDENTIFIER OTHER SPACE
+%type <ival> punctuator SPACE
+%type <str> IDENTIFIER OTHER
 %type <string_list> identifier_list
 %type <token> preprocessing_token
 %type <token_list> pp_tokens replacement_list text_line
@@ -235,7 +235,7 @@ preprocessing_token:
                $$ = _token_create_str (parser, OTHER, $1);
        }
 |      SPACE {
-               $$ = _token_create_str (parser, SPACE, $1);     
+               $$ = _token_create_ival (parser, SPACE, SPACE);
        }
 ;
 
@@ -495,9 +495,11 @@ _token_print (token_t *token)
        switch (token->type) {
        case IDENTIFIER:
        case OTHER:
-       case SPACE:
                printf ("%s", token->value.str);
                break;
+       case SPACE:
+               printf (" ");
+               break;
        case LEFT_SHIFT:
                printf ("<<");
                break;

diff --git a/tests/000-content-with-spaces.c b/tests/000-content-with-spaces.c
index a7fc918c90856fb2c6d8579b40afca5cb32b5cfd..696cb3a74fc2003863adb1157c5715e269fb3c3b 100644 (file)
--- a/tests/000-content-with-spaces.c
+++ b/tests/000-content-with-spaces.c
@@ -1 +1 @@
-this is four tokens
+this is  four  tokens