Perform "re lexing" on string list values rathern than on text.

Previously, we would pass original strings back to the original lexer
whenever we needed to re-lex something, (such as an expanded macro or
a macro argument). Now, we instead parse the macro or argument
originally to a string list, and then re-lex by simply returning each
string from this list in turn.

We do this in the recently added glcpp_parser_lex function that sits
on top of the lower-level glcpp_lex that only deals with text.

This doesn't change any behavior (at least according to the existing
test suite which all still passes) but it brings us much closer to
being able to "finalize" an unexpanded macro as required by the
specification.

diff --git a/glcpp-lex.l b/glcpp-lex.l
index 52be1b1ea437441bd35353ce6f56ebb3f8e6ba93..aec967964b06c3effc444720526542a73ecc05ef 100644 (file)
--- a/glcpp-lex.l
+++ b/glcpp-lex.l
@@ -84,12 +84,12 @@ TOKEN               [^[:space:](),]+
 
 <ST_DEFINE_OBJ_OR_FUNC>\n {
        BEGIN INITIAL;
-       yylval.str = xtalloc_strdup (yyextra, "");
-       return REPLACEMENT;
+       return NEWLINE;
 }
 
 <ST_DEFINE_OBJ_OR_FUNC>{HSPACE}+ {
        BEGIN ST_DEFINE_VALUE;
+       return SPACE;
 }
 
 <ST_DEFINE_OBJ_OR_FUNC>"(" {
@@ -113,10 +113,21 @@ TOKEN             [^[:space:](),]+
 
 <ST_DEFINE_PARAMETER>{HSPACE}+
 
-<ST_DEFINE_VALUE>.*\n {
+<ST_DEFINE_VALUE>{TOKEN} {
+       yylval.str = xtalloc_strdup (yyextra, yytext);
+       return TOKEN;
+}
+
+<ST_DEFINE_VALUE>[(),] {
+       yylval.str = xtalloc_strdup (yyextra, yytext);
+       return TOKEN;
+}
+
+<ST_DEFINE_VALUE>{HSPACE}+
+
+<ST_DEFINE_VALUE>\n {
        BEGIN INITIAL;
-       yylval.str = xtalloc_strndup (yyextra, yytext, strlen (yytext) - 1);
-       return REPLACEMENT;
+       return NEWLINE;
 }
 
 {IDENTIFIER} {
@@ -161,54 +172,4 @@ TOKEN              [^[:space:](),]+
 
 {HSPACE}+
 
-<<EOF>> {
-       int done;
-
-       done = glcpp_lex_stack_pop (yyextra->lex_stack);
-
-       if (done)
-               yyterminate ();
-
-       glcpp_parser_pop_expansion (yyextra);
-}
-
 %%
-
-void
-glcpp_lex_stack_push (glcpp_lex_stack_t *stack, const char *string)
-{
-       struct yyguts_t *yyg = (struct yyguts_t*) stack->parser->scanner;
-       glcpp_lex_node_t *node;
-
-       /* Save the current buffer on the top of the stack. */
-       node = xtalloc (stack, glcpp_lex_node_t);
-       node->buffer = YY_CURRENT_BUFFER;
-
-       node->next = stack->head;
-       stack->head = node;
-
-       /* Then switch to a new scan buffer for string. */
-       yy_scan_string (string, stack->parser->scanner);
-}
-
-int
-glcpp_lex_stack_pop (glcpp_lex_stack_t *stack)
-{
-       struct yyguts_t *yyg = (struct yyguts_t*) stack->parser->scanner;
-       glcpp_lex_node_t *node;
-
-       node = stack->head;
-
-       if (node == NULL)
-               return 1;
-
-       stack->head = node->next;
-
-       yy_delete_buffer (YY_CURRENT_BUFFER, stack->parser->scanner);
-       yy_switch_to_buffer ((YY_BUFFER_STATE) node->buffer,
-                            stack->parser->scanner);
-
-       talloc_free (node);
-
-       return 0;
-}

diff --git a/glcpp-parse.y b/glcpp-parse.y
index bb57b300982656f4114bb8f006cc282e7edfe704..2383c93117f3a541b339f972b201b0fa4a909ee3 100644 (file)
--- a/glcpp-parse.y
+++ b/glcpp-parse.y
@@ -34,13 +34,13 @@ yyerror (void *scanner, const char *error);
 void
 _define_object_macro (glcpp_parser_t *parser,
                      const char *macro,
-                     const char *replacement);
+                     string_list_t *replacements);
 
 void
 _define_function_macro (glcpp_parser_t *parser,
                        const char *macro,
                        string_list_t *parameters,
-                       const char *replacement);
+                       string_list_t *replacements);
 
 void
 _expand_object_macro (glcpp_parser_t *parser, const char *identifier);
@@ -80,6 +80,14 @@ _argument_list_length (argument_list_t *list);
 string_list_t *
 _argument_list_member_at (argument_list_t *list, int index);
 
+static void
+glcpp_parser_push_expansion_macro (glcpp_parser_t *parser,
+                                  macro_t *macro,
+                                  argument_list_t *arguments);
+
+static void
+glcpp_parser_pop_expansion (glcpp_parser_t *parser);
+
 #define yylex glcpp_parser_lex
 
 static int
@@ -96,9 +104,9 @@ glcpp_parser_lex (glcpp_parser_t *parser);
 %parse-param {glcpp_parser_t *parser}
 %lex-param {glcpp_parser_t *parser}
 
-%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO REPLACEMENT TOKEN UNDEF
-%type <str> argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO REPLACEMENT TOKEN
-%type <string_list> argument macro parameter_list
+%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO NEWLINE SPACE TOKEN UNDEF
+%type <str> argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO TOKEN
+%type <string_list> argument macro parameter_list replacement_list pp_tokens
 %type <argument_list> argument_list
 
 /* Hard to remove shift/reduce conflicts documented as follows:
@@ -194,10 +202,14 @@ argument_word:
 
 
 directive:
-       DEFINE IDENTIFIER REPLACEMENT {
-               _define_object_macro (parser, $2, $3);
+       DEFINE IDENTIFIER NEWLINE {
+               string_list_t *list = _string_list_create (parser);
+               _define_object_macro (parser, $2, list);
        }
-|      DEFINE IDENTIFIER '(' parameter_list ')' REPLACEMENT {
+|      DEFINE IDENTIFIER SPACE replacement_list NEWLINE {
+               _define_object_macro (parser, $2, $4);
+       }
+|      DEFINE IDENTIFIER '(' parameter_list ')' replacement_list NEWLINE {
                _define_function_macro (parser, $2, $4, $6);
        }
 |      UNDEF IDENTIFIER {
@@ -229,6 +241,27 @@ parameter_list:
        }
 ;
 
+replacement_list:
+       /* empty */ {
+               $$ = _string_list_create (parser);
+       }
+|      pp_tokens {
+               $$ = $1;
+       }
+;
+
+
+pp_tokens:
+       TOKEN {
+               $$ = _string_list_create (parser);
+               _string_list_append_item ($$, $1);
+       }
+|      pp_tokens TOKEN {
+               _string_list_append_item ($1, $2);
+               $$ = $1;
+       }
+;
+
 %%
 
 string_list_t *
@@ -413,10 +446,6 @@ glcpp_parser_create (void)
                                           hash_table_string_compare);
        parser->expansions = NULL;
 
-       parser->lex_stack = xtalloc (parser, glcpp_lex_stack_t);
-       parser->lex_stack->parser = parser;
-       parser->lex_stack->head = NULL;
-
        return parser;
 }
 
@@ -495,7 +524,7 @@ glcpp_parser_classify_token (glcpp_parser_t *parser,
 void
 _define_object_macro (glcpp_parser_t *parser,
                      const char *identifier,
-                     const char *replacement)
+                     string_list_t *replacements)
 {
        macro_t *macro;
 
@@ -504,7 +533,7 @@ _define_object_macro (glcpp_parser_t *parser,
        macro->is_function = 0;
        macro->parameters = NULL;
        macro->identifier = talloc_strdup (macro, identifier);
-       macro->replacement = talloc_steal (macro, replacement);
+       macro->replacements = talloc_steal (macro, replacements);
 
        hash_table_insert (parser->defines, macro, identifier);
 }
@@ -513,7 +542,7 @@ void
 _define_function_macro (glcpp_parser_t *parser,
                        const char *identifier,
                        string_list_t *parameters,
-                       const char *replacement)
+                       string_list_t *replacements)
 {
        macro_t *macro;
 
@@ -522,7 +551,7 @@ _define_function_macro (glcpp_parser_t *parser,
        macro->is_function = 1;
        macro->parameters = talloc_steal (macro, parameters);
        macro->identifier = talloc_strdup (macro, identifier);
-       macro->replacement = talloc_steal (macro, replacement);
+       macro->replacements = talloc_steal (macro, replacements);
 
        hash_table_insert (parser->defines, macro, identifier);
 }
@@ -531,7 +560,7 @@ static void
 _glcpp_parser_push_expansion_internal (glcpp_parser_t *parser,
                                       macro_t *macro,
                                       argument_list_t *arguments,
-                                      const char * replacement)
+                                      string_node_t *replacements)
 {
        expansion_node_t *node;
 
@@ -539,20 +568,19 @@ _glcpp_parser_push_expansion_internal (glcpp_parser_t *parser,
 
        node->macro = macro;
        node->arguments = arguments;
+       node->replacements = replacements;
 
        node->next = parser->expansions;
        parser->expansions = node;
-               
-       glcpp_lex_stack_push (parser->lex_stack, replacement);
 }
 
-void
+static void
 glcpp_parser_push_expansion_macro (glcpp_parser_t *parser,
                                   macro_t *macro,
                                   argument_list_t *arguments)
 {
        _glcpp_parser_push_expansion_internal (parser, macro, arguments,
-                                              macro->replacement);
+                                              macro->replacements->head);
 }
 
 void
@@ -561,38 +589,16 @@ glcpp_parser_push_expansion_argument (glcpp_parser_t *parser,
 {
        argument_list_t *arguments;
        string_list_t *argument;
-       string_node_t *node;
-       char *argument_str, *s;
-       int length;
 
        arguments = parser->expansions->arguments;
 
        argument = _argument_list_member_at (arguments, argument_index);
 
-       length = 0;
-       for (node = argument->head; node; node = node->next)
-               length += strlen (node->str) + 1;
-
-       argument_str = xtalloc_size (parser, length);
-
-       *argument_str = '\0';
-       s = argument_str;
-       for (node = argument->head; node; node = node->next) {
-               strcpy (s, node->str);
-               s += strlen (node->str);
-               if (node->next) {
-                       *s = ' ';
-                       s++;
-                       *s = '\0';
-               }
-       }
-
        _glcpp_parser_push_expansion_internal (parser, NULL, NULL,
-                                              argument_str);
+                                              argument->head);
 }
 
-/* The lexer calls this when it exhausts a string. */
-void
+static void
 glcpp_parser_pop_expansion (glcpp_parser_t *parser)
 {
        expansion_node_t *node;
@@ -649,5 +655,55 @@ _expand_function_macro (glcpp_parser_t *parser,
 static int
 glcpp_parser_lex (glcpp_parser_t *parser)
 {
-       return glcpp_lex (parser->scanner);
+       expansion_node_t *expansion;
+       string_node_t *replacements;
+       int parameter_index;
+
+    /* Who says C can't do efficient tail recursion? */
+    RECURSE:
+
+       expansion = parser->expansions;
+
+       if (expansion == NULL)
+               return glcpp_lex (parser->scanner);
+
+       replacements = expansion->replacements;
+
+       /* Pop expansion when replacements is exhausted. */
+       if (replacements == NULL) {
+               glcpp_parser_pop_expansion (parser);
+               goto RECURSE;
+       }
+
+       expansion->replacements = replacements->next;
+
+       if (strcmp (replacements->str, "(") == 0)
+               return '(';
+       else if (strcmp (replacements->str, ")") == 0)
+               return ')';
+       else if (strcmp (replacements->str, ",") == 0)
+               return ',';
+
+       yylval.str = xtalloc_strdup (parser, replacements->str);
+
+       switch (glcpp_parser_classify_token (parser, yylval.str,
+                                            &parameter_index))
+       {
+       case TOKEN_CLASS_ARGUMENT:
+               talloc_free (yylval.str);
+               glcpp_parser_push_expansion_argument (parser,
+                                                     parameter_index);
+               goto RECURSE;
+               break;
+       case TOKEN_CLASS_IDENTIFIER:
+               return IDENTIFIER;
+               break;
+       case TOKEN_CLASS_FUNC_MACRO:
+               return FUNC_MACRO;
+               break;
+       default:
+       case TOKEN_CLASS_OBJ_MACRO:
+               return OBJ_MACRO;
+               break;
+       }
 }

diff --git a/glcpp.h b/glcpp.h
index 6aabf6f18238ba2e1b7c882fa3b20fc97d22ec94..ef821a7637acc9646498f060d5a237445c967737 100644 (file)
--- a/glcpp.h
+++ b/glcpp.h
@@ -54,24 +54,6 @@ typedef struct argument_list {
 
 typedef struct glcpp_parser glcpp_parser_t;
 
-/* Support for temporarily lexing/parsing tokens from a string. */
-
-typedef struct glcpp_lex_node {
-       void *buffer;
-       struct glcpp_lex_node *next;
-} glcpp_lex_node_t;
-
-typedef struct {
-       glcpp_parser_t *parser;
-       glcpp_lex_node_t *head;
-} glcpp_lex_stack_t;
-
-void
-glcpp_lex_stack_push (glcpp_lex_stack_t *stack, const char *string);
-
-int
-glcpp_lex_stack_pop (glcpp_lex_stack_t *stack);
-
 typedef enum {
        TOKEN_CLASS_ARGUMENT,
        TOKEN_CLASS_IDENTIFIER,
@@ -88,12 +70,13 @@ typedef struct {
        int is_function;
        string_list_t *parameters;
        const char *identifier;
-       const char *replacement;
+       string_list_t *replacements;
 } macro_t;
 
 typedef struct expansion_node {
        macro_t *macro;
        argument_list_t *arguments;
+       string_node_t *replacements;
        struct expansion_node *next;
 } expansion_node_t;
 
@@ -101,9 +84,12 @@ struct glcpp_parser {
        yyscan_t scanner;
        struct hash_table *defines;
        expansion_node_t *expansions;
-       glcpp_lex_stack_t *lex_stack;
 };
 
+void
+glcpp_parser_push_expansion_argument (glcpp_parser_t *parser,
+                                     int argument_index);
+
 glcpp_parser_t *
 glcpp_parser_create (void);
 
@@ -113,18 +99,6 @@ glcpp_parser_parse (glcpp_parser_t *parser);
 void
 glcpp_parser_destroy (glcpp_parser_t *parser);
 
-void
-glcpp_parser_push_expansion_macro (glcpp_parser_t *parser,
-                                  macro_t *macro,
-                                  argument_list_t *arguments);
-
-void
-glcpp_parser_push_expansion_argument (glcpp_parser_t *parser,
-                                     int argument_index);
-
-void
-glcpp_parser_pop_expansion (glcpp_parser_t *parser);
-
 /* Generated by glcpp-lex.l to glcpp-lex.c */
 
 int