From: Carl Worth Date: Wed, 19 May 2010 20:28:24 +0000 (-0700) Subject: Perform "re lexing" on string list values rathern than on text. X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=aaa9acbf10b7a8e7dac061885ef95823ad27f80e;p=mesa.git Perform "re lexing" on string list values rathern than on text. Previously, we would pass original strings back to the original lexer whenever we needed to re-lex something, (such as an expanded macro or a macro argument). Now, we instead parse the macro or argument originally to a string list, and then re-lex by simply returning each string from this list in turn. We do this in the recently added glcpp_parser_lex function that sits on top of the lower-level glcpp_lex that only deals with text. This doesn't change any behavior (at least according to the existing test suite which all still passes) but it brings us much closer to being able to "finalize" an unexpanded macro as required by the specification. --- diff --git a/glcpp-lex.l b/glcpp-lex.l index 52be1b1ea43..aec967964b0 100644 --- a/glcpp-lex.l +++ b/glcpp-lex.l @@ -84,12 +84,12 @@ TOKEN [^[:space:](),]+ \n { BEGIN INITIAL; - yylval.str = xtalloc_strdup (yyextra, ""); - return REPLACEMENT; + return NEWLINE; } {HSPACE}+ { BEGIN ST_DEFINE_VALUE; + return SPACE; } "(" { @@ -113,10 +113,21 @@ TOKEN [^[:space:](),]+ {HSPACE}+ -.*\n { +{TOKEN} { + yylval.str = xtalloc_strdup (yyextra, yytext); + return TOKEN; +} + +[(),] { + yylval.str = xtalloc_strdup (yyextra, yytext); + return TOKEN; +} + +{HSPACE}+ + +\n { BEGIN INITIAL; - yylval.str = xtalloc_strndup (yyextra, yytext, strlen (yytext) - 1); - return REPLACEMENT; + return NEWLINE; } {IDENTIFIER} { @@ -161,54 +172,4 @@ TOKEN [^[:space:](),]+ {HSPACE}+ -<> { - int done; - - done = glcpp_lex_stack_pop (yyextra->lex_stack); - - if (done) - yyterminate (); - - glcpp_parser_pop_expansion (yyextra); -} - %% - -void -glcpp_lex_stack_push (glcpp_lex_stack_t *stack, const char *string) -{ - struct yyguts_t *yyg = (struct yyguts_t*) stack->parser->scanner; - glcpp_lex_node_t *node; - - /* Save the current buffer on the top of the stack. */ - node = xtalloc (stack, glcpp_lex_node_t); - node->buffer = YY_CURRENT_BUFFER; - - node->next = stack->head; - stack->head = node; - - /* Then switch to a new scan buffer for string. */ - yy_scan_string (string, stack->parser->scanner); -} - -int -glcpp_lex_stack_pop (glcpp_lex_stack_t *stack) -{ - struct yyguts_t *yyg = (struct yyguts_t*) stack->parser->scanner; - glcpp_lex_node_t *node; - - node = stack->head; - - if (node == NULL) - return 1; - - stack->head = node->next; - - yy_delete_buffer (YY_CURRENT_BUFFER, stack->parser->scanner); - yy_switch_to_buffer ((YY_BUFFER_STATE) node->buffer, - stack->parser->scanner); - - talloc_free (node); - - return 0; -} diff --git a/glcpp-parse.y b/glcpp-parse.y index bb57b300982..2383c93117f 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -34,13 +34,13 @@ yyerror (void *scanner, const char *error); void _define_object_macro (glcpp_parser_t *parser, const char *macro, - const char *replacement); + string_list_t *replacements); void _define_function_macro (glcpp_parser_t *parser, const char *macro, string_list_t *parameters, - const char *replacement); + string_list_t *replacements); void _expand_object_macro (glcpp_parser_t *parser, const char *identifier); @@ -80,6 +80,14 @@ _argument_list_length (argument_list_t *list); string_list_t * _argument_list_member_at (argument_list_t *list, int index); +static void +glcpp_parser_push_expansion_macro (glcpp_parser_t *parser, + macro_t *macro, + argument_list_t *arguments); + +static void +glcpp_parser_pop_expansion (glcpp_parser_t *parser); + #define yylex glcpp_parser_lex static int @@ -96,9 +104,9 @@ glcpp_parser_lex (glcpp_parser_t *parser); %parse-param {glcpp_parser_t *parser} %lex-param {glcpp_parser_t *parser} -%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO REPLACEMENT TOKEN UNDEF -%type argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO REPLACEMENT TOKEN -%type argument macro parameter_list +%token DEFINE FUNC_MACRO IDENTIFIER OBJ_MACRO NEWLINE SPACE TOKEN UNDEF +%type argument_word FUNC_MACRO IDENTIFIER OBJ_MACRO TOKEN +%type argument macro parameter_list replacement_list pp_tokens %type argument_list /* Hard to remove shift/reduce conflicts documented as follows: @@ -194,10 +202,14 @@ argument_word: directive: - DEFINE IDENTIFIER REPLACEMENT { - _define_object_macro (parser, $2, $3); + DEFINE IDENTIFIER NEWLINE { + string_list_t *list = _string_list_create (parser); + _define_object_macro (parser, $2, list); } -| DEFINE IDENTIFIER '(' parameter_list ')' REPLACEMENT { +| DEFINE IDENTIFIER SPACE replacement_list NEWLINE { + _define_object_macro (parser, $2, $4); + } +| DEFINE IDENTIFIER '(' parameter_list ')' replacement_list NEWLINE { _define_function_macro (parser, $2, $4, $6); } | UNDEF IDENTIFIER { @@ -229,6 +241,27 @@ parameter_list: } ; +replacement_list: + /* empty */ { + $$ = _string_list_create (parser); + } +| pp_tokens { + $$ = $1; + } +; + + +pp_tokens: + TOKEN { + $$ = _string_list_create (parser); + _string_list_append_item ($$, $1); + } +| pp_tokens TOKEN { + _string_list_append_item ($1, $2); + $$ = $1; + } +; + %% string_list_t * @@ -413,10 +446,6 @@ glcpp_parser_create (void) hash_table_string_compare); parser->expansions = NULL; - parser->lex_stack = xtalloc (parser, glcpp_lex_stack_t); - parser->lex_stack->parser = parser; - parser->lex_stack->head = NULL; - return parser; } @@ -495,7 +524,7 @@ glcpp_parser_classify_token (glcpp_parser_t *parser, void _define_object_macro (glcpp_parser_t *parser, const char *identifier, - const char *replacement) + string_list_t *replacements) { macro_t *macro; @@ -504,7 +533,7 @@ _define_object_macro (glcpp_parser_t *parser, macro->is_function = 0; macro->parameters = NULL; macro->identifier = talloc_strdup (macro, identifier); - macro->replacement = talloc_steal (macro, replacement); + macro->replacements = talloc_steal (macro, replacements); hash_table_insert (parser->defines, macro, identifier); } @@ -513,7 +542,7 @@ void _define_function_macro (glcpp_parser_t *parser, const char *identifier, string_list_t *parameters, - const char *replacement) + string_list_t *replacements) { macro_t *macro; @@ -522,7 +551,7 @@ _define_function_macro (glcpp_parser_t *parser, macro->is_function = 1; macro->parameters = talloc_steal (macro, parameters); macro->identifier = talloc_strdup (macro, identifier); - macro->replacement = talloc_steal (macro, replacement); + macro->replacements = talloc_steal (macro, replacements); hash_table_insert (parser->defines, macro, identifier); } @@ -531,7 +560,7 @@ static void _glcpp_parser_push_expansion_internal (glcpp_parser_t *parser, macro_t *macro, argument_list_t *arguments, - const char * replacement) + string_node_t *replacements) { expansion_node_t *node; @@ -539,20 +568,19 @@ _glcpp_parser_push_expansion_internal (glcpp_parser_t *parser, node->macro = macro; node->arguments = arguments; + node->replacements = replacements; node->next = parser->expansions; parser->expansions = node; - - glcpp_lex_stack_push (parser->lex_stack, replacement); } -void +static void glcpp_parser_push_expansion_macro (glcpp_parser_t *parser, macro_t *macro, argument_list_t *arguments) { _glcpp_parser_push_expansion_internal (parser, macro, arguments, - macro->replacement); + macro->replacements->head); } void @@ -561,38 +589,16 @@ glcpp_parser_push_expansion_argument (glcpp_parser_t *parser, { argument_list_t *arguments; string_list_t *argument; - string_node_t *node; - char *argument_str, *s; - int length; arguments = parser->expansions->arguments; argument = _argument_list_member_at (arguments, argument_index); - length = 0; - for (node = argument->head; node; node = node->next) - length += strlen (node->str) + 1; - - argument_str = xtalloc_size (parser, length); - - *argument_str = '\0'; - s = argument_str; - for (node = argument->head; node; node = node->next) { - strcpy (s, node->str); - s += strlen (node->str); - if (node->next) { - *s = ' '; - s++; - *s = '\0'; - } - } - _glcpp_parser_push_expansion_internal (parser, NULL, NULL, - argument_str); + argument->head); } -/* The lexer calls this when it exhausts a string. */ -void +static void glcpp_parser_pop_expansion (glcpp_parser_t *parser) { expansion_node_t *node; @@ -649,5 +655,55 @@ _expand_function_macro (glcpp_parser_t *parser, static int glcpp_parser_lex (glcpp_parser_t *parser) { - return glcpp_lex (parser->scanner); + expansion_node_t *expansion; + string_node_t *replacements; + int parameter_index; + + /* Who says C can't do efficient tail recursion? */ + RECURSE: + + expansion = parser->expansions; + + if (expansion == NULL) + return glcpp_lex (parser->scanner); + + replacements = expansion->replacements; + + /* Pop expansion when replacements is exhausted. */ + if (replacements == NULL) { + glcpp_parser_pop_expansion (parser); + goto RECURSE; + } + + expansion->replacements = replacements->next; + + if (strcmp (replacements->str, "(") == 0) + return '('; + else if (strcmp (replacements->str, ")") == 0) + return ')'; + else if (strcmp (replacements->str, ",") == 0) + return ','; + + yylval.str = xtalloc_strdup (parser, replacements->str); + + switch (glcpp_parser_classify_token (parser, yylval.str, + ¶meter_index)) + { + case TOKEN_CLASS_ARGUMENT: + talloc_free (yylval.str); + glcpp_parser_push_expansion_argument (parser, + parameter_index); + goto RECURSE; + break; + case TOKEN_CLASS_IDENTIFIER: + return IDENTIFIER; + break; + case TOKEN_CLASS_FUNC_MACRO: + return FUNC_MACRO; + break; + default: + case TOKEN_CLASS_OBJ_MACRO: + return OBJ_MACRO; + break; + } } diff --git a/glcpp.h b/glcpp.h index 6aabf6f1823..ef821a7637a 100644 --- a/glcpp.h +++ b/glcpp.h @@ -54,24 +54,6 @@ typedef struct argument_list { typedef struct glcpp_parser glcpp_parser_t; -/* Support for temporarily lexing/parsing tokens from a string. */ - -typedef struct glcpp_lex_node { - void *buffer; - struct glcpp_lex_node *next; -} glcpp_lex_node_t; - -typedef struct { - glcpp_parser_t *parser; - glcpp_lex_node_t *head; -} glcpp_lex_stack_t; - -void -glcpp_lex_stack_push (glcpp_lex_stack_t *stack, const char *string); - -int -glcpp_lex_stack_pop (glcpp_lex_stack_t *stack); - typedef enum { TOKEN_CLASS_ARGUMENT, TOKEN_CLASS_IDENTIFIER, @@ -88,12 +70,13 @@ typedef struct { int is_function; string_list_t *parameters; const char *identifier; - const char *replacement; + string_list_t *replacements; } macro_t; typedef struct expansion_node { macro_t *macro; argument_list_t *arguments; + string_node_t *replacements; struct expansion_node *next; } expansion_node_t; @@ -101,9 +84,12 @@ struct glcpp_parser { yyscan_t scanner; struct hash_table *defines; expansion_node_t *expansions; - glcpp_lex_stack_t *lex_stack; }; +void +glcpp_parser_push_expansion_argument (glcpp_parser_t *parser, + int argument_index); + glcpp_parser_t * glcpp_parser_create (void); @@ -113,18 +99,6 @@ glcpp_parser_parse (glcpp_parser_t *parser); void glcpp_parser_destroy (glcpp_parser_t *parser); -void -glcpp_parser_push_expansion_macro (glcpp_parser_t *parser, - macro_t *macro, - argument_list_t *arguments); - -void -glcpp_parser_push_expansion_argument (glcpp_parser_t *parser, - int argument_index); - -void -glcpp_parser_pop_expansion (glcpp_parser_t *parser); - /* Generated by glcpp-lex.l to glcpp-lex.c */ int