src/compiler/glsl/glcpp/glcpp-lex.l

   1 %{
   2 /*
   3  * Copyright © 2010 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <ctype.h>
  28
  29 #include "glcpp.h"
  30 #include "glcpp-parse.h"
  31
  32 /* Flex annoyingly generates some functions without making them
  33  * static. Let's declare them here. */
  34 int glcpp_get_column  (yyscan_t yyscanner);
  35 void glcpp_set_column (int  column_no , yyscan_t yyscanner);
  36
  37 #ifdef _MSC_VER
  38 #define YY_NO_UNISTD_H
  39 #endif
  40
  41 #define YY_NO_INPUT
  42
  43 #define YY_USER_ACTION                                                  \
  44         do {                                                            \
  45                 if (parser->has_new_line_number)                        \
  46                         yylineno = parser->new_line_number;             \
  47                 if (parser->has_new_source_number)                      \
  48                         yylloc->source = parser->new_source_number;     \
  49                 yylloc->first_column = yycolumn + 1;                    \
  50                 yylloc->first_line = yylloc->last_line = yylineno;      \
  51                 yycolumn += yyleng;                                     \
  52                 yylloc->last_column = yycolumn + 1;                     \
  53                 parser->has_new_line_number = 0;                        \
  54                 parser->has_new_source_number = 0;                      \
  55         } while(0);
  56
  57 #define YY_USER_INIT                    \
  58         do {                            \
  59                 yylineno = 1;           \
  60                 yycolumn = 0;           \
  61                 yylloc->source = 0;     \
  62         } while(0)
  63
  64 /* It's ugly to have macros that have return statements inside of
  65  * them, but flex-based lexer generation is all built around the
  66  * return statement.
  67  *
  68  * To mitigate the ugliness, we defer as much of the logic as possible
  69  * to an actual function, not a macro (see
  70  * glcpplex_update_state_per_token) and we make the word RETURN
  71  * prominent in all of the macros which may return.
  72  *
  73  * The most-commonly-used macro is RETURN_TOKEN which will perform all
  74  * necessary state updates based on the provided token,, then
  75  * conditionally return the token. It will not return a token if the
  76  * parser is currently skipping tokens, (such as within #if
  77  * 0...#else).
  78  *
  79  * The RETURN_TOKEN_NEVER_SKIP macro is a lower-level variant that
  80  * makes the token returning unconditional. This is needed for things
  81  * like #if and the tokens of its condition, (since these must be
  82  * evaluated by the parser even when otherwise skipping).
  83  *
  84  * Finally, RETURN_STRING_TOKEN is a simple convenience wrapper on top
  85  * of RETURN_TOKEN that performs a string copy of yytext before the
  86  * return.
  87  */
  88 #define RETURN_TOKEN_NEVER_SKIP(token)                                  \
  89         do {                                                            \
  90                 if (glcpp_lex_update_state_per_token (parser, token))   \
  91                         return token;                                   \
  92         } while (0)
  93
  94 #define RETURN_TOKEN(token)                                             \
  95         do {                                                            \
  96                 if (! parser->skipping) {                               \
  97                         RETURN_TOKEN_NEVER_SKIP(token);                 \
  98                 }                                                       \
  99         } while(0)
 100
 101 #define RETURN_STRING_TOKEN(token)                                      \
 102         do {                                                            \
 103                 if (! parser->skipping) {                               \
 104                         yylval->str = ralloc_strdup (yyextra, yytext);  \
 105                         RETURN_TOKEN_NEVER_SKIP (token);                \
 106                 }                                                       \
 107         } while(0)
 108
 109
 110 /* Update all state necessary for each token being returned.
 111  *
 112  * Here we'll be tracking newlines and spaces so that the lexer can
 113  * alter its behavior as necessary, (for example, '#' has special
 114  * significance if it is the first non-whitespace, non-comment token
 115  * in a line, but does not otherwise).
 116  *
 117  * NOTE: If this function returns FALSE, then no token should be
 118  * returned at all. This is used to suprress duplicate SPACE tokens.
 119  */
 120 static int
 121 glcpp_lex_update_state_per_token (glcpp_parser_t *parser, int token)
 122 {
 123         if (token != NEWLINE && token != SPACE && token != HASH_TOKEN &&
 124             !parser->lexing_version_directive) {
 125                 glcpp_parser_resolve_implicit_version(parser);
 126         }
 127
 128         /* After the first non-space token in a line, we won't
 129          * allow any '#' to introduce a directive. */
 130         if (token == NEWLINE) {
 131                 parser->first_non_space_token_this_line = 1;
 132         } else if (token != SPACE) {
 133                 parser->first_non_space_token_this_line = 0;
 134         }
 135
 136         /* Track newlines just to know whether a newline needs
 137          * to be inserted if end-of-file comes early. */
 138         if (token == NEWLINE) {
 139                 parser->last_token_was_newline = 1;
 140         } else {
 141                 parser->last_token_was_newline = 0;
 142         }
 143
 144         /* Track spaces to avoid emitting multiple SPACE
 145          * tokens in a row. */
 146         if (token == SPACE) {
 147                 if (! parser->last_token_was_space) {
 148                         parser->last_token_was_space = 1;
 149                         return 1;
 150                 } else {
 151                         parser->last_token_was_space = 1;
 152                         return 0;
 153                 }
 154         } else {
 155                 parser->last_token_was_space = 0;
 156                 return 1;
 157         }
 158 }
 159
 160
 161 %}
 162
 163 %option bison-bridge bison-locations reentrant noyywrap
 164 %option extra-type="glcpp_parser_t *"
 165 %option prefix="glcpp_"
 166 %option stack
 167 %option never-interactive
 168 %option warn nodefault
 169
 170         /* Note: When adding any start conditions to this list, you must also
 171          * update the "Internal compiler error" catch-all rule near the end of
 172          * this file. */
 173
 174 %x COMMENT DEFINE DONE HASH NEWLINE_CATCHUP UNREACHABLE
 175
 176 SPACE           [[:space:]]
 177 NONSPACE        [^[:space:]]
 178 HSPACE          [ \t]
 179 HASH            #
 180 NEWLINE         (\r\n|\n\r|\r|\n)
 181 IDENTIFIER      [_a-zA-Z][_a-zA-Z0-9]*
 182 PP_NUMBER       [.]?[0-9]([._a-zA-Z0-9]|[eEpP][-+])*
 183 PUNCTUATION     [][(){}.&*~!/%<>^|;,=+-]
 184
 185 /* The OTHER class is simply a catch-all for things that the CPP
 186 parser just doesn't care about. Since flex regular expressions that
 187 match longer strings take priority over those matching shorter
 188 strings, we have to be careful to avoid OTHER matching and hiding
 189 something that CPP does care about. So we simply exclude all
 190 characters that appear in any other expressions. */
 191
 192 OTHER           [^][_#[:space:]#a-zA-Z0-9(){}.&*~!/%<>^|;,=+-]
 193
 194 DIGITS                  [0-9][0-9]*
 195 DECIMAL_INTEGER         [1-9][0-9]*[uU]?
 196 OCTAL_INTEGER           0[0-7]*[uU]?
 197 HEXADECIMAL_INTEGER     0[xX][0-9a-fA-F]+[uU]?
 198
 199 %%
 200
 201         glcpp_parser_t *parser = yyextra;
 202
 203         /* When we lex a multi-line comment, we replace it (as
 204          * specified) with a single space. But if the comment spanned
 205          * multiple lines, then subsequent parsing stages will not
 206          * count correct line numbers. To avoid this problem we keep
 207          * track of all newlines that were commented out by a
 208          * multi-line comment, and we emit a NEWLINE token for each at
 209          * the next legal opportunity, (which is when the lexer would
 210          * be emitting a NEWLINE token anyway).
 211          */
 212         if (YY_START == NEWLINE_CATCHUP) {
 213                 if (parser->commented_newlines)
 214                         parser->commented_newlines--;
 215                 if (parser->commented_newlines == 0)
 216                         BEGIN INITIAL;
 217                 RETURN_TOKEN_NEVER_SKIP (NEWLINE);
 218         }
 219
 220         /* Set up the parser->skipping bit here before doing any lexing.
 221          *
 222          * This bit controls whether tokens are skipped, (as implemented by
 223          * RETURN_TOKEN), such as between "#if 0" and "#endif".
 224          *
 225          * The parser maintains a skip_stack indicating whether we should be
 226          * skipping, (and nested levels of #if/#ifdef/#ifndef/#endif) will
 227          * push and pop items from the stack.
 228          *
 229          * Here are the rules for determining whether we are skipping:
 230          *
 231          *      1. If the skip stack is NULL, we are outside of all #if blocks
 232          *         and we are not skipping.
 233          *
 234          *      2. If the skip stack is non-NULL, the type of the top node in
 235          *         the stack determines whether to skip. A type of
 236          *         SKIP_NO_SKIP is used for blocks wheere we are emitting
 237          *         tokens, (such as between #if 1 and #endif, or after the
 238          *         #else of an #if 0, etc.).
 239          *
 240          *      3. The lexing_directive bit overrides the skip stack. This bit
 241          *         is set when we are actively lexing the expression for a
 242          *         pre-processor condition, (such as #if, #elif, or #else). In
 243          *         this case, even if otherwise skipping, we need to emit the
 244          *         tokens for this condition so that the parser can evaluate
 245          *         the expression. (For, #else, there's no expression, but we
 246          *         emit tokens so the parser can generate a nice error message
 247          *         if there are any tokens here).
 248          */
 249         if (parser->skip_stack &&
 250             parser->skip_stack->type != SKIP_NO_SKIP &&
 251             ! parser->lexing_directive)
 252         {
 253                 parser->skipping = 1;
 254         } else {
 255                 parser->skipping = 0;
 256         }
 257
 258         /* Single-line comments */
 259 <INITIAL,DEFINE,HASH>"//"[^\r\n]* {
 260 }
 261
 262         /* Multi-line comments */
 263 <INITIAL,DEFINE,HASH>"/*"   { yy_push_state(COMMENT, yyscanner); }
 264 <COMMENT>[^*\r\n]*
 265 <COMMENT>[^*\r\n]*{NEWLINE} { yylineno++; yycolumn = 0; parser->commented_newlines++; }
 266 <COMMENT>"*"+[^*/\r\n]*
 267 <COMMENT>"*"+[^*/\r\n]*{NEWLINE} { yylineno++; yycolumn = 0; parser->commented_newlines++; }
 268 <COMMENT>"*"+"/"        {
 269         yy_pop_state(yyscanner);
 270         /* In the <HASH> start condition, we don't want any SPACE token. */
 271         if (yyextra->space_tokens && YY_START != HASH)
 272                 RETURN_TOKEN (SPACE);
 273 }
 274
 275 {HASH} {
 276
 277         /* If the '#' is the first non-whitespace, non-comment token on this
 278          * line, then it introduces a directive, switch to the <HASH> start
 279          * condition.
 280          *
 281          * Otherwise, this is just punctuation, so return the HASH_TOKEN
 282          * token. */
 283         if (parser->first_non_space_token_this_line) {
 284                 BEGIN HASH;
 285         }
 286
 287         RETURN_TOKEN_NEVER_SKIP (HASH_TOKEN);
 288 }
 289
 290 <HASH>version{HSPACE}+ {
 291         BEGIN INITIAL;
 292         yyextra->space_tokens = 0;
 293         yyextra->lexing_version_directive = 1;
 294         RETURN_STRING_TOKEN (VERSION_TOKEN);
 295 }
 296
 297         /* Swallow empty #pragma directives, (to avoid confusing the
 298          * downstream compiler).
 299          *
 300          * Note: We use a simple regular expression for the lookahead
 301          * here. Specifically, we cannot use the complete {NEWLINE} expression
 302          * since it uses alternation and we've found that there's a flex bug
 303          * where using alternation in the lookahead portion of a pattern
 304          * triggers a buffer overrun. */
 305 <HASH>pragma{HSPACE}*/[\r\n] {
 306         BEGIN INITIAL;
 307 }
 308
 309         /* glcpp doesn't handle #extension, #version, or #pragma directives.
 310          * Simply pass them through to the main compiler's lexer/parser. */
 311 <HASH>(extension|pragma)[^\r\n]* {
 312         BEGIN INITIAL;
 313         RETURN_STRING_TOKEN (PRAGMA);
 314 }
 315
 316 <HASH>line{HSPACE}+ {
 317         BEGIN INITIAL;
 318         RETURN_TOKEN (LINE);
 319 }
 320
 321 <HASH>{NEWLINE} {
 322         BEGIN INITIAL;
 323         yyextra->space_tokens = 0;
 324         yylineno++;
 325         yycolumn = 0;
 326         RETURN_TOKEN_NEVER_SKIP (NEWLINE);
 327 }
 328
 329         /* For the pre-processor directives, we return these tokens
 330          * even when we are otherwise skipping. */
 331 <HASH>ifdef {
 332         BEGIN INITIAL;
 333         yyextra->lexing_directive = 1;
 334         yyextra->space_tokens = 0;
 335         RETURN_TOKEN_NEVER_SKIP (IFDEF);
 336 }
 337
 338 <HASH>ifndef {
 339         BEGIN INITIAL;
 340         yyextra->lexing_directive = 1;
 341         yyextra->space_tokens = 0;
 342         RETURN_TOKEN_NEVER_SKIP (IFNDEF);
 343 }
 344
 345 <HASH>if/[^_a-zA-Z0-9] {
 346         BEGIN INITIAL;
 347         yyextra->lexing_directive = 1;
 348         yyextra->space_tokens = 0;
 349         RETURN_TOKEN_NEVER_SKIP (IF);
 350 }
 351
 352 <HASH>elif/[^_a-zA-Z0-9] {
 353         BEGIN INITIAL;
 354         yyextra->lexing_directive = 1;
 355         yyextra->space_tokens = 0;
 356         RETURN_TOKEN_NEVER_SKIP (ELIF);
 357 }
 358
 359 <HASH>else {
 360         BEGIN INITIAL;
 361         yyextra->space_tokens = 0;
 362         RETURN_TOKEN_NEVER_SKIP (ELSE);
 363 }
 364
 365 <HASH>endif {
 366         BEGIN INITIAL;
 367         yyextra->space_tokens = 0;
 368         RETURN_TOKEN_NEVER_SKIP (ENDIF);
 369 }
 370
 371 <HASH>error[^\r\n]* {
 372         BEGIN INITIAL;
 373         RETURN_STRING_TOKEN (ERROR_TOKEN);
 374 }
 375
 376         /* After we see a "#define" we enter the <DEFINE> start state
 377          * for the lexer. Within <DEFINE> we are looking for the first
 378          * identifier and specifically checking whether the identifier
 379          * is followed by a '(' or not, (to lex either a
 380          * FUNC_IDENTIFIER or an OBJ_IDENITIFIER token).
 381          *
 382          * While in the <DEFINE> state we also need to explicitly
 383          * handle a few other things that may appear before the
 384          * identifier:
 385          *
 386          *      * Comments, (handled above with the main support for
 387          *        comments).
 388          *
 389          *      * Whitespace (simply ignored)
 390          *
 391          *      * Anything else, (not an identifier, not a comment,
 392          *        and not whitespace). This will generate an error.
 393          */
 394 <HASH>define{HSPACE}* {
 395         if (! parser->skipping) {
 396                 BEGIN DEFINE;
 397                 yyextra->space_tokens = 0;
 398                 RETURN_TOKEN (DEFINE_TOKEN);
 399         }
 400 }
 401
 402 <HASH>undef {
 403         BEGIN INITIAL;
 404         yyextra->space_tokens = 0;
 405         RETURN_TOKEN (UNDEF);
 406 }
 407
 408 <HASH>{HSPACE}+ {
 409         /* Nothing to do here. Importantly, don't leave the <HASH>
 410          * start condition, since it's legal to have space between the
 411          * '#' and the directive.. */
 412 }
 413
 414         /* This will catch any non-directive garbage after a HASH */
 415 <HASH>{NONSPACE} {
 416         BEGIN INITIAL;
 417         RETURN_TOKEN (GARBAGE);
 418 }
 419
 420         /* An identifier immediately followed by '(' */
 421 <DEFINE>{IDENTIFIER}/"(" {
 422         BEGIN INITIAL;
 423         RETURN_STRING_TOKEN (FUNC_IDENTIFIER);
 424 }
 425
 426         /* An identifier not immediately followed by '(' */
 427 <DEFINE>{IDENTIFIER} {
 428         BEGIN INITIAL;
 429         RETURN_STRING_TOKEN (OBJ_IDENTIFIER);
 430 }
 431
 432         /* Whitespace */
 433 <DEFINE>{HSPACE}+ {
 434         /* Just ignore it. Nothing to do here. */
 435 }
 436
 437         /* '/' not followed by '*', so not a comment. This is an error. */
 438 <DEFINE>[/][^*]{NONSPACE}* {
 439         BEGIN INITIAL;
 440         glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
 441         RETURN_STRING_TOKEN (INTEGER_STRING);
 442 }
 443
 444         /* A character that can't start an identifier, comment, or
 445          * space. This is an error. */
 446 <DEFINE>[^_a-zA-Z/[:space:]]{NONSPACE}* {
 447         BEGIN INITIAL;
 448         glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
 449         RETURN_STRING_TOKEN (INTEGER_STRING);
 450 }
 451
 452 {DECIMAL_INTEGER} {
 453         RETURN_STRING_TOKEN (INTEGER_STRING);
 454 }
 455
 456 {OCTAL_INTEGER} {
 457         RETURN_STRING_TOKEN (INTEGER_STRING);
 458 }
 459
 460 {HEXADECIMAL_INTEGER} {
 461         RETURN_STRING_TOKEN (INTEGER_STRING);
 462 }
 463
 464 "<<"  {
 465         RETURN_TOKEN (LEFT_SHIFT);
 466 }
 467
 468 ">>" {
 469         RETURN_TOKEN (RIGHT_SHIFT);
 470 }
 471
 472 "<=" {
 473         RETURN_TOKEN (LESS_OR_EQUAL);
 474 }
 475
 476 ">=" {
 477         RETURN_TOKEN (GREATER_OR_EQUAL);
 478 }
 479
 480 "==" {
 481         RETURN_TOKEN (EQUAL);
 482 }
 483
 484 "!=" {
 485         RETURN_TOKEN (NOT_EQUAL);
 486 }
 487
 488 "&&" {
 489         RETURN_TOKEN (AND);
 490 }
 491
 492 "||" {
 493         RETURN_TOKEN (OR);
 494 }
 495
 496 "++" {
 497         RETURN_TOKEN (PLUS_PLUS);
 498 }
 499
 500 "--" {
 501         RETURN_TOKEN (MINUS_MINUS);
 502 }
 503
 504 "##" {
 505         if (! parser->skipping) {
 506                 if (parser->is_gles)
 507                         glcpp_error(yylloc, yyextra, "Token pasting (##) is illegal in GLES");
 508                 RETURN_TOKEN (PASTE);
 509         }
 510 }
 511
 512 "defined" {
 513         RETURN_TOKEN (DEFINED);
 514 }
 515
 516 {IDENTIFIER} {
 517         RETURN_STRING_TOKEN (IDENTIFIER);
 518 }
 519
 520 {PP_NUMBER} {
 521         RETURN_STRING_TOKEN (OTHER);
 522 }
 523
 524 {PUNCTUATION} {
 525         RETURN_TOKEN (yytext[0]);
 526 }
 527
 528 {OTHER}+ {
 529         RETURN_STRING_TOKEN (OTHER);
 530 }
 531
 532 {HSPACE} {
 533         if (yyextra->space_tokens) {
 534                 RETURN_TOKEN (SPACE);
 535         }
 536 }
 537
 538         /* We preserve all newlines, even between #if 0..#endif, so no
 539         skipping.. */
 540 <*>{NEWLINE} {
 541         if (parser->commented_newlines) {
 542                 BEGIN NEWLINE_CATCHUP;
 543         } else {
 544                 BEGIN INITIAL;
 545         }
 546         yyextra->space_tokens = 1;
 547         yyextra->lexing_directive = 0;
 548         yyextra->lexing_version_directive = 0;
 549         yylineno++;
 550         yycolumn = 0;
 551         RETURN_TOKEN_NEVER_SKIP (NEWLINE);
 552 }
 553
 554 <INITIAL,COMMENT,DEFINE,HASH><<EOF>> {
 555         if (YY_START == COMMENT)
 556                 glcpp_error(yylloc, yyextra, "Unterminated comment");
 557         BEGIN DONE; /* Don't keep matching this rule forever. */
 558         yyextra->lexing_directive = 0;
 559         yyextra->lexing_version_directive = 0;
 560         if (! parser->last_token_was_newline)
 561                 RETURN_TOKEN (NEWLINE);
 562 }
 563
 564         /* This is a catch-all to avoid the annoying default flex action which
 565          * matches any character and prints it. If any input ever matches this
 566          * rule, then we have made a mistake above and need to fix one or more
 567          * of the preceding patterns to match that input. */
 568
 569 <*>. {
 570         glcpp_error(yylloc, yyextra, "Internal compiler error: Unexpected character: %s", yytext);
 571
 572         /* We don't actually use the UNREACHABLE start condition. We
 573         only have this block here so that we can pretend to call some
 574         generated functions, (to avoid "defined but not used"
 575         warnings. */
 576         if (YY_START == UNREACHABLE) {
 577                 unput('.');
 578                 yy_top_state(yyextra);
 579         }
 580 }
 581
 582 %%
 583
 584 void
 585 glcpp_lex_set_source_string(glcpp_parser_t *parser, const char *shader)
 586 {
 587         yy_scan_string(shader, parser->scanner);
 588 }