src/glsl/glcpp/glcpp-lex.l

   1 %{
   2 /*
   3  * Copyright © 2010 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <ctype.h>
  28
  29 #include "glcpp.h"
  30 #include "glcpp-parse.h"
  31
  32 /* Flex annoyingly generates some functions without making them
  33  * static. Let's declare them here. */
  34 int glcpp_get_column  (yyscan_t yyscanner);
  35 void glcpp_set_column (int  column_no , yyscan_t yyscanner);
  36
  37 #ifdef _MSC_VER
  38 #define YY_NO_UNISTD_H
  39 #endif
  40
  41 #define YY_NO_INPUT
  42
  43 #define YY_USER_ACTION                                                  \
  44         do {                                                            \
  45                 if (parser->has_new_line_number)                        \
  46                         yylineno = parser->new_line_number;             \
  47                 if (parser->has_new_source_number)                      \
  48                         yylloc->source = parser->new_source_number;     \
  49                 yylloc->first_column = yycolumn + 1;                    \
  50                 yylloc->first_line = yylloc->last_line = yylineno;      \
  51                 yycolumn += yyleng;                                     \
  52                 yylloc->last_column = yycolumn + 1;                     \
  53                 parser->has_new_line_number = 0;                        \
  54                 parser->has_new_source_number = 0;                      \
  55  } while(0);
  56
  57 #define YY_USER_INIT                    \
  58         do {                            \
  59                 yylineno = 1;           \
  60                 yycolumn = 0;           \
  61                 yylloc->source = 0;     \
  62         } while(0)
  63
  64 /* It's ugly to have macros that have return statements inside of
  65  * them, but flex-based lexer generation is all built around the
  66  * return statement.
  67  *
  68  * To mitigate the ugliness, we defer as much of the logic as possible
  69  * to an actual function, not a macro (see
  70  * glcpplex_update_state_per_token) and we make the word RETURN
  71  * prominent in all of the macros which may return.
  72  *
  73  * The most-commonly-used macro is RETURN_TOKEN which will perform all
  74  * necessary state updates based on the provided token,, then
  75  * conditionally return the token. It will not return a token if the
  76  * parser is currently skipping tokens, (such as within #if
  77  * 0...#else).
  78  *
  79  * The RETURN_TOKEN_NEVER_SKIP macro is a lower-level variant that
  80  * makes the token returning unconditional. This is needed for things
  81  * like #if and the tokens of its condition, (since these must be
  82  * evaluated by the parser even when otherwise skipping).
  83  *
  84  * Finally, RETURN_STRING_TOKEN is a simple convenience wrapper on top
  85  * of RETURN_TOKEN that performs a string copy of yytext before the
  86  * return.
  87  */
  88 #define RETURN_TOKEN_NEVER_SKIP(token)                          \
  89         do {                                                    \
  90                 if (token == NEWLINE)                           \
  91                         parser->last_token_was_newline = 1;     \
  92                 else                                            \
  93                         parser->last_token_was_newline = 0;     \
  94                 return (token);                                 \
  95         } while (0)
  96
  97 #define RETURN_TOKEN(token)                                             \
  98         do {                                                            \
  99                 if (! parser->skipping) {                               \
 100                         RETURN_TOKEN_NEVER_SKIP(token);                 \
 101                 }                                                       \
 102         } while(0)
 103
 104 #define RETURN_STRING_TOKEN(token)                                      \
 105         do {                                                            \
 106                 if (! parser->skipping) {                               \
 107                         yylval->str = ralloc_strdup (yyextra, yytext);  \
 108                         RETURN_TOKEN_NEVER_SKIP (token);                \
 109                 }                                                       \
 110         } while(0)
 111
 112 %}
 113
 114 %option bison-bridge bison-locations reentrant noyywrap
 115 %option extra-type="glcpp_parser_t *"
 116 %option prefix="glcpp_"
 117 %option stack
 118 %option never-interactive
 119
 120 %x DONE COMMENT UNREACHABLE DEFINE NEWLINE_CATCHUP
 121
 122 SPACE           [[:space:]]
 123 NONSPACE        [^[:space:]]
 124 NEWLINE         [\n]
 125 HSPACE          [ \t]
 126 HASH            ^{HSPACE}*#{HSPACE}*
 127 IDENTIFIER      [_a-zA-Z][_a-zA-Z0-9]*
 128 PP_NUMBER       [.]?[0-9]([._a-zA-Z0-9]|[eEpP][-+])*
 129 PUNCTUATION     [][(){}.&*~!/%<>^|;,=+-]
 130
 131 /* The OTHER class is simply a catch-all for things that the CPP
 132 parser just doesn't care about. Since flex regular expressions that
 133 match longer strings take priority over those matching shorter
 134 strings, we have to be careful to avoid OTHER matching and hiding
 135 something that CPP does care about. So we simply exclude all
 136 characters that appear in any other expressions. */
 137
 138 OTHER           [^][_#[:space:]#a-zA-Z0-9(){}.&*~!/%<>^|;,=+-]
 139
 140 DIGITS                  [0-9][0-9]*
 141 DECIMAL_INTEGER         [1-9][0-9]*[uU]?
 142 OCTAL_INTEGER           0[0-7]*[uU]?
 143 HEXADECIMAL_INTEGER     0[xX][0-9a-fA-F]+[uU]?
 144
 145 %%
 146
 147         glcpp_parser_t *parser = yyextra;
 148
 149         /* When we lex a multi-line comment, we replace it (as
 150          * specified) with a single space. But if the comment spanned
 151          * multiple lines, then subsequent parsing stages will not
 152          * count correct line numbers. To avoid this problem we keep
 153          * track of all newlines that were commented out by a
 154          * multi-line comment, and we emit a NEWLINE token for each at
 155          * the next legal opportunity, (which is when the lexer would
 156          * be emitting a NEWLINE token anyway).
 157          */
 158         if (YY_START == NEWLINE_CATCHUP) {
 159                 if (parser->commented_newlines)
 160                         parser->commented_newlines--;
 161                 if (parser->commented_newlines == 0)
 162                         BEGIN INITIAL;
 163                 RETURN_TOKEN (NEWLINE);
 164         }
 165
 166         /* Set up the parser->skipping bit here before doing any lexing.
 167          *
 168          * This bit controls whether tokens are skipped, (as implemented by
 169          * RETURN_TOKEN), such as between "#if 0" and "#endif".
 170          *
 171          * The parser maintains a skip_stack indicating whether we should be
 172          * skipping, (and nested levels of #if/#ifdef/#ifndef/#endif) will
 173          * push and pop items from the stack.
 174          *
 175          * Here are the rules for determining whether we are skipping:
 176          *
 177          *      1. If the skip stack is NULL, we are outside of all #if blocks
 178          *         and we are not skipping.
 179          *
 180          *      2. If the skip stack is non-NULL, the type of the top node in
 181          *         the stack determines whether to skip. A type of
 182          *         SKIP_NO_SKIP is used for blocks wheere we are emitting
 183          *         tokens, (such as between #if 1 and #endif, or after the
 184          *         #else of an #if 0, etc.).
 185          *
 186          *      3. The lexing_directive bit overrides the skip stack. This bit
 187          *         is set when we are actively lexing the expression for a
 188          *         pre-processor condition, (such as #if, #elif, or #else). In
 189          *         this case, even if otherwise skipping, we need to emit the
 190          *         tokens for this condition so that the parser can evaluate
 191          *         the expression. (For, #else, there's no expression, but we
 192          *         emit tokens so the parser can generate a nice error message
 193          *         if there are any tokens here).
 194          */
 195         if (parser->skip_stack &&
 196             parser->skip_stack->type != SKIP_NO_SKIP &&
 197             ! parser->lexing_directive)
 198         {
 199                 parser->skipping = 1;
 200         } else {
 201                 parser->skipping = 0;
 202         }
 203
 204         /* Single-line comments */
 205 "//"[^\n]* {
 206 }
 207
 208         /* Multi-line comments */
 209 <DEFINE,INITIAL>"/*"                    { yy_push_state(COMMENT, yyscanner); }
 210 <COMMENT>[^*\n]*
 211 <COMMENT>[^*\n]*\n      { yylineno++; yycolumn = 0; parser->commented_newlines++; }
 212 <COMMENT>"*"+[^*/\n]*
 213 <COMMENT>"*"+[^*/\n]*\n { yylineno++; yycolumn = 0; parser->commented_newlines++; }
 214 <COMMENT>"*"+"/"        {
 215         yy_pop_state(yyscanner);
 216         if (yyextra->space_tokens)
 217                 RETURN_TOKEN (SPACE);
 218 }
 219
 220 {HASH}version{HSPACE}+ {
 221         yyextra->space_tokens = 0;
 222         RETURN_STRING_TOKEN (HASH_VERSION);
 223 }
 224
 225         /* glcpp doesn't handle #extension, #version, or #pragma directives.
 226          * Simply pass them through to the main compiler's lexer/parser. */
 227 {HASH}(extension|pragma)[^\n]* {
 228         yylineno++;
 229         yycolumn = 0;
 230         RETURN_STRING_TOKEN (OTHER);
 231 }
 232
 233 {HASH}line{HSPACE}+ {
 234         RETURN_TOKEN (HASH_LINE);
 235 }
 236
 237         /* For the pre-processor directives, we return these tokens
 238          * even when we are otherwise skipping. */
 239 {HASH}ifdef {
 240         yyextra->lexing_directive = 1;
 241         yyextra->space_tokens = 0;
 242         RETURN_TOKEN_NEVER_SKIP (HASH_IFDEF);
 243 }
 244
 245 {HASH}ifndef {
 246         yyextra->lexing_directive = 1;
 247         yyextra->space_tokens = 0;
 248         RETURN_TOKEN_NEVER_SKIP (HASH_IFNDEF);
 249 }
 250
 251 {HASH}if/[^_a-zA-Z0-9] {
 252         yyextra->lexing_directive = 1;
 253         yyextra->space_tokens = 0;
 254         RETURN_TOKEN_NEVER_SKIP (HASH_IF);
 255 }
 256
 257 {HASH}elif/[^_a-zA-Z0-9] {
 258         yyextra->lexing_directive = 1;
 259         yyextra->space_tokens = 0;
 260         RETURN_TOKEN_NEVER_SKIP (HASH_ELIF);
 261 }
 262
 263 {HASH}else {
 264         yyextra->space_tokens = 0;
 265         RETURN_TOKEN_NEVER_SKIP (HASH_ELSE);
 266 }
 267
 268 {HASH}endif {
 269         yyextra->space_tokens = 0;
 270         RETURN_TOKEN_NEVER_SKIP (HASH_ENDIF);
 271 }
 272
 273 {HASH}error.* {
 274         if (! parser->skipping) {
 275                 char *p;
 276                 for (p = yytext; !isalpha(p[0]); p++); /* skip "  #   " */
 277                 p += 5; /* skip "error" */
 278                 glcpp_error(yylloc, yyextra, "#error%s", p);
 279         }
 280 }
 281
 282         /* After we see a "#define" we enter the <DEFINE> start state
 283          * for the lexer. Within <DEFINE> we are looking for the first
 284          * identifier and specifically checking whether the identifier
 285          * is followed by a '(' or not, (to lex either a
 286          * FUNC_IDENTIFIER or an OBJ_IDENITIFIER token).
 287          *
 288          * While in the <DEFINE> state we also need to explicitly
 289          * handle a few other things that may appear before the
 290          * identifier:
 291          *
 292          *      * Comments, (handled above with the main support for
 293          *        comments).
 294          *
 295          *      * Whitespace (simply ignored)
 296          *
 297          *      * Anything else, (not an identifier, not a comment,
 298          *        and not whitespace). This will generate an error.
 299          */
 300 {HASH}define{HSPACE}+ {
 301         if (! parser->skipping) {
 302                 BEGIN DEFINE;
 303                 yyextra->space_tokens = 0;
 304                 RETURN_TOKEN (HASH_DEFINE);
 305         }
 306 }
 307
 308         /* An identifier immediately followed by '(' */
 309 <DEFINE>{IDENTIFIER}/"(" {
 310         BEGIN INITIAL;
 311         RETURN_STRING_TOKEN (FUNC_IDENTIFIER);
 312 }
 313
 314         /* An identifier not immediately followed by '(' */
 315 <DEFINE>{IDENTIFIER} {
 316         BEGIN INITIAL;
 317         RETURN_STRING_TOKEN (OBJ_IDENTIFIER);
 318 }
 319
 320         /* Whitespace */
 321 <DEFINE>{HSPACE}+ {
 322         /* Just ignore it. Nothing to do here. */
 323 }
 324
 325         /* '/' not followed by '*', so not a comment. This is an error. */
 326 <DEFINE>[/][^*]{NONSPACE}* {
 327         BEGIN INITIAL;
 328         glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
 329         RETURN_STRING_TOKEN (INTEGER_STRING);
 330 }
 331
 332         /* A character that can't start an identifier, comment, or
 333          * space. This is an error. */
 334 <DEFINE>[^_a-zA-Z/[:space:]]{NONSPACE}* {
 335         BEGIN INITIAL;
 336         glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
 337         RETURN_STRING_TOKEN (INTEGER_STRING);
 338 }
 339
 340 {HASH}undef {
 341         yyextra->space_tokens = 0;
 342         RETURN_TOKEN (HASH_UNDEF);
 343 }
 344
 345 {HASH} {
 346         yyextra->space_tokens = 0;
 347         RETURN_TOKEN (HASH_TOKEN);
 348 }
 349
 350 {DECIMAL_INTEGER} {
 351         RETURN_STRING_TOKEN (INTEGER_STRING);
 352 }
 353
 354 {OCTAL_INTEGER} {
 355         RETURN_STRING_TOKEN (INTEGER_STRING);
 356 }
 357
 358 {HEXADECIMAL_INTEGER} {
 359         RETURN_STRING_TOKEN (INTEGER_STRING);
 360 }
 361
 362 "<<"  {
 363         RETURN_TOKEN (LEFT_SHIFT);
 364 }
 365
 366 ">>" {
 367         RETURN_TOKEN (RIGHT_SHIFT);
 368 }
 369
 370 "<=" {
 371         RETURN_TOKEN (LESS_OR_EQUAL);
 372 }
 373
 374 ">=" {
 375         RETURN_TOKEN (GREATER_OR_EQUAL);
 376 }
 377
 378 "==" {
 379         RETURN_TOKEN (EQUAL);
 380 }
 381
 382 "!=" {
 383         RETURN_TOKEN (NOT_EQUAL);
 384 }
 385
 386 "&&" {
 387         RETURN_TOKEN (AND);
 388 }
 389
 390 "||" {
 391         RETURN_TOKEN (OR);
 392 }
 393
 394 "##" {
 395         if (! parser->skipping) {
 396                 if (parser->is_gles)
 397                         glcpp_error(yylloc, yyextra, "Token pasting (##) is illegal in GLES");
 398                 RETURN_TOKEN (PASTE);
 399         }
 400 }
 401
 402 "defined" {
 403         RETURN_TOKEN (DEFINED);
 404 }
 405
 406 {IDENTIFIER} {
 407         RETURN_STRING_TOKEN (IDENTIFIER);
 408 }
 409
 410 {PP_NUMBER} {
 411         RETURN_STRING_TOKEN (OTHER);
 412 }
 413
 414 {PUNCTUATION} {
 415         RETURN_TOKEN (yytext[0]);
 416 }
 417
 418 {OTHER}+ {
 419         RETURN_STRING_TOKEN (OTHER);
 420 }
 421
 422 {HSPACE} {
 423         if (yyextra->space_tokens) {
 424                 RETURN_TOKEN (SPACE);
 425         }
 426 }
 427
 428         /* We preserve all newlines, even between #if 0..#endif, so no
 429         skipping.. */
 430 \n {
 431         if (parser->commented_newlines) {
 432                 BEGIN NEWLINE_CATCHUP;
 433         }
 434         yyextra->space_tokens = 1;
 435         yyextra->lexing_directive = 0;
 436         yylineno++;
 437         yycolumn = 0;
 438         RETURN_TOKEN_NEVER_SKIP (NEWLINE);
 439 }
 440
 441 <INITIAL,COMMENT,DEFINE><<EOF>> {
 442         if (YY_START == COMMENT)
 443                 glcpp_error(yylloc, yyextra, "Unterminated comment");
 444         if (YY_START == DEFINE)
 445                 glcpp_error(yylloc, yyextra, "#define without macro name");
 446         BEGIN DONE; /* Don't keep matching this rule forever. */
 447         yyextra->lexing_directive = 0;
 448         if (! parser->last_token_was_newline)
 449                 RETURN_TOKEN (NEWLINE);
 450 }
 451
 452         /* We don't actually use the UNREACHABLE start condition. We
 453         only have this action here so that we can pretend to call some
 454         generated functions, (to avoid "defined but not used"
 455         warnings. */
 456 <UNREACHABLE>. {
 457         unput('.');
 458         yy_top_state(yyextra);
 459 }
 460
 461 %%
 462
 463 void
 464 glcpp_lex_set_source_string(glcpp_parser_t *parser, const char *shader)
 465 {
 466         yy_scan_string(shader, parser->scanner);
 467 }