src/glsl/pp/sl_pp_token.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <assert.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include "sl_pp_public.h"
  32 #include "sl_pp_context.h"
  33 #include "sl_pp_token.h"
  34
  35
  36 #define PURE_ERROR 256
  37
  38 static int
  39 _pure_getc(struct sl_pp_context *context)
  40 {
  41    char c;
  42
  43    if (context->getc_buf_size) {
  44       return context->getc_buf[--context->getc_buf_size];
  45    }
  46
  47    if (sl_pp_purify_getc(&context->pure, &c, &context->error_line, context->error_msg, sizeof(context->error_msg)) == 0) {
  48       return PURE_ERROR;
  49    }
  50    return c;
  51 }
  52
  53
  54 static void
  55 _pure_ungetc(struct sl_pp_context *context,
  56              int c)
  57 {
  58    assert(c != PURE_ERROR);
  59
  60    if (context->getc_buf_size == context->getc_buf_capacity) {
  61       context->getc_buf_capacity += 64;
  62       context->getc_buf = realloc(context->getc_buf, context->getc_buf_capacity * sizeof(char));
  63       assert(context->getc_buf);
  64    }
  65
  66    context->getc_buf[context->getc_buf_size++] = (char)c;
  67 }
  68
  69
  70 struct lookahead_state {
  71    char buf[256];
  72    unsigned int pos;
  73    struct sl_pp_context *context;
  74 };
  75
  76
  77 static void
  78 _lookahead_init(struct lookahead_state *lookahead,
  79                 struct sl_pp_context *context)
  80 {
  81    lookahead->pos = 0;
  82    lookahead->context = context;
  83 }
  84
  85
  86 static unsigned int
  87 _lookahead_tell(const struct lookahead_state *lookahead)
  88 {
  89    return lookahead->pos;
  90 }
  91
  92
  93 static const void *
  94 _lookahead_buf(const struct lookahead_state *lookahead)
  95 {
  96    return lookahead->buf;
  97 }
  98
  99
 100 static void
 101 _lookahead_revert(struct lookahead_state *lookahead,
 102                   unsigned int pos)
 103 {
 104    assert(pos <= lookahead->pos);
 105
 106    while (lookahead->pos > pos) {
 107       _pure_ungetc(lookahead->context, lookahead->buf[--lookahead->pos]);
 108    }
 109 }
 110
 111
 112 static int
 113 _lookahead_getc(struct lookahead_state *lookahead)
 114 {
 115    int c;
 116
 117    assert(lookahead->pos < sizeof(lookahead->buf) / sizeof(lookahead->buf[0]));
 118
 119    c = _pure_getc(lookahead->context);
 120    if (c != PURE_ERROR) {
 121       lookahead->buf[lookahead->pos++] = (char)c;
 122    }
 123    return c;
 124 }
 125
 126
 127 static int
 128 _is_identifier_char(char c)
 129 {
 130    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
 131 }
 132
 133
 134 static int
 135 _tokenise_identifier(struct sl_pp_context *context,
 136                      struct sl_pp_token_info *out)
 137 {
 138    int c;
 139    char identifier[256];   /* XXX: Remove this artifical limit. */
 140    unsigned int i = 0;
 141
 142    out->token = SL_PP_IDENTIFIER;
 143    out->data.identifier = -1;
 144
 145    c = _pure_getc(context);
 146    if (c == PURE_ERROR) {
 147       return -1;
 148    }
 149    identifier[i++] = (char)c;
 150    for (;;) {
 151       c = _pure_getc(context);
 152       if (c == PURE_ERROR) {
 153          return -1;
 154       }
 155
 156       if (_is_identifier_char((char)c)) {
 157          if (i >= sizeof(identifier) / sizeof(char) - 1) {
 158             strcpy(context->error_msg, "out of memory");
 159             _pure_ungetc(context, c);
 160             while (i) {
 161                _pure_ungetc(context, identifier[--i]);
 162             }
 163             return -1;
 164          }
 165          identifier[i++] = (char)c;
 166       } else {
 167          _pure_ungetc(context, c);
 168          break;
 169       }
 170    }
 171    identifier[i] = '\0';
 172
 173    out->data.identifier = sl_pp_context_add_unique_str(context, identifier);
 174    if (out->data.identifier == -1) {
 175       while (i) {
 176          _pure_ungetc(context, identifier[--i]);
 177       }
 178       return -1;
 179    }
 180
 181    return 0;
 182 }
 183
 184
 185 /*
 186  * Return the number of consecutive decimal digits in the input stream.
 187  */
 188 static unsigned int
 189 _parse_float_digits(struct lookahead_state *lookahead)
 190 {
 191    unsigned int eaten;
 192
 193    for (eaten = 0;; eaten++) {
 194       unsigned int pos = _lookahead_tell(lookahead);
 195       char c = _lookahead_getc(lookahead);
 196
 197       if (c < '0' || c > '9') {
 198          _lookahead_revert(lookahead, pos);
 199          break;
 200       }
 201    }
 202    return eaten;
 203 }
 204
 205
 206 /*
 207  * Try to match one of the following patterns for the fractional part
 208  * of a floating point number.
 209  *
 210  * digits . [digits]
 211  * . digits
 212  *
 213  * Return 0 if the pattern could not be matched, otherwise the number
 214  * of eaten characters from the input stream.
 215  */
 216 static unsigned int
 217 _parse_float_frac(struct lookahead_state *lookahead)
 218 {
 219    unsigned int pos;
 220    int c;
 221    unsigned int eaten;
 222
 223    pos = _lookahead_tell(lookahead);
 224    c = _lookahead_getc(lookahead);
 225    if (c == '.') {
 226       eaten = _parse_float_digits(lookahead);
 227       if (eaten) {
 228          return eaten + 1;
 229       }
 230       _lookahead_revert(lookahead, pos);
 231       return 0;
 232    }
 233
 234    _lookahead_revert(lookahead, pos);
 235    eaten = _parse_float_digits(lookahead);
 236    if (eaten) {
 237       c = _lookahead_getc(lookahead);
 238       if (c == '.') {
 239          return eaten + 1 + _parse_float_digits(lookahead);
 240       }
 241    }
 242
 243    _lookahead_revert(lookahead, pos);
 244    return 0;
 245 }
 246
 247
 248 /*
 249  * Try to match the following pattern for the exponential part
 250  * of a floating point number.
 251  *
 252  * (e|E) [(+|-)] digits
 253  *
 254  * Return 0 if the pattern could not be matched, otherwise the number
 255  * of eaten characters from the input stream.
 256  */
 257 static unsigned int
 258 _parse_float_exp(struct lookahead_state *lookahead)
 259 {
 260    unsigned int pos, pos2;
 261    int c;
 262    unsigned int eaten, digits;
 263
 264    pos = _lookahead_tell(lookahead);
 265    c = _lookahead_getc(lookahead);
 266    if (c != 'e' && c != 'E') {
 267       _lookahead_revert(lookahead, pos);
 268       return 0;
 269    }
 270
 271    pos2 = _lookahead_tell(lookahead);
 272    c = _lookahead_getc(lookahead);
 273    if (c == '-' || c == '+') {
 274       eaten = 2;
 275    } else {
 276       _lookahead_revert(lookahead, pos2);
 277       eaten = 1;
 278    }
 279
 280    digits = _parse_float_digits(lookahead);
 281    if (!digits) {
 282       _lookahead_revert(lookahead, pos);
 283       return 0;
 284    }
 285
 286    return eaten + digits;
 287 }
 288
 289
 290 /*
 291  * Try to match one of the following patterns for a floating point number.
 292  *
 293  * fract [exp] [(f|F)]
 294  * digits exp [(f|F)]
 295  *
 296  * Return 0 if the pattern could not be matched, otherwise the number
 297  * of eaten characters from the input stream.
 298  */
 299 static unsigned int
 300 _parse_float(struct lookahead_state *lookahead)
 301 {
 302    unsigned int eaten;
 303
 304    eaten = _parse_float_frac(lookahead);
 305    if (eaten) {
 306       unsigned int pos;
 307       int c;
 308
 309       eaten += _parse_float_exp(lookahead);
 310
 311       pos = _lookahead_tell(lookahead);
 312       c = _lookahead_getc(lookahead);
 313       if (c == 'f' || c == 'F') {
 314          eaten++;
 315       } else {
 316          _lookahead_revert(lookahead, pos);
 317       }
 318
 319       return eaten;
 320    }
 321
 322    eaten = _parse_float_digits(lookahead);
 323    if (eaten) {
 324       unsigned int exponent;
 325
 326       exponent = _parse_float_exp(lookahead);
 327       if (exponent) {
 328          unsigned int pos;
 329          int c;
 330
 331          eaten += exponent;
 332
 333          pos = _lookahead_tell(lookahead);
 334          c = _lookahead_getc(lookahead);
 335          if (c == 'f' || c == 'F') {
 336             eaten++;
 337          } else {
 338             _lookahead_revert(lookahead, pos);
 339          }
 340
 341          return eaten;
 342       }
 343    }
 344
 345    _lookahead_revert(lookahead, 0);
 346    return 0;
 347 }
 348
 349
 350 static unsigned int
 351 _parse_hex(struct lookahead_state *lookahead)
 352 {
 353    int c;
 354    unsigned int n;
 355
 356    c = _lookahead_getc(lookahead);
 357    if (c != '0') {
 358       _lookahead_revert(lookahead, 0);
 359       return 0;
 360    }
 361
 362    c = _lookahead_getc(lookahead);
 363    if (c != 'x' && c != 'X') {
 364       _lookahead_revert(lookahead, 0);
 365       return 0;
 366    }
 367
 368    for (n = 2;;) {
 369       unsigned int pos = _lookahead_tell(lookahead);
 370
 371       c = _lookahead_getc(lookahead);
 372       if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
 373          n++;
 374       } else {
 375          _lookahead_revert(lookahead, pos);
 376          break;
 377       }
 378    }
 379
 380    if (n > 2) {
 381       return n;
 382    }
 383
 384    _lookahead_revert(lookahead, 0);
 385    return 0;
 386 }
 387
 388
 389 static unsigned int
 390 _parse_oct(struct lookahead_state *lookahead)
 391 {
 392    int c;
 393    unsigned int n;
 394
 395    c = _lookahead_getc(lookahead);
 396    if (c != '0') {
 397       _lookahead_revert(lookahead, 0);
 398       return 0;
 399    }
 400
 401    for (n = 1;;) {
 402       unsigned int pos = _lookahead_tell(lookahead);
 403
 404       c = _lookahead_getc(lookahead);
 405       if ((c >= '0' && c <= '7')) {
 406          n++;
 407       } else {
 408          _lookahead_revert(lookahead, pos);
 409          break;
 410       }
 411    }
 412
 413    return n;
 414 }
 415
 416
 417 static unsigned int
 418 _parse_dec(struct lookahead_state *lookahead)
 419 {
 420    unsigned int n = 0;
 421
 422    for (;;) {
 423       unsigned int pos = _lookahead_tell(lookahead);
 424       int c = _lookahead_getc(lookahead);
 425
 426       if ((c >= '0' && c <= '9')) {
 427          n++;
 428       } else {
 429          _lookahead_revert(lookahead, pos);
 430          break;
 431       }
 432    }
 433
 434    return n;
 435 }
 436
 437
 438 static int
 439 _tokenise_number(struct sl_pp_context *context,
 440                  struct sl_pp_token_info *out)
 441 {
 442    struct lookahead_state lookahead;
 443    unsigned int eaten;
 444    unsigned int is_float = 0;
 445    unsigned int pos;
 446    int c;
 447    char number[256];   /* XXX: Remove this artifical limit. */
 448
 449    _lookahead_init(&lookahead, context);
 450
 451    eaten = _parse_float(&lookahead);
 452    if (!eaten) {
 453       eaten = _parse_hex(&lookahead);
 454       if (!eaten) {
 455          eaten = _parse_oct(&lookahead);
 456          if (!eaten) {
 457             eaten = _parse_dec(&lookahead);
 458          }
 459       }
 460    } else {
 461       is_float = 1;
 462    }
 463
 464    if (!eaten) {
 465       strcpy(context->error_msg, "expected a number");
 466       return -1;
 467    }
 468
 469    pos = _lookahead_tell(&lookahead);
 470    c = _lookahead_getc(&lookahead);
 471    _lookahead_revert(&lookahead, pos);
 472
 473    if (_is_identifier_char(c)) {
 474       strcpy(context->error_msg, "expected a number");
 475       _lookahead_revert(&lookahead, 0);
 476       return -1;
 477    }
 478
 479    if (eaten > sizeof(number) - 1) {
 480       strcpy(context->error_msg, "out of memory");
 481       _lookahead_revert(&lookahead, 0);
 482       return -1;
 483    }
 484
 485    assert(_lookahead_tell(&lookahead) == eaten);
 486
 487    memcpy(number, _lookahead_buf(&lookahead), eaten);
 488    number[eaten] = '\0';
 489
 490    if (is_float) {
 491       out->token = SL_PP_FLOAT;
 492       out->data._float = sl_pp_context_add_unique_str(context, number);
 493       if (out->data._float == -1) {
 494          _lookahead_revert(&lookahead, 0);
 495          return -1;
 496       }
 497    } else {
 498       out->token = SL_PP_UINT;
 499       out->data._uint = sl_pp_context_add_unique_str(context, number);
 500       if (out->data._uint == -1) {
 501          _lookahead_revert(&lookahead, 0);
 502          return -1;
 503       }
 504    }
 505
 506    return 0;
 507 }
 508
 509
 510 int
 511 sl_pp_token_get(struct sl_pp_context *context,
 512                 struct sl_pp_token_info *out)
 513 {
 514    int c = _pure_getc(context);
 515
 516    switch (c) {
 517    case ' ':
 518    case '\t':
 519       out->token = SL_PP_WHITESPACE;
 520       break;
 521
 522    case '\n':
 523       out->token = SL_PP_NEWLINE;
 524       break;
 525
 526    case '#':
 527       out->token = SL_PP_HASH;
 528       break;
 529
 530    case ',':
 531       out->token = SL_PP_COMMA;
 532       break;
 533
 534    case ';':
 535       out->token = SL_PP_SEMICOLON;
 536       break;
 537
 538    case '{':
 539       out->token = SL_PP_LBRACE;
 540       break;
 541
 542    case '}':
 543       out->token = SL_PP_RBRACE;
 544       break;
 545
 546    case '(':
 547       out->token = SL_PP_LPAREN;
 548       break;
 549
 550    case ')':
 551       out->token = SL_PP_RPAREN;
 552       break;
 553
 554    case '[':
 555       out->token = SL_PP_LBRACKET;
 556       break;
 557
 558    case ']':
 559       out->token = SL_PP_RBRACKET;
 560       break;
 561
 562    case '.':
 563       {
 564          int c2 = _pure_getc(context);
 565
 566          if (c2 == PURE_ERROR) {
 567             return -1;
 568          }
 569          if (c2 >= '0' && c2 <= '9') {
 570             _pure_ungetc(context, c2);
 571             _pure_ungetc(context, c);
 572             if (_tokenise_number(context, out)) {
 573                return -1;
 574             }
 575          } else {
 576             _pure_ungetc(context, c2);
 577             out->token = SL_PP_DOT;
 578          }
 579       }
 580       break;
 581
 582    case '+':
 583       c = _pure_getc(context);
 584       if (c == PURE_ERROR) {
 585          return -1;
 586       }
 587       if (c == '+') {
 588          out->token = SL_PP_INCREMENT;
 589       } else if (c == '=') {
 590          out->token = SL_PP_ADDASSIGN;
 591       } else {
 592          _pure_ungetc(context, c);
 593          out->token = SL_PP_PLUS;
 594       }
 595       break;
 596
 597    case '-':
 598       c = _pure_getc(context);
 599       if (c == PURE_ERROR) {
 600          return -1;
 601       }
 602       if (c == '-') {
 603          out->token = SL_PP_DECREMENT;
 604       } else if (c == '=') {
 605          out->token = SL_PP_SUBASSIGN;
 606       } else {
 607          _pure_ungetc(context, c);
 608          out->token = SL_PP_MINUS;
 609       }
 610       break;
 611
 612    case '~':
 613       out->token = SL_PP_BITNOT;
 614       break;
 615
 616    case '!':
 617       c = _pure_getc(context);
 618       if (c == PURE_ERROR) {
 619          return -1;
 620       }
 621       if (c == '=') {
 622          out->token = SL_PP_NOTEQUAL;
 623       } else {
 624          _pure_ungetc(context, c);
 625          out->token = SL_PP_NOT;
 626       }
 627       break;
 628
 629    case '*':
 630       c = _pure_getc(context);
 631       if (c == PURE_ERROR) {
 632          return -1;
 633       }
 634       if (c == '=') {
 635          out->token = SL_PP_MULASSIGN;
 636       } else {
 637          _pure_ungetc(context, c);
 638          out->token = SL_PP_STAR;
 639       }
 640       break;
 641
 642    case '/':
 643       c = _pure_getc(context);
 644       if (c == PURE_ERROR) {
 645          return -1;
 646       }
 647       if (c == '=') {
 648          out->token = SL_PP_DIVASSIGN;
 649       } else {
 650          _pure_ungetc(context, c);
 651          out->token = SL_PP_SLASH;
 652       }
 653       break;
 654
 655    case '%':
 656       c = _pure_getc(context);
 657       if (c == PURE_ERROR) {
 658          return -1;
 659       }
 660       if (c == '=') {
 661          out->token = SL_PP_MODASSIGN;
 662       } else {
 663          _pure_ungetc(context, c);
 664          out->token = SL_PP_MODULO;
 665       }
 666       break;
 667
 668    case '<':
 669       c = _pure_getc(context);
 670       if (c == PURE_ERROR) {
 671          return -1;
 672       }
 673       if (c == '<') {
 674          c = _pure_getc(context);
 675          if (c == PURE_ERROR) {
 676             return -1;
 677          }
 678          if (c == '=') {
 679             out->token = SL_PP_LSHIFTASSIGN;
 680          } else {
 681             _pure_ungetc(context, c);
 682             out->token = SL_PP_LSHIFT;
 683          }
 684       } else if (c == '=') {
 685          out->token = SL_PP_LESSEQUAL;
 686       } else {
 687          _pure_ungetc(context, c);
 688          out->token = SL_PP_LESS;
 689       }
 690       break;
 691
 692    case '>':
 693       c = _pure_getc(context);
 694       if (c == PURE_ERROR) {
 695          return -1;
 696       }
 697       if (c == '>') {
 698          c = _pure_getc(context);
 699          if (c == PURE_ERROR) {
 700             return -1;
 701          }
 702          if (c == '=') {
 703             out->token = SL_PP_RSHIFTASSIGN;
 704          } else {
 705             _pure_ungetc(context, c);
 706             out->token = SL_PP_RSHIFT;
 707          }
 708       } else if (c == '=') {
 709          out->token = SL_PP_GREATEREQUAL;
 710       } else {
 711          _pure_ungetc(context, c);
 712          out->token = SL_PP_GREATER;
 713       }
 714       break;
 715
 716    case '=':
 717       c = _pure_getc(context);
 718       if (c == PURE_ERROR) {
 719          return -1;
 720       }
 721       if (c == '=') {
 722          out->token = SL_PP_EQUAL;
 723       } else {
 724          _pure_ungetc(context, c);
 725          out->token = SL_PP_ASSIGN;
 726       }
 727       break;
 728
 729    case '&':
 730       c = _pure_getc(context);
 731       if (c == PURE_ERROR) {
 732          return -1;
 733       }
 734       if (c == '&') {
 735          out->token = SL_PP_AND;
 736       } else if (c == '=') {
 737          out->token = SL_PP_BITANDASSIGN;
 738       } else {
 739          _pure_ungetc(context, c);
 740          out->token = SL_PP_BITAND;
 741       }
 742       break;
 743
 744    case '^':
 745       c = _pure_getc(context);
 746       if (c == PURE_ERROR) {
 747          return -1;
 748       }
 749       if (c == '^') {
 750          out->token = SL_PP_XOR;
 751       } else if (c == '=') {
 752          out->token = SL_PP_BITXORASSIGN;
 753       } else {
 754          _pure_ungetc(context, c);
 755          out->token = SL_PP_BITXOR;
 756       }
 757       break;
 758
 759    case '|':
 760       c = _pure_getc(context);
 761       if (c == PURE_ERROR) {
 762          return -1;
 763       }
 764       if (c == '|') {
 765          out->token = SL_PP_OR;
 766       } else if (c == '=') {
 767          out->token = SL_PP_BITORASSIGN;
 768       } else {
 769          _pure_ungetc(context, c);
 770          out->token = SL_PP_BITOR;
 771       }
 772       break;
 773
 774    case '?':
 775       out->token = SL_PP_QUESTION;
 776       break;
 777
 778    case ':':
 779       out->token = SL_PP_COLON;
 780       break;
 781
 782    case '\0':
 783       out->token = SL_PP_EOF;
 784       break;
 785
 786    case PURE_ERROR:
 787       return -1;
 788
 789    default:
 790       if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') {
 791          _pure_ungetc(context, c);
 792          if (_tokenise_identifier(context, out)) {
 793             return -1;
 794          }
 795       } else if (c >= '0' && c <= '9') {
 796          _pure_ungetc(context, c);
 797          if (_tokenise_number(context, out)) {
 798             return -1;
 799          }
 800       } else {
 801          out->data.other = c;
 802          out->token = SL_PP_OTHER;
 803       }
 804    }
 805
 806    return 0;
 807 }
 808
 809
 810 int
 811 sl_pp_tokenise(struct sl_pp_context *context,
 812                struct sl_pp_token_info **output)
 813 {
 814    struct sl_pp_token_info *out = NULL;
 815    unsigned int out_len = 0;
 816    unsigned int out_max = 0;
 817
 818    for (;;) {
 819       struct sl_pp_token_info info;
 820
 821       if (sl_pp_token_buffer_get(&context->tokens, &info)) {
 822          free(out);
 823          return -1;
 824       }
 825
 826       if (out_len >= out_max) {
 827          unsigned int new_max = out_max;
 828
 829          if (new_max < 0x100) {
 830             new_max = 0x100;
 831          } else if (new_max < 0x10000) {
 832             new_max *= 2;
 833          } else {
 834             new_max += 0x10000;
 835          }
 836
 837          out = realloc(out, new_max * sizeof(struct sl_pp_token_info));
 838          if (!out) {
 839             strcpy(context->error_msg, "out of memory");
 840             return -1;
 841          }
 842          out_max = new_max;
 843       }
 844
 845       out[out_len++] = info;
 846
 847       if (info.token == SL_PP_EOF) {
 848          break;
 849       }
 850    }
 851
 852    *output = out;
 853    return 0;
 854 }