src/soc/decoder/power_pseudo.py

   1 # Based on GardenSnake - a parser generator demonstration program
   2 # GardenSnake was released into the Public Domain by Andrew Dalke.
   3
   4 # Portions of this work are derived from Python's Grammar definition
   5 # and may be covered under the Python copyright and license
   6 #
   7 #          Andrew Dalke / Dalke Scientific Software, LLC
   8 #             30 August 2006 / Cape Town, South Africa
   9
  10 # Modifications for inclusion in PLY distribution
  11 import sys
  12 from pprint import pprint
  13 from copy import copy
  14 from ply import lex, yacc
  15 import astor
  16
  17 ##### Lexer ######
  18 #import lex
  19 import decimal
  20
  21 tokens = (
  22     'DEF',
  23     'IF',
  24     'THEN',
  25     'ELSE',
  26     'FOR',
  27     'TO',
  28     'DO',
  29     'WHILE',
  30     'BREAK',
  31     'NAME',
  32     'NUMBER',  # Python decimals
  33     'STRING',  # single quoted strings only; syntax of raw strings
  34     'LPAR',
  35     'RPAR',
  36     'LBRACK',
  37     'RBRACK',
  38     'COLON',
  39     'EQ',
  40     'ASSIGN',
  41     'LT',
  42     'GT',
  43     'PLUS',
  44     'MINUS',
  45     'MULT',
  46     'DIV',
  47     'APPEND',
  48     'RETURN',
  49     'WS',
  50     'NEWLINE',
  51     'COMMA',
  52     'SEMICOLON',
  53     'INDENT',
  54     'DEDENT',
  55     'ENDMARKER',
  56     )
  57
  58 #t_NUMBER = r'\d+'
  59 # taken from decmial.py but without the leading sign
  60 def t_NUMBER(t):
  61     r"""(\d+(\.\d*)?|\.\d+)([eE][-+]? \d+)?"""
  62     t.value = int(t.value)
  63     return t
  64
  65 def t_STRING(t):
  66     r"'([^\\']+|\\'|\\\\)*'"  # I think this is right ...
  67     t.value=t.value[1:-1].decode("string-escape") # .swapcase() # for fun
  68     return t
  69
  70 t_COLON = r':'
  71 t_EQ = r'='
  72 t_ASSIGN = r'<-'
  73 t_LT = r'<'
  74 t_GT = r'>'
  75 t_PLUS = r'\+'
  76 t_MINUS = r'-'
  77 t_MULT = r'\*'
  78 t_DIV = r'/'
  79 t_COMMA = r','
  80 t_SEMICOLON = r';'
  81 t_APPEND = r'\|\|'
  82
  83 # Ply nicely documented how to do this.
  84
  85 RESERVED = {
  86   "def": "DEF",
  87   "if": "IF",
  88   "then": "THEN",
  89   "else": "ELSE",
  90   "leave": "BREAK",
  91   "for": "FOR",
  92   "to": "TO",
  93   "while": "WHILE",
  94   "do": "DO",
  95   "return": "RETURN",
  96   }
  97
  98 def t_NAME(t):
  99     r'[a-zA-Z_][a-zA-Z0-9_]*'
 100     t.type = RESERVED.get(t.value, "NAME")
 101     return t
 102
 103 # Putting this before t_WS let it consume lines with only comments in
 104 # them so the latter code never sees the WS part.  Not consuming the
 105 # newline.  Needed for "if 1: #comment"
 106 def t_comment(t):
 107     r"[ ]*\043[^\n]*"  # \043 is '#'
 108     pass
 109
 110
 111 # Whitespace
 112 def t_WS(t):
 113     r'[ ]+'
 114     if t.lexer.at_line_start and t.lexer.paren_count == 0 and \
 115                                  t.lexer.brack_count == 0:
 116         return t
 117
 118 # Don't generate newline tokens when inside of parenthesis, eg
 119 #   a = (1,
 120 #        2, 3)
 121 def t_newline(t):
 122     r'\n+'
 123     t.lexer.lineno += len(t.value)
 124     t.type = "NEWLINE"
 125     if t.lexer.paren_count == 0 and t.lexer.brack_count == 0:
 126         return t
 127
 128 def t_LBRACK(t):
 129     r'\['
 130     t.lexer.brack_count += 1
 131     return t
 132
 133 def t_RBRACK(t):
 134     r'\]'
 135     # check for underflow?  should be the job of the parser
 136     t.lexer.brack_count -= 1
 137     return t
 138
 139 def t_LPAR(t):
 140     r'\('
 141     t.lexer.paren_count += 1
 142     return t
 143
 144 def t_RPAR(t):
 145     r'\)'
 146     # check for underflow?  should be the job of the parser
 147     t.lexer.paren_count -= 1
 148     return t
 149
 150 #t_ignore = " "
 151
 152 def t_error(t):
 153     raise SyntaxError("Unknown symbol %r" % (t.value[0],))
 154     print ("Skipping", repr(t.value[0]))
 155     t.lexer.skip(1)
 156
 157 ## I implemented INDENT / DEDENT generation as a post-processing filter
 158
 159 # The original lex token stream contains WS and NEWLINE characters.
 160 # WS will only occur before any other tokens on a line.
 161
 162 # I have three filters.  One tags tokens by adding two attributes.
 163 # "must_indent" is True if the token must be indented from the
 164 # previous code.  The other is "at_line_start" which is True for WS
 165 # and the first non-WS/non-NEWLINE on a line.  It flags the check so
 166 # see if the new line has changed indication level.
 167
 168 # Python's syntax has three INDENT states
 169 #  0) no colon hence no need to indent
 170 #  1) "if 1: go()" - simple statements have a COLON but no need for an indent
 171 #  2) "if 1:\n  go()" - complex statements have a COLON NEWLINE and must indent
 172 NO_INDENT = 0
 173 MAY_INDENT = 1
 174 MUST_INDENT = 2
 175
 176 # turn into python-like colon syntax from pseudo-code syntax
 177 def python_colonify(lexer, tokens):
 178
 179     forwhile_seen = False
 180     for token in tokens:
 181         print ("track colon token", token, token.type)
 182
 183         if token.type == 'THEN':
 184             # turn then into colon
 185             token.type = "COLON"
 186             yield token
 187         elif token.type == 'ELSE':
 188             yield token
 189             token = copy(token)
 190             token.type = "COLON"
 191             yield token
 192         elif token.type in ['WHILE', 'FOR']:
 193             forwhile_seen = True
 194             yield token
 195         elif token.type == 'NEWLINE':
 196             if forwhile_seen:
 197                 ctok = copy(token)
 198                 ctok.type = "COLON"
 199                 yield ctok
 200                 forwhile_seen = False
 201             yield token
 202         else:
 203             yield token
 204
 205
 206 # only care about whitespace at the start of a line
 207 def track_tokens_filter(lexer, tokens):
 208     oldignore = lexer.lexignore
 209     lexer.at_line_start = at_line_start = True
 210     indent = NO_INDENT
 211     saw_colon = False
 212     for token in tokens:
 213         print ("track token", token, token.type)
 214         token.at_line_start = at_line_start
 215
 216         if token.type == "COLON":
 217             at_line_start = False
 218             indent = MAY_INDENT
 219             token.must_indent = False
 220
 221         elif token.type == "NEWLINE":
 222             at_line_start = True
 223             if indent == MAY_INDENT:
 224                 indent = MUST_INDENT
 225             token.must_indent = False
 226
 227         elif token.type == "WS":
 228             assert token.at_line_start == True
 229             at_line_start = True
 230             token.must_indent = False
 231
 232         else:
 233             # A real token; only indent after COLON NEWLINE
 234             if indent == MUST_INDENT:
 235                 token.must_indent = True
 236             else:
 237                 token.must_indent = False
 238             at_line_start = False
 239             indent = NO_INDENT
 240
 241         # really bad hack that changes ignore lexer state.
 242         # when "must indent" is seen (basically "real tokens" seen)
 243         # then ignore whitespace.
 244         if token.must_indent:
 245             lexer.lexignore = ('ignore', ' ')
 246         else:
 247             lexer.lexignore = oldignore
 248
 249         token.indent = indent
 250         yield token
 251         lexer.at_line_start = at_line_start
 252
 253 def _new_token(type, lineno):
 254     tok = lex.LexToken()
 255     tok.type = type
 256     tok.value = None
 257     tok.lineno = lineno
 258     tok.lexpos = -1
 259     return tok
 260
 261 # Synthesize a DEDENT tag
 262 def DEDENT(lineno):
 263     return _new_token("DEDENT", lineno)
 264
 265 # Synthesize an INDENT tag
 266 def INDENT(lineno):
 267     return _new_token("INDENT", lineno)
 268
 269
 270 # Track the indentation level and emit the right INDENT / DEDENT events.
 271 def indentation_filter(tokens):
 272     # A stack of indentation levels; will never pop item 0
 273     levels = [0]
 274     token = None
 275     depth = 0
 276     prev_was_ws = False
 277     for token in tokens:
 278         if 1:
 279             print ("Process", depth, token.indent, token,)
 280             if token.at_line_start:
 281                 print ("at_line_start",)
 282             if token.must_indent:
 283                 print ("must_indent",)
 284             print
 285
 286         # WS only occurs at the start of the line
 287         # There may be WS followed by NEWLINE so
 288         # only track the depth here.  Don't indent/dedent
 289         # until there's something real.
 290         if token.type == "WS":
 291             assert depth == 0
 292             depth = len(token.value)
 293             prev_was_ws = True
 294             # WS tokens are never passed to the parser
 295             continue
 296
 297         if token.type == "NEWLINE":
 298             depth = 0
 299             if prev_was_ws or token.at_line_start:
 300                 # ignore blank lines
 301                 continue
 302             # pass the other cases on through
 303             yield token
 304             continue
 305
 306         # then it must be a real token (not WS, not NEWLINE)
 307         # which can affect the indentation level
 308
 309         prev_was_ws = False
 310         if token.must_indent:
 311             # The current depth must be larger than the previous level
 312             if not (depth > levels[-1]):
 313                 raise IndentationError("expected an indented block")
 314
 315             levels.append(depth)
 316             yield INDENT(token.lineno)
 317
 318         elif token.at_line_start:
 319             # Must be on the same level or one of the previous levels
 320             if depth == levels[-1]:
 321                 # At the same level
 322                 pass
 323             elif depth > levels[-1]:
 324                 raise IndentationError("indentation increase but not in new block")
 325             else:
 326                 # Back up; but only if it matches a previous level
 327                 try:
 328                     i = levels.index(depth)
 329                 except ValueError:
 330                     raise IndentationError("inconsistent indentation")
 331                 for _ in range(i+1, len(levels)):
 332                     yield DEDENT(token.lineno)
 333                     levels.pop()
 334
 335         yield token
 336
 337     ### Finished processing ###
 338
 339     # Must dedent any remaining levels
 340     if len(levels) > 1:
 341         assert token is not None
 342         for _ in range(1, len(levels)):
 343             yield DEDENT(token.lineno)
 344
 345
 346 # The top-level filter adds an ENDMARKER, if requested.
 347 # Python's grammar uses it.
 348 def filter(lexer, add_endmarker = True):
 349     token = None
 350     tokens = iter(lexer.token, None)
 351     tokens = python_colonify(lexer, tokens)
 352     tokens = track_tokens_filter(lexer, tokens)
 353     for token in indentation_filter(tokens):
 354         yield token
 355
 356     if add_endmarker:
 357         lineno = 1
 358         if token is not None:
 359             lineno = token.lineno
 360         yield _new_token("ENDMARKER", lineno)
 361
 362 # Combine Ply and my filters into a new lexer
 363
 364 class IndentLexer(object):
 365     def __init__(self, debug=0, optimize=0, lextab='lextab', reflags=0):
 366         self.lexer = lex.lex(debug=debug, optimize=optimize, lextab=lextab, reflags=reflags)
 367         self.token_stream = None
 368     def input(self, s, add_endmarker=True):
 369         self.lexer.paren_count = 0
 370         self.lexer.brack_count = 0
 371         self.lexer.input(s)
 372         self.token_stream = filter(self.lexer, add_endmarker)
 373     def token(self):
 374         try:
 375             return next(self.token_stream)
 376         except StopIteration:
 377             return None
 378
 379 ##########   Parser (tokens -> AST) ######
 380
 381 # also part of Ply
 382 #import yacc
 383
 384 # I use the Python AST
 385 #from compiler import ast
 386 import ast
 387
 388 # Helper function
 389 def Assign(left, right):
 390     names = []
 391     if isinstance(left, ast.Name):
 392         # Single assignment on left
 393         return ast.Assign([ast.Name(left.id, ast.Store())], right)
 394     elif isinstance(left, ast.Tuple):
 395         # List of things - make sure they are Name nodes
 396         names = []
 397         for child in left.getChildren():
 398             if not isinstance(child, ast.Name):
 399                 raise SyntaxError("that assignment not supported")
 400             names.append(child.name)
 401         ass_list = [ast.AssName(name, 'OP_ASSIGN') for name in names]
 402         return ast.Assign([ast.AssTuple(ass_list)], right)
 403     else:
 404         raise SyntaxError("Can't do that yet")
 405
 406
 407 # The grammar comments come from Python's Grammar/Grammar file
 408
 409 ## NB: compound_stmt in single_input is followed by extra NEWLINE!
 410 # file_input: (NEWLINE | stmt)* ENDMARKER
 411 def p_file_input_end(p):
 412     """file_input_end : file_input ENDMARKER"""
 413     print ("end", p[1])
 414     p[0] = p[1]
 415
 416 def p_file_input(p):
 417     """file_input : file_input NEWLINE
 418                   | file_input stmt
 419                   | NEWLINE
 420                   | stmt"""
 421     if isinstance(p[len(p)-1], str):
 422         if len(p) == 3:
 423             p[0] = p[1]
 424         else:
 425             p[0] = [] # p == 2 --> only a blank line
 426     else:
 427         if len(p) == 3:
 428             p[0] = p[1] + p[2]
 429         else:
 430             p[0] = p[1]
 431
 432
 433 # funcdef: [decorators] 'def' NAME parameters ':' suite
 434 # ignoring decorators
 435 def p_funcdef(p):
 436     "funcdef : DEF NAME parameters COLON suite"
 437     p[0] = ast.Function(None, p[2], list(p[3]), (), 0, None, p[5])
 438
 439 # parameters: '(' [varargslist] ')'
 440 def p_parameters(p):
 441     """parameters : LPAR RPAR
 442                   | LPAR varargslist RPAR"""
 443     if len(p) == 3:
 444         p[0] = []
 445     else:
 446         p[0] = p[2]
 447
 448
 449 # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) |
 450 # highly simplified
 451 def p_varargslist(p):
 452     """varargslist : varargslist COMMA NAME
 453                    | NAME"""
 454     if len(p) == 4:
 455         p[0] = p[1] + p[3]
 456     else:
 457         p[0] = [p[1]]
 458
 459 # stmt: simple_stmt | compound_stmt
 460 def p_stmt_simple(p):
 461     """stmt : simple_stmt"""
 462     # simple_stmt is a list
 463     p[0] = p[1]
 464
 465 def p_stmt_compound(p):
 466     """stmt : compound_stmt"""
 467     p[0] = [p[1]]
 468
 469 # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 470 def p_simple_stmt(p):
 471     """simple_stmt : small_stmts NEWLINE
 472                    | small_stmts SEMICOLON NEWLINE"""
 473     p[0] = p[1]
 474
 475 def p_small_stmts(p):
 476     """small_stmts : small_stmts SEMICOLON small_stmt
 477                    | small_stmt"""
 478     if len(p) == 4:
 479         p[0] = p[1] + [p[3]]
 480     else:
 481         p[0] = [p[1]]
 482
 483 # small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
 484 #    import_stmt | global_stmt | exec_stmt | assert_stmt
 485 def p_small_stmt(p):
 486     """small_stmt : flow_stmt
 487                   | break_stmt
 488                   | expr_stmt"""
 489     p[0] = p[1]
 490
 491 # expr_stmt: testlist (augassign (yield_expr|testlist) |
 492 #                      ('=' (yield_expr|testlist))*)
 493 # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
 494 #             '<<=' | '>>=' | '**=' | '//=')
 495 def p_expr_stmt(p):
 496     """expr_stmt : testlist ASSIGN testlist
 497                  | testlist """
 498     if len(p) == 2:
 499         # a list of expressions
 500         #p[0] = ast.Discard(p[1])
 501         p[0] = p[1]
 502     else:
 503         p[0] = Assign(p[1], p[3])
 504
 505 def p_flow_stmt(p):
 506     "flow_stmt : return_stmt"
 507     p[0] = p[1]
 508
 509 # return_stmt: 'return' [testlist]
 510 def p_return_stmt(p):
 511     "return_stmt : RETURN testlist"
 512     p[0] = ast.Return(p[2])
 513
 514
 515 def p_compound_stmt(p):
 516     """compound_stmt : if_stmt
 517                      | while_stmt
 518                      | for_stmt
 519                      | funcdef
 520     """
 521     p[0] = p[1]
 522
 523 def p_break_stmt(p):
 524     """break_stmt : BREAK
 525     """
 526     p[0] = ast.Break()
 527
 528 def p_for_stmt(p):
 529     """for_stmt : FOR test EQ test TO test COLON suite
 530     """
 531     p[0] = ast.While(p[2], p[4], [])
 532     # auto-add-one (sigh) due to python range
 533     start = p[4]
 534     end = ast.BinOp(p[6], ast.Add(), ast.Constant(1))
 535     it = ast.Call(ast.Name("range"), [start, end], [])
 536     p[0] = ast.For(p[2], it, p[8], [])
 537
 538 def p_while_stmt(p):
 539     """while_stmt : DO WHILE test COLON suite ELSE COLON suite
 540                   | DO WHILE test COLON suite
 541     """
 542     if len(p) == 6:
 543         p[0] = ast.While(p[3], p[5], [])
 544     else:
 545         p[0] = ast.While(p[3], p[5], p[8])
 546
 547 def p_if_stmt(p):
 548     """if_stmt : IF test COLON suite ELSE COLON suite
 549                | IF test COLON suite
 550     """
 551     if len(p) == 5:
 552         p[0] = ast.If(p[2], p[4], [])
 553     else:
 554         p[0] = ast.If(p[2], p[4], p[7])
 555
 556 def p_suite(p):
 557     """suite : simple_stmt
 558              | NEWLINE INDENT stmts DEDENT"""
 559     if len(p) == 2:
 560         p[0] = p[1]
 561     else:
 562         p[0] = p[3]
 563
 564
 565 def p_stmts(p):
 566     """stmts : stmts stmt
 567              | stmt"""
 568     if len(p) == 3:
 569         p[0] = p[1] + p[2]
 570     else:
 571         p[0] = p[1]
 572
 573 ## No using Python's approach because Ply supports precedence
 574
 575 # comparison: expr (comp_op expr)*
 576 # arith_expr: term (('+'|'-') term)*
 577 # term: factor (('*'|'/'|'%'|'//') factor)*
 578 # factor: ('+'|'-'|'~') factor | power
 579 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
 580
 581 def make_lt_compare(arg):
 582     (left, right) = arg
 583     return ast.Compare(left, [ast.Lt()], [right])
 584 def make_gt_compare(arg):
 585     (left, right) = arg
 586     return ast.Compare(left, [ast.Gt()], [right])
 587 def make_eq_compare(arg):
 588     (left, right) = arg
 589     return ast.Compare(left, [ast.Eq()], [right])
 590
 591
 592 binary_ops = {
 593     "+": ast.Add(),
 594     "-": ast.Sub(),
 595     "*": ast.Mult(),
 596     "/": ast.Div(),
 597     "<": make_lt_compare,
 598     ">": make_gt_compare,
 599     "=": make_eq_compare,
 600 }
 601 unary_ops = {
 602     "+": ast.Add,
 603     "-": ast.Sub,
 604     }
 605 precedence = (
 606     ("left", "EQ", "GT", "LT"),
 607     ("left", "PLUS", "MINUS"),
 608     ("left", "MULT", "DIV"),
 609     )
 610
 611 def check_concat(node): # checks if the comparison is already a concat
 612     print (node)
 613     if not isinstance(node, ast.Call):
 614         return [node]
 615     if node[0].id != 'concat':
 616         return node
 617     return node[1]
 618
 619 def p_comparison(p):
 620     """comparison : comparison PLUS comparison
 621                   | comparison MINUS comparison
 622                   | comparison MULT comparison
 623                   | comparison DIV comparison
 624                   | comparison LT comparison
 625                   | comparison EQ comparison
 626                   | comparison GT comparison
 627                   | PLUS comparison
 628                   | MINUS comparison
 629                   | comparison APPEND comparison
 630                   | power"""
 631     if len(p) == 4:
 632         print (list(p))
 633         if p[2] == '||':
 634             l = check_concat(p[1]) + check_concat(p[3])
 635             p[0] = ast.Call(ast.Name("concat"), l, [])
 636         elif p[2] in ['<', '>', '=']:
 637             p[0] = binary_ops[p[2]]((p[1],p[3]))
 638         else:
 639             p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 640     elif len(p) == 3:
 641         p[0] = unary_ops[p[1]](p[2])
 642     else:
 643         p[0] = p[1]
 644
 645 # power: atom trailer* ['**' factor]
 646 # trailers enables function calls (and subscripts).
 647 # I only allow one level of calls
 648 # so this is 'trailer'
 649 def p_power(p):
 650     """power : atom
 651              | atom trailer"""
 652     if len(p) == 2:
 653         p[0] = p[1]
 654     else:
 655         if p[2][0] == "CALL":
 656             p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
 657             #if p[1].id == 'print':
 658             #    p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
 659             #else:
 660             #    p[0] = ast.CallFunc(p[1], p[2][1], None, None)
 661         else:
 662             print (p[2][1])
 663             #raise AssertionError("not implemented %s" % p[2][0])
 664             subs = p[2][1]
 665             if len(subs) == 1:
 666                 idx = subs[0]
 667             else:
 668                 idx = ast.Slice(subs[0], subs[1], None)
 669             p[0] = ast.Subscript(p[1], idx)
 670
 671 def p_atom_name(p):
 672     """atom : NAME"""
 673     p[0] = ast.Name(p[1], ctx=ast.Load())
 674
 675 def p_atom_number(p):
 676     """atom : NUMBER
 677             | STRING"""
 678     p[0] = ast.Constant(p[1])
 679
 680 #'[' [listmaker] ']' |
 681
 682 def p_atom_listmaker(p):
 683     """atom : LBRACK listmaker RBRACK"""
 684     p[0] = p[2]
 685
 686 def p_listmaker(p):
 687     """listmaker : test COMMA listmaker
 688                  | test
 689     """
 690     if len(p) == 2:
 691         p[0] = ast.List([p[1]])
 692     else:
 693         p[0] = ast.List([p[1]] + p[3].nodes)
 694
 695 def p_atom_tuple(p):
 696     """atom : LPAR testlist RPAR"""
 697     p[0] = p[2]
 698
 699 # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 700 def p_trailer(p):
 701     """trailer : trailer_arglist
 702                | trailer_subscript
 703     """
 704     p[0] = p[1]
 705
 706 def p_trailer_arglist(p):
 707     "trailer_arglist : LPAR arglist RPAR"
 708     p[0] = ("CALL", p[2])
 709
 710 def p_trailer_subscript(p):
 711     "trailer_subscript : LBRACK subscript RBRACK"
 712     p[0] = ("SUBS", p[2])
 713
 714 #subscript: '.' '.' '.' | test | [test] ':' [test]
 715
 716 def p_subscript(p):
 717     """subscript : test COLON test
 718                  | test
 719     """
 720     if len(p) == 4:
 721         p[0] = [p[1], p[3]]
 722     else:
 723         p[0] = [p[1]]
 724
 725
 726 # testlist: test (',' test)* [',']
 727 # Contains shift/reduce error
 728 def p_testlist(p):
 729     """testlist : testlist_multi COMMA
 730                 | testlist_multi """
 731     if len(p) == 2:
 732         p[0] = p[1]
 733     else:
 734         # May need to promote singleton to tuple
 735         if isinstance(p[1], list):
 736             p[0] = p[1]
 737         else:
 738             p[0] = [p[1]]
 739     # Convert into a tuple?
 740     if isinstance(p[0], list):
 741         p[0] = ast.Tuple(p[0])
 742
 743 def p_testlist_multi(p):
 744     """testlist_multi : testlist_multi COMMA test
 745                       | test"""
 746     if len(p) == 2:
 747         # singleton
 748         p[0] = p[1]
 749     else:
 750         if isinstance(p[1], list):
 751             p[0] = p[1] + [p[3]]
 752         else:
 753             # singleton -> tuple
 754             p[0] = [p[1], p[3]]
 755
 756
 757 # test: or_test ['if' or_test 'else' test] | lambdef
 758 #  as I don't support 'and', 'or', and 'not' this works down to 'comparison'
 759 def p_test(p):
 760     "test : comparison"
 761     p[0] = p[1]
 762
 763
 764
 765 # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
 766 # XXX INCOMPLETE: this doesn't allow the trailing comma
 767 def p_arglist(p):
 768     """arglist : arglist COMMA argument
 769                | argument"""
 770     if len(p) == 4:
 771         p[0] = p[1] + [p[3]]
 772     else:
 773         p[0] = [p[1]]
 774
 775 # argument: test [gen_for] | test '=' test  # Really [keyword '='] test
 776 def p_argument(p):
 777     "argument : test"
 778     p[0] = p[1]
 779
 780 def p_error(p):
 781     #print "Error!", repr(p)
 782     raise SyntaxError(p)
 783
 784
 785 class GardenSnakeParser(object):
 786     def __init__(self, lexer = None):
 787         if lexer is None:
 788             lexer = IndentLexer(debug=1)
 789         self.lexer = lexer
 790         self.parser = yacc.yacc(start="file_input_end",
 791                                 debug=False, write_tables=False)
 792
 793     def parse(self, code):
 794         self.lexer.input(code)
 795         result = self.parser.parse(lexer = self.lexer, debug=False)
 796         return ast.Module(result)
 797
 798
 799 ###### Code generation ######
 800
 801 #from compiler import misc, syntax, pycodegen
 802
 803 class GardenSnakeCompiler(object):
 804     def __init__(self):
 805         self.parser = GardenSnakeParser()
 806     def compile(self, code, mode="exec", filename="<string>"):
 807         tree = self.parser.parse(code)
 808         print ("snake")
 809         pprint(tree)
 810         return tree
 811         #misc.set_filename(filename, tree)
 812         return compile(tree, mode="exec", filename="<string>")
 813         #syntax.check(tree)
 814         gen = pycodegen.ModuleCodeGenerator(tree)
 815         code = gen.getCode()
 816         return code
 817
 818 ####### Test code #######
 819
 820 from soc.decoder.power_fieldsn import create_sigdecode
 821
 822 bpermd = r"""
 823 perm <- [0] * 8
 824 if index < 64:
 825     index <- (RS)[8*i:8*i+7]
 826 RA <- [0]*56 || perm[0:7]
 827 print (RA)
 828 """
 829
 830 bpermd = r"""
 831 if index < 64 then index <- 0
 832 else index <- 5
 833 do while index < 5
 834     index <- 0
 835     leave
 836 for i = 0 to 7
 837     index <- 0
 838 """
 839
 840 _bpermd = r"""
 841 for i = 0 to 7
 842    index <- (RS)[8*i:8*i+7]
 843    if index < 64 then
 844         permi <- (RB)[index]
 845    else
 846         permi <- 0
 847 RA <- [0]*56|| perm[0:7]
 848 """
 849
 850 cnttzd = """
 851 n  <- 0
 852 do while n < 64
 853    if (RS)[63-n] = 0b1 then
 854         leave
 855    n  <- n + 1
 856 RA <- EXTZ64(n)
 857 """
 858
 859 code = cnttzd
 860 #code = bpermd
 861
 862 lexer = IndentLexer(debug=1)
 863 # Give the lexer some input
 864 print ("code")
 865 print (code)
 866 lexer.input(code)
 867
 868 # Tokenize
 869 while True:
 870     tok = lexer.token()
 871     if not tok:
 872         break      # No more input
 873     print(tok)
 874
 875 #sys.exit(0)
 876
 877 # Set up the GardenSnake run-time environment
 878 def print_(*args):
 879     print ("args", args)
 880     print ("-->", " ".join(map(str,args)))
 881
 882 #d = copy(globals())
 883 d = {}
 884 d["print"] = print_
 885
 886 sd = create_sigdecode()
 887 print ("forms", sd.df.forms)
 888 for f in sd.df.FormX:
 889     print (f)
 890
 891 _compile = GardenSnakeCompiler().compile
 892
 893 tree = _compile(code, mode="single", filename="string")
 894 import ast
 895 tree = ast.fix_missing_locations(tree)
 896 print ( ast.dump(tree) )
 897
 898 print ("astor dump")
 899 print (astor.dump_tree(tree))
 900 print ("to source")
 901 source = astor.to_source(tree)
 902 print (source)
 903
 904 #from compiler import parse
 905 #tree = parse(code, "exec")
 906
 907 print (compiled_code)
 908
 909 exec (compiled_code, d)
 910 print ("Done")
 911
 912 #print d
 913 #print l