parser.py

   1 # Based on GardenSnake - a parser generator demonstration program
   2 # GardenSnake was released into the Public Domain by Andrew Dalke.
   3
   4 # Portions of this work are derived from Python's Grammar definition
   5 # and may be covered under the Python copyright and license
   6 #
   7 #          Andrew Dalke / Dalke Scientific Software, LLC
   8 #             30 August 2006 / Cape Town, South Africa
   9
  10 # Modifications for inclusion in PLY distribution
  11 from pprint import pprint
  12 from ply import lex, yacc
  13 import astor
  14 from copy import deepcopy
  15
  16 from openpower.decoder.power_decoder import create_pdecode
  17 from openpower.decoder.pseudo.lexer import (
  18     IndentLexer, raise_syntax_error, SyntaxError2)
  19 from openpower.decoder.orderedset import OrderedSet
  20 from openpower.decoder.power_enums import BFP_FLAG_NAMES
  21
  22 # I use the Python AST
  23 #from compiler import ast
  24 import ast
  25
  26 # Helper function
  27
  28 regs = ['RA', 'RS', 'RB', 'RC', 'RT']
  29 fregs = ['FRA', 'FRS', 'FRB', 'FRC', 'FRT', 'FRS']
  30 SPECIAL_HELPERS = {'concat', 'MEM', 'GPR', 'FPR', 'SPR', 'pow'}
  31 SUBS_TO_ATTR_EXCEPTIONS = SPECIAL_HELPERS | {'CR'}
  32
  33
  34 # I implemented INDENT / DEDENT generation as a post-processing filter
  35
  36 # The original lex token stream contains WS and NEWLINE characters.
  37 # WS will only occur before any other tokens on a line.
  38
  39 # I have three filters.  One tags tokens by adding two attributes.
  40 # "must_indent" is True if the token must be indented from the
  41 # previous code.  The other is "at_line_start" which is True for WS
  42 # and the first non-WS/non-NEWLINE on a line.  It flags the check so
  43 # see if the new line has changed indication level.
  44
  45
  46 # No using Python's approach because Ply supports precedence
  47
  48 # comparison: expr (comp_op expr)*
  49 # arith_expr: term (('+'|'-') term)*
  50 # term: factor (('*'|'/'|'%'|'//') factor)*
  51 # factor: ('+'|'-'|'~') factor | power
  52 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
  53
  54 def make_le_compare(arg):
  55     (left, right) = arg
  56     return ast.Call(ast.Name("le", ast.Load()), [left, right], [])
  57
  58
  59 def make_ge_compare(arg):
  60     (left, right) = arg
  61     return ast.Call(ast.Name("ge", ast.Load()), [left, right], [])
  62
  63
  64 def make_lt_compare(arg):
  65     (left, right) = arg
  66     return ast.Call(ast.Name("lt", ast.Load()), [left, right], [])
  67
  68
  69 def make_gt_compare(arg):
  70     (left, right) = arg
  71     return ast.Call(ast.Name("gt", ast.Load()), [left, right], [])
  72
  73
  74 def make_eq_compare(arg):
  75     (left, right) = arg
  76     return ast.Call(ast.Name("eq", ast.Load()), [left, right], [])
  77
  78
  79 def make_ne_compare(arg):
  80     (left, right) = arg
  81     return ast.Call(ast.Name("ne", ast.Load()), [left, right], [])
  82
  83
  84 binary_ops = {
  85     "^": ast.BitXor(),
  86     "&": ast.BitAnd(),
  87     "|": ast.BitOr(),
  88     "+": ast.Add(),
  89     "-": ast.Sub(),
  90     "<<": ast.LShift(),
  91     ">>": ast.RShift(),
  92     "*": ast.Mult(),
  93     "/": ast.FloorDiv(),
  94     "%": ast.Mod(),
  95     "<=": make_le_compare,
  96     ">=": make_ge_compare,
  97     "<": make_lt_compare,
  98     ">": make_gt_compare,
  99     "=": make_eq_compare,
 100     "!=": make_ne_compare,
 101 }
 102 unary_ops = {
 103     "+": ast.UAdd(),
 104     "-": ast.USub(),
 105     "¬": ast.Invert(),
 106 }
 107
 108
 109 def check_concat(node):  # checks if the comparison is already a concat
 110     print("check concat", node)
 111     if not isinstance(node, ast.Call):
 112         return [node]
 113     node_func_id = getattr(node.func, "id", None)
 114     print("func", node_func_id)
 115     if node_func_id != 'concat':
 116         return [node]
 117     if node.keywords:  # a repeated list-constant, don't optimise
 118         return [node]
 119     return node.args
 120
 121
 122 # identify SelectableInt pattern [something] * N
 123 # must return concat(something, repeat=N)
 124 # here is a TEMPORARY hack to support minimal expressions
 125 # such as (XLEN-16), looking for ast.BinOp
 126 # have to keep an eye on this
 127 def identify_sint_mul_pattern(p):
 128     """here we are looking for patterns of this type:
 129         [item] * something
 130     these must specifically be returned as concat(item, repeat=something)
 131     """
 132     #print ("identify_sint_mul_pattern")
 133     # for pat in p:
 134     #    print("    ", astor.dump_tree(pat))
 135     if p[2] != '*':  # multiply
 136         return False
 137     if (not isinstance(p[3], ast.Constant) and  # rhs = Num
 138         not isinstance(p[3], ast.BinOp) and     # rhs = (XLEN-something)
 139             (not isinstance(p[3], ast.Name) and  # rhs = {a variable}
 140              not isinstance(p[3], ast.Attribute))):   # rhs = XLEN
 141         return False
 142     if not isinstance(p[1], ast.List):  # lhs is a list
 143         return False
 144     l = p[1].elts
 145     if len(l) != 1:  # lhs is a list of length 1
 146         return False
 147     return True  # yippee!
 148
 149
 150 ##########   Parser (tokens -> AST) ######
 151
 152 # also part of Ply
 153 #import yacc
 154
 155 # https://www.mathcs.emory.edu/~valerie/courses/fall10/155/resources/op_precedence.html
 156 # python operator precedence
 157 # Highest precedence at top, lowest at bottom.
 158 # Operators in the same box evaluate left to right.
 159 #
 160 # Operator Description
 161 # ()                                                     Parentheses (grouping)
 162 # f(args...)                                             Function call
 163 # x[index:index]                                         Slicing
 164 # x[index]                                               Subscription
 165 # x.attribute                                            Attribute reference
 166 # **                                                     Exponentiation
 167 # ~x                                                     Bitwise not
 168 # +x, -x                                                 Positive, negative
 169 # *, /, %                                                mul, div, remainder
 170 # +, -                                                   Addition, subtraction
 171 # <<, >>                                                 Bitwise shifts
 172 # &                                                      Bitwise AND
 173 # ^                                                      Bitwise XOR
 174 # |                                                      Bitwise OR
 175 # in, not in, is, is not, <, <=,  >,  >=, <>, !=, ==     comp, membership, ident
 176 # not x                                                  Boolean NOT
 177 # and                                                    Boolean AND
 178 # or                                                     Boolean OR
 179 # lambda                                                 Lambda expression
 180
 181
 182 class PowerParser:
 183
 184     precedence = (
 185         ("left", "EQ", "NE", "GT", "LT", "LE", "GE", "LTU", "GTU"),
 186         ("left", "BITOR"),
 187         ("left", "BITXOR"),
 188         ("left", "BITAND"),
 189         ("left", "LSHIFT", "RSHIFT"),
 190         ("left", "PLUS", "MINUS"),
 191         ("left", "MULT", "DIV", "MOD"),
 192         ("left", "INVERT"),
 193     )
 194
 195     def reset(self):
 196         self.gprs = {}
 197         if self.form is not None:
 198             form = self.sd.sigforms[self.form]
 199             print(form)
 200             formkeys = form._asdict().keys()
 201         else:
 202             formkeys = []
 203         self.declared_vars = set()
 204         self.fnparm_vars = set()
 205         for rname in regs + fregs:
 206             self.gprs[rname] = None
 207             self.declared_vars.add(rname)
 208         self.available_op_fields = set()
 209         for k in formkeys:
 210             if k not in self.gprs:
 211                 if k == 'SPR':  # sigh, lower-case to not conflict
 212                     k = k.lower()
 213                 self.available_op_fields.add(k)
 214         self.op_fields = OrderedSet()
 215         self.read_regs = OrderedSet()
 216         self.uninit_regs = OrderedSet()
 217         self.write_regs = OrderedSet()
 218         self.special_regs = OrderedSet()  # see p_atom_name
 219
 220     def __init__(self, form, include_carry_in_write=False, helper=False):
 221         self.include_ca_in_write = include_carry_in_write
 222         self.helper = helper
 223         self.form = form
 224         self.filename = None
 225         self.input_text = None
 226         self.reset()
 227
 228     # The grammar comments come from Python's Grammar/Grammar file
 229
 230     # NB: compound_stmt in single_input is followed by extra NEWLINE!
 231     # file_input: (NEWLINE | stmt)* ENDMARKER
 232
 233     def p_file_input_end(self, p):
 234         """file_input_end : file_input ENDMARKER"""
 235         print("end", p[1])
 236         p[0] = p[1]
 237
 238     def p_file_input(self, p):
 239         """file_input : file_input NEWLINE
 240                       | file_input stmt
 241                       | NEWLINE
 242                       | stmt"""
 243         if isinstance(p[len(p)-1], str):
 244             if len(p) == 3:
 245                 p[0] = p[1]
 246             else:
 247                 p[0] = []  # p == 2 --> only a blank line
 248         else:
 249             if len(p) == 3:
 250                 p[0] = p[1] + p[2]
 251             else:
 252                 p[0] = p[1]
 253
 254     # funcdef: [decorators] 'def' NAME parameters ':' suite
 255     # ignoring decorators
 256
 257     def p_funcdef(self, p):
 258         "funcdef : DEF NAME parameters COLON suite"
 259         # pypy workaround
 260         p[0] = ast.FunctionDef()
 261         p[0].name = p[2]
 262         p[0].args = p[3]
 263         p[0].body = p[5]
 264         p[0].decorator_list = []
 265         p[0].lineno = p.lineno(2)
 266         # reset function parameters after suite is identified
 267         self.fnparm_vars = set()
 268
 269     # parameters: '(' [varargslist] ')'
 270     def p_parameters(self, p):
 271         """parameters : LPAR RPAR
 272                       | LPAR varargslist RPAR"""
 273         args = []
 274         if self.helper:
 275             args.append("self")
 276         if len(p) != 3:
 277             args += p[2]
 278         p[0] = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[])
 279         # during the time between parameters identified and suite is not
 280         # there is a window of opportunity to declare the function parameters
 281         # in-scope, for use to not overwrite them with auto-assign
 282         self.fnparm_vars = set()
 283         for arg in args:
 284             print("adding fn parm", arg)
 285             self.fnparm_vars.add(arg)
 286
 287     # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] |
 288     # '**' NAME) |
 289     # highly simplified
 290
 291     def p_varargslist(self, p):
 292         """varargslist : varargslist COMMA NAME
 293                        | NAME"""
 294         if len(p) == 4:
 295             print(p[1], p[3])
 296             p[0] = p[1] + [p[3]]
 297         else:
 298             p[0] = [p[1]]
 299
 300     # stmt: simple_stmt | compound_stmt
 301     def p_stmt_simple(self, p):
 302         """stmt : simple_stmt"""
 303         # simple_stmt is a list
 304         p[0] = p[1]
 305
 306     def p_stmt_compound(self, p):
 307         """stmt : compound_stmt"""
 308         p[0] = [p[1]]
 309
 310     # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 311     def p_simple_stmt(self, p):
 312         """simple_stmt : small_stmts NEWLINE
 313                        | small_stmts SEMICOLON NEWLINE"""
 314         p[0] = p[1]
 315
 316     def p_small_stmts(self, p):
 317         """small_stmts : small_stmts SEMICOLON small_stmt
 318                        | small_stmt"""
 319         if len(p) == 4:
 320             p[0] = p[1] + [p[3]]
 321         elif isinstance(p[1], list):
 322             p[0] = p[1]
 323         else:
 324             p[0] = [p[1]]
 325
 326     # small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
 327     #    import_stmt | global_stmt | exec_stmt | assert_stmt
 328     def p_small_stmt(self, p):
 329         """small_stmt : flow_stmt
 330                       | break_stmt
 331                       | expr_stmt"""
 332         if isinstance(p[1], ast.Call):
 333             p[0] = ast.Expr(p[1])
 334         elif isinstance(p[1], ast.Name) and p[1].id not in SPECIAL_HELPERS:
 335             fname = p[1].id
 336             name = ast.Name("self", ast.Load())
 337             name = ast.Attribute(name, fname, ast.Load())
 338             p[0] = ast.Expr(ast.Call(name, [], []))
 339         elif isinstance(p[1], ast.expr):
 340             p[0] = ast.Expr(p[1])
 341         else:
 342             p[0] = p[1]
 343
 344     # expr_stmt: testlist (augassign (yield_expr|testlist) |
 345     #                      ('=' (yield_expr|testlist))*)
 346     # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
 347     #             '<<=' | '>>=' | '**=' | '//=')
 348     def p_expr_stmt(self, p):
 349         """expr_stmt : testlist ASSIGNEA testlist
 350                      | testlist ASSIGN testlist
 351                      | testlist """
 352         print("expr_stmt", p)
 353         if len(p) == 2:
 354             # a list of expressions
 355             #p[0] = ast.Discard(p[1])
 356             p[0] = p[1]
 357         else:
 358             iea_mode = p[2] == '<-iea'
 359             if iea_mode:
 360                 print ("iea_mode")
 361             name = None
 362             autoassign = False
 363             if isinstance(p[1], ast.Name):
 364                 name = p[1].id
 365             elif isinstance(p[1], ast.Subscript):
 366                 print("assign subscript", p[1].value,
 367                       self.declared_vars,
 368                       self.fnparm_vars,
 369                       self.special_regs)
 370                 print(astor.dump_tree(p[1]))
 371                 print(astor.dump_tree(p[3]))
 372                 if isinstance(p[1].value, ast.Name):
 373                     name = p[1].value.id
 374                     print("assign subscript value to name", name)
 375                     if name in self.gprs:
 376                         # add to list of uninitialised
 377                         self.uninit_regs.add(name)
 378                     # work out if this is an ininitialised variable
 379                     # that should be auto-assigned simply by being "sliced"
 380                     autoassign = (name not in self.declared_vars and
 381                                   name not in self.fnparm_vars and
 382                                   name not in self.special_regs)
 383             elif isinstance(p[1], (ast.Attribute, ast.Tuple)):
 384                 pass
 385             elif isinstance(p[1], ast.Call) and p[1].func.id in \
 386                     ['GPR', 'FPR', 'SPR']:
 387                 print(astor.dump_tree(p[1]))
 388                 # replace GPR(x) with GPR[x]
 389                 idx = p[1].args[0].id
 390                 if idx in regs + fregs:
 391                     ridx = ast.Name("_%s" % idx, ast.Load())
 392                 else:
 393                     ridx = ast.Name(idx, ast.Load())
 394                 p[1] = ast.Subscript(p[1].func, ridx, ast.Load())
 395                 if idx in self.gprs:
 396                     self.read_regs.add(idx)  # add to list of regs to read
 397             elif isinstance(p[1], ast.Call) and p[1].func.id == 'MEM':
 398                 print("mem assign")
 399                 print(astor.dump_tree(p[1]))
 400                 p[1].func.id = "memassign"  # change function name to set
 401                 p[1].args.append(p[3])
 402                 p[0] = p[1]
 403                 print("mem rewrite")
 404                 print(astor.dump_tree(p[0]))
 405                 return
 406             else:
 407                 print(astor.dump_tree(p[1]))
 408                 raise_syntax_error("help, help", self.filename,
 409                                    p.slice[2].lineno, p.slice[2].lexpos,
 410                                    self.input_text)
 411             print("expr assign", name, p[1], "to", p[3])
 412             print(astor.dump_tree(p[3]))
 413             if isinstance(p[3], ast.Name):
 414                 toname = p[3].id
 415                 if toname in self.gprs:
 416                     self.read_regs.add(toname)
 417             if name and name in self.gprs:
 418                 self.write_regs.add(name)  # add to list of regs to write
 419
 420             # copy rhs -- see openpower.decoder.helpers.copy_assign_rhs()'s
 421             # documentation for why we need this
 422             copy_fn = ast.Name("copy_assign_rhs", ast.Load())
 423             rhs = ast.Call(copy_fn, (p[3],), [])
 424             p[0] = self.Assign(autoassign, name, p[1], rhs, iea_mode,
 425                                p.slice[2])
 426             if name:
 427                 self.declared_vars.add(name)
 428
 429     def p_flow_stmt(self, p):
 430         "flow_stmt : return_stmt"
 431         p[0] = p[1]
 432
 433     # return_stmt: 'return' [testlist]
 434     def p_return_stmt(self, p):
 435         "return_stmt : RETURN testlist"
 436         p[0] = ast.Return(p[2])
 437
 438     def p_compound_stmt(self, p):
 439         """compound_stmt : if_stmt
 440                          | while_stmt
 441                          | switch_stmt
 442                          | for_stmt
 443                          | funcdef
 444         """
 445         p[0] = p[1]
 446
 447     def p_break_stmt(self, p):
 448         """break_stmt : BREAK
 449         """
 450         p[0] = ast.Break()
 451
 452     def p_for_stmt(self, p):
 453         """for_stmt : FOR atom EQ comparison TO comparison COLON suite
 454                     | DO atom EQ comparison TO comparison COLON suite
 455         """
 456         start = p[4]
 457         end = p[6]
 458         it = ast.Call(ast.Name("RANGE", ast.Load()), (start, end), [])
 459         p[0] = ast.For(p[2], it, p[8], [])
 460
 461     def p_while_stmt(self, p):
 462         """while_stmt : DO WHILE test COLON suite ELSE COLON suite
 463                       | DO WHILE test COLON suite
 464         """
 465         if len(p) == 6:
 466             p[0] = ast.While(p[3], p[5], [])
 467         else:
 468             p[0] = ast.While(p[3], p[5], p[8])
 469
 470     def p_switch_smt(self, p):
 471         """switch_stmt : SWITCH LPAR atom RPAR COLON NEWLINE INDENT switches DEDENT
 472         """
 473         switchon = p[3]
 474         print("switch stmt")
 475         print(astor.dump_tree(p[1]))
 476
 477         cases = []
 478         current_cases = []  # for deferral
 479         for (case, suite) in p[8]:
 480             print("for", case, suite)
 481             if suite is None:
 482                 for c in case:
 483                     current_cases.append(ast.Num(c))
 484                 continue
 485             if case == 'default':  # last
 486                 break
 487             for c in case:
 488                 current_cases.append(ast.Num(c))
 489             print("cases", current_cases)
 490             compare = ast.Compare(switchon, [ast.In()],
 491                                   [ast.List(current_cases, ast.Load())])
 492             current_cases = []
 493             cases.append((compare, suite))
 494
 495         print("ended", case, current_cases)
 496         if case == 'default':
 497             if current_cases:
 498                 compare = ast.Compare(switchon, [ast.In()],
 499                                       [ast.List(current_cases, ast.Load())])
 500                 cases.append((compare, suite))
 501             cases.append((None, suite))
 502
 503         cases.reverse()
 504         res = []
 505         for compare, suite in cases:
 506             print("after rev", compare, suite)
 507             if compare is None:
 508                 assert len(res) == 0, "last case should be default"
 509                 res = suite
 510             else:
 511                 if not isinstance(res, list):
 512                     res = [res]
 513                 res = ast.If(compare, suite, res)
 514         p[0] = res
 515
 516     def p_switches(self, p):
 517         """switches : switch_list switch_default
 518                     | switch_default
 519         """
 520         if len(p) == 3:
 521             p[0] = p[1] + [p[2]]
 522         else:
 523             p[0] = [p[1]]
 524
 525     def p_switch_list(self, p):
 526         """switch_list : switch_case switch_list
 527                        | switch_case
 528         """
 529         if len(p) == 3:
 530             p[0] = [p[1]] + p[2]
 531         else:
 532             p[0] = [p[1]]
 533
 534     def p_switch_case(self, p):
 535         """switch_case : CASE LPAR atomlist RPAR COLON suite
 536         """
 537         # XXX bad hack
 538         if isinstance(p[6][0], ast.Name) and p[6][0].id == 'fallthrough':
 539             p[6] = None
 540         p[0] = (p[3], p[6])
 541
 542     def p_switch_default(self, p):
 543         """switch_default : DEFAULT COLON suite
 544         """
 545         p[0] = ('default', p[3])
 546
 547     def p_atomlist(self, p):
 548         """atomlist : atom COMMA atomlist
 549                     | atom
 550         """
 551         assert isinstance(p[1], ast.Constant), "case must be numbers"
 552         if len(p) == 4:
 553             p[0] = [p[1].value] + p[3]
 554         else:
 555             p[0] = [p[1].value]
 556
 557     def p_if_stmt(self, p):
 558         """if_stmt : IF test COLON suite ELSE COLON if_stmt
 559                    | IF test COLON suite ELSE COLON suite
 560                    | IF test COLON suite
 561         """
 562         if len(p) == 8 and isinstance(p[7], ast.If):
 563             p[0] = ast.If(p[2], p[4], [p[7]])
 564         elif len(p) == 5:
 565             p[0] = ast.If(p[2], p[4], [])
 566         else:
 567             p[0] = ast.If(p[2], p[4], p[7])
 568
 569     def p_suite(self, p):
 570         """suite : simple_stmt
 571                  | NEWLINE INDENT stmts DEDENT"""
 572         if len(p) == 2:
 573             p[0] = p[1]
 574         else:
 575             p[0] = p[3]
 576
 577     def p_stmts(self, p):
 578         """stmts : stmts stmt
 579                  | stmt"""
 580         if len(p) == 3:
 581             p[0] = p[1] + p[2]
 582         else:
 583             p[0] = p[1]
 584
 585     def p_comparison(self, p):
 586         """comparison : comparison PLUS comparison
 587                       | comparison MINUS comparison
 588                       | comparison MULT comparison
 589                       | comparison LSHIFT comparison
 590                       | comparison RSHIFT comparison
 591                       | comparison DIV comparison
 592                       | comparison MOD comparison
 593                       | comparison EQ comparison
 594                       | comparison NE comparison
 595                       | comparison LE comparison
 596                       | comparison GE comparison
 597                       | comparison LTU comparison
 598                       | comparison GTU comparison
 599                       | comparison LT comparison
 600                       | comparison GT comparison
 601                       | comparison BITOR comparison
 602                       | comparison BITXOR comparison
 603                       | comparison BITAND comparison
 604                       | PLUS comparison
 605                       | MINUS comparison
 606                       | INVERT comparison
 607                       | comparison APPEND comparison
 608                       | power"""
 609         if len(p) == 4:
 610             print(list(p))
 611             if p[2] == '<u':
 612                 p[0] = ast.Call(ast.Name("ltu", ast.Load()), (p[1], p[3]), [])
 613             elif p[2] == '>u':
 614                 p[0] = ast.Call(ast.Name("gtu", ast.Load()), (p[1], p[3]), [])
 615             elif p[2] == '||':
 616                 l = check_concat(p[1]) + check_concat(p[3])
 617                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, [])
 618             elif p[2] in ['/', '%']:
 619                 # bad hack: if % or / used anywhere other than div/mod ops,
 620                 # do % or /.  however if the argument names are "dividend"
 621                 # we must call the special trunc_divs and trunc_rems functions
 622                 l, r = p[1], p[3]
 623                 # actual call will be "dividend / divisor" - just check
 624                 # LHS name
 625                 # XXX DISABLE BAD HACK (False)
 626                 if False and isinstance(l, ast.Name) and l.id == 'dividend':
 627                     if p[2] == '/':
 628                         fn = 'trunc_divs'
 629                     else:
 630                         fn = 'trunc_rems'
 631                     # return "function trunc_xxx(l, r)"
 632                     p[0] = ast.Call(ast.Name(fn, ast.Load()), (l, r), [])
 633                 else:
 634                     # return "l {binop} r"
 635                     p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 636             elif p[2] in ['<', '>', '=', '<=', '>=', '!=']:
 637                 p[0] = binary_ops[p[2]]((p[1], p[3]))
 638             elif identify_sint_mul_pattern(p):
 639                 keywords = [ast.keyword(arg='repeat', value=p[3])]
 640                 l = p[1].elts
 641                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 642             else:
 643                 p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 644                 # HORRENDOUS hack, add brackets around the bin-op by
 645                 # creating a function call with a *blank* function name!
 646                 # XXX argh doesn't work because of analysis of
 647                 # identify_sint_pattern
 648                 #p[0] = ast.Call(ast.Name("", ast.Load()), [p[0]], [])
 649         elif len(p) == 3:
 650             if isinstance(p[2], str) and p[2] == '-':
 651                 p[0] = ast.UnaryOp(unary_ops[p[2]], p[1])
 652             else:
 653                 p[0] = ast.UnaryOp(unary_ops[p[1]], p[2])
 654         else:
 655             p[0] = p[1]
 656
 657     # power: atom trailer* ['**' factor]
 658     # trailers enables function calls (and subscripts).
 659     # so this is 'trailerlist'
 660     def p_power(self, p):
 661         """power : atom
 662                  | atom trailerlist"""
 663         if len(p) == 2:
 664             print("power dump atom notrailer")
 665             print(astor.dump_tree(p[1]))
 666             p[0] = p[1]
 667         else:
 668             print("power dump atom")
 669             print(astor.dump_tree(p[1]))
 670             print("power dump trailerlist")
 671             print(astor.dump_tree(p[2]))
 672             p[0] = self.apply_trailer(p[1], p[2], self.read_regs)
 673             if isinstance(p[1], ast.Name):
 674                 name = p[1].id
 675                 if name in regs + fregs:
 676                     self.read_regs.add(name)
 677
 678     def p_atom_name(self, p):
 679         """atom : NAME"""
 680         name = p[1]
 681         if name in self.available_op_fields:
 682             self.op_fields.add(name)
 683         if name in ['overflow', 'CR0']:
 684             self.write_regs.add(name)
 685         if self.include_ca_in_write:
 686             if name in ['CA', 'CA32']:
 687                 self.write_regs.add(name)
 688         if name in ['CR', 'LR', 'CTR', 'TAR', 'FPSCR', 'MSR',
 689                     'SVSTATE', 'SVREMAP', 'SRR0', 'SRR1',
 690                     'SVSHAPE0', 'SVSHAPE1', 'SVSHAPE2', 'SVSHAPE3']:
 691             self.special_regs.add(name)
 692             self.write_regs.add(name)  # and add to list to write
 693         if name in ('XLEN', ) or name in BFP_FLAG_NAMES:
 694             attr = ast.Name("self", ast.Load())
 695             p[0] = ast.Attribute(attr, name, ast.Load(), lineno=p.lineno(1))
 696         else:
 697             p[0] = ast.Name(id=name, ctx=ast.Load(), lineno=p.lineno(1))
 698
 699     def p_atom_number(self, p):
 700         """atom : BINARY
 701                 | NUMBER
 702                 | HEX
 703                 | STRING"""
 704         # SelectableInt isn't a valid ast.Constant,
 705         # but we use it anyway since it produces nicer source (hex constants):
 706         # if not isinstance(p[1], (str, int)):
 707         #     # SelectableInt isn't a valid ast.Constant
 708         #     p[0] = ast.parse(repr(p[1]), mode='eval').body
 709         #     return
 710         p[0] = ast.Constant(p[1])
 711
 712     # '[' [listmaker] ']' |
 713
 714     def p_atom_listmaker(self, p):
 715         """atom : LBRACK listmaker RBRACK"""
 716         p[0] = p[2]
 717
 718     def p_listmaker(self, p):
 719         """listmaker : test COMMA listmaker
 720                      | test
 721         """
 722         if len(p) == 2:
 723             p[0] = ast.List([p[1]], ast.Load())
 724         else:
 725             p[0] = ast.List([p[1]] + p[3].nodes, ast.Load())
 726
 727     def p_atom_tuple(self, p):
 728         """atom : LPAR testlist RPAR"""
 729         print("tuple", p[2])
 730         print("astor dump")
 731         print(astor.dump_tree(p[2]))
 732
 733         if isinstance(p[2], ast.Name):
 734             name = p[2].id
 735             print("tuple name", name)
 736             if name in self.gprs:
 737                 self.read_regs.add(name)  # add to list of regs to read
 738                 #p[0] = ast.Subscript(ast.Name("GPR", ast.Load()), ast.Str(p[2].id))
 739                 # return
 740             p[0] = p[2]
 741         elif isinstance(p[2], ast.BinOp):
 742             if isinstance(p[2].left, ast.Name) and \
 743                isinstance(p[2].right, ast.Constant) and \
 744                     p[2].right.value == 0 and \
 745                     p[2].left.id in self.gprs:
 746                 rid = p[2].left.id
 747                 self.read_regs.add(rid)  # add to list of regs to read
 748                 # create special call to GPR.getz or FPR.getz
 749                 if rid in fregs:
 750                     gprz = ast.Name("FPR", ast.Load())
 751                 else:
 752                     gprz = ast.Name("GPR", ast.Load())
 753                 # get testzero function
 754                 gprz = ast.Attribute(gprz, "getz", ast.Load())
 755                 # *sigh* see class GPR.  we need index itself not reg value
 756                 ridx = ast.Name("_%s" % rid, ast.Load())
 757                 rvalue = ast.Name(rid, ast.Load())
 758                 p[0] = ast.Call(gprz, [ridx, rvalue], [])
 759                 print("tree", astor.dump_tree(p[0]))
 760             else:
 761                 p[0] = p[2]
 762         else:
 763             p[0] = p[2]
 764
 765     def p_trailerlist(self, p):
 766         """trailerlist : trailer trailerlist
 767                        | trailer
 768         """
 769         if len(p) == 2:
 770             p[0] = p[1]
 771         else:
 772             p[0] = ("TLIST", p[1], p[2])
 773
 774     # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 775     def p_trailer(self, p):
 776         """trailer : trailer_arglist
 777                    | trailer_subscript
 778                    | trailer_attr
 779         """
 780         p[0] = p[1]
 781
 782     def p_trailer_arglist(self, p):
 783         """trailer_arglist : LPAR arglist RPAR
 784                            | LPAR RPAR
 785         """
 786         args = [] if len(p) == 3 else p[2]
 787         p[0] = ("CALL", args, p.slice[1])
 788
 789     def p_trailer_subscript(self, p):
 790         "trailer_subscript : LBRACK subscript RBRACK"
 791         p[0] = ("SUBS", p[2])
 792
 793     def p_trailer_attr(self, p):
 794         "trailer_attr : PERIOD NAME"
 795         p[0] = ("ATTR", p[2])
 796
 797     # subscript: '.' '.' '.' | test | [test] ':' [test]
 798
 799     def p_subscript(self, p):
 800         """subscript : test COLON test
 801                      | test
 802         """
 803         if len(p) == 4:
 804             # add one to end
 805             if isinstance(p[3], ast.Constant):
 806                 end = ast.Constant(p[3].value+1)
 807             else:
 808                 end = ast.BinOp(p[3], ast.Add(), ast.Constant(1))
 809             p[0] = [p[1], end]
 810         else:
 811             p[0] = [p[1]]
 812
 813     # testlist: test (',' test)* [',']
 814     # Contains shift/reduce error
 815
 816     def p_testlist(self, p):
 817         """testlist : testlist_multi COMMA
 818                     | testlist_multi """
 819         if len(p) == 2:
 820             p[0] = p[1]
 821         else:
 822             # May need to promote singleton to tuple
 823             if isinstance(p[1], list):
 824                 p[0] = p[1]
 825             else:
 826                 p[0] = [p[1]]
 827         # Convert into a tuple?
 828         if isinstance(p[0], list):
 829             v = ast.Tuple()  # pypy workaround
 830             v.elts = p[0]
 831             p[0] = v
 832
 833     def p_testlist_multi(self, p):
 834         """testlist_multi : testlist_multi COMMA test
 835                           | test"""
 836         if len(p) == 2:
 837             # singleton
 838             p[0] = p[1]
 839         else:
 840             if isinstance(p[1], list):
 841                 p[0] = p[1] + [p[3]]
 842             else:
 843                 # singleton -> tuple
 844                 p[0] = [p[1], p[3]]
 845
 846     # test: or_test ['if' or_test 'else' test] | lambdef
 847     #  as I don't support 'and', 'or', and 'not' this works down to 'comparison'
 848
 849     def p_test(self, p):
 850         """test : comparison
 851                 | comparison QMARK test COLON test"""
 852         if len(p) == 2:
 853             p[0] = p[1]
 854         else:
 855             p[0] = ast.IfExp(test=p[1], body=p[3], orelse=p[5])
 856
 857     # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
 858     # | '**' test)
 859     # XXX INCOMPLETE: this doesn't allow the trailing comma
 860
 861     def p_arglist(self, p):
 862         """arglist : arglist COMMA argument
 863                    | argument"""
 864         if len(p) == 4:
 865             p[0] = p[1] + [p[3]]
 866         else:
 867             p[0] = [p[1]]
 868
 869     # argument: test [gen_for] | test '=' test  # Really [keyword '='] test
 870     def p_argument(self, p):
 871         "argument : test"
 872         p[0] = p[1]
 873
 874     def p_error(self, p):
 875         # print "Error!", repr(p)
 876         raise_syntax_error(str(p), self.filename, p.lineno, p.lexpos,
 877                            self.input_text)
 878
 879     def Assign(self, autoassign, assignname, left, right, iea_mode, eq_tok):
 880         names = []
 881         print("Assign", autoassign, assignname, left, right)
 882         if isinstance(left, ast.Name):
 883             # Single assignment on left
 884             # XXX when doing IntClass, which will have an "eq" function,
 885             # this is how to access it
 886             #   eq = ast.Attribute(left, "eq")   # get eq fn
 887             #   return ast.Call(eq, [right], []) # now call left.eq(right)
 888             return ast.Assign([ast.Name(left.id, ast.Store())], right)
 889         elif isinstance(left, ast.Tuple):
 890             # List of things - make sure they are Name nodes
 891             names = []
 892             for child in left.elts:
 893                 if not isinstance(child, ast.Name):
 894                     raise_syntax_error(
 895                         "that assignment not supported", self.filename,
 896                         eq_tok.lineno, eq_tok.lexpos, self.input_text)
 897                 names.append(child.id)
 898             ass_list = [ast.Name(name, ast.Store()) for name in names]
 899             v = ast.Tuple()  # pypy workaround
 900             v.elts = ass_list
 901             return ast.Assign([v], right)
 902         elif isinstance(left, ast.Attribute):
 903             return ast.Assign([
 904                 ast.Attribute(left.value, left.attr, ast.Store())], right)
 905         elif isinstance(left, ast.Subscript):
 906             ls = left.slice
 907             # XXX changing meaning of "undefined" to a function
 908             # if (isinstance(ls, ast.Slice) and isinstance(right, ast.Name) and
 909             #        right.id == 'undefined'):
 910             #    # undefined needs to be copied the exact same slice
 911             #    right = ast.Subscript(right, ls, ast.Load())
 912             #    return ast.Assign([left], right)
 913             res = ast.Assign([left], right)
 914             if autoassign and isinstance(ls, ast.Slice):
 915                 # hack to create a variable pre-declared based on a slice.
 916                 # dividend[0:32] = (RA)[0:32] will create
 917                 #       dividend = [0] * 32
 918                 #       dividend[0:32] = (RA)[0:32]
 919                 # the declaration makes the slice-assignment "work"
 920                 lower, upper, step = ls.lower, ls.upper, ls.step
 921                 print("lower, upper, step", repr(lower), repr(upper), step)
 922                 # XXX relax constraint that indices on auto-assign have
 923                 # to be constants x[0:32]
 924                 # if not isinstance(lower, ast.Constant) or \
 925                 #   not isinstance(upper, ast.Constant):
 926                 #    return res
 927                 qty = ast.BinOp(upper, binary_ops['-'], lower)
 928                 keywords = [ast.keyword(arg='repeat', value=qty)]
 929                 l = [ast.Num(0)]
 930                 right = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 931                 declare = ast.Assign(
 932                     [ast.Name(assignname, ast.Store())], right)
 933                 return [declare, res]
 934             return res
 935             # XXX HMMM probably not needed...
 936             ls = left.slice
 937             if isinstance(ls, ast.Slice):
 938                 lower, upper, step = ls.lower, ls.upper, ls.step
 939                 print("slice assign", lower, upper, step)
 940                 if step is None:
 941                     ls = (lower, upper, None)
 942                 else:
 943                     ls = (lower, upper, step)
 944                 ls = ast.Tuple(ls)
 945             return ast.Call(ast.Name("selectassign", ast.Load()),
 946                             [left.value, ls, right], [])
 947         else:
 948             print("Assign fail")
 949             raise_syntax_error("Can't do that yet", self.filename,
 950                                eq_tok.lineno, eq_tok.lexpos, self.input_text)
 951
 952     def try_extract_name(self, atom):
 953         if isinstance(atom, ast.Attribute):
 954             if isinstance(atom.value, ast.Name) and atom.value.id == "self":
 955                 return atom.attr
 956             return None
 957         if isinstance(atom, ast.Name):
 958             return atom.id
 959         return None
 960
 961     def try_extract_uppercase_name(self, atom):
 962         name = self.try_extract_name(atom)
 963         if name is None:
 964             return None
 965         # we want to filter out names like _123 that have no letters with
 966         # casing at all, hence the lower() check
 967         if name.upper() != name or name.lower() == name:
 968             return None
 969         return name
 970
 971     def try_get_attr(self, atom, trailer):
 972         if trailer[0] == "ATTR":
 973             return trailer[1]
 974         if trailer[0] != "SUBS":
 975             return None
 976         base_name = self.try_extract_uppercase_name(atom)
 977         if base_name is None or base_name in SUBS_TO_ATTR_EXCEPTIONS:
 978             return None
 979         if len(trailer[1]) != 1:
 980             return None
 981         return self.try_extract_uppercase_name(trailer[1][0])
 982
 983     def apply_trailer(self, atom, trailer, read_regs):
 984         if trailer[0] == "TLIST":
 985             # assume depth of one
 986             atom = self.apply_trailer(atom, trailer[1], read_regs)
 987             trailer = trailer[2]
 988         if trailer[0] == "CALL":
 989             #p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
 990             for arg in trailer[1]:
 991                 if isinstance(arg, ast.Name):
 992                     name = arg.id
 993                     if name in regs + fregs:
 994                         read_regs.add(name)
 995             if atom.id == "SetFX":
 996                 if len(trailer[1]) != 1 or \
 997                         not isinstance(trailer[1][0], ast.Attribute):
 998                     raise_syntax_error(
 999                         "SetFX call must have only one argument that is an "
1000                         "attribute access -- like `SetFX(FPSCR.VXSNAN)`",
1001                         self.filename, trailer[2].lineno, trailer[2].lexpos,
1002                         self.input_text)
1003                 arg = trailer[1][0]
1004                 args = [arg.value, ast.Str(arg.attr)]
1005                 name = ast.Name("self", ast.Load())
1006                 atom = ast.Attribute(name, atom.id, ast.Load())
1007                 return ast.Call(atom, args, [])
1008             # special-case, these functions must NOT be made "self.xxxx"
1009             if atom.id not in SPECIAL_HELPERS:
1010                 name = ast.Name("self", ast.Load())
1011                 atom = ast.Attribute(name, atom.id, ast.Load())
1012             return ast.Call(atom, trailer[1], [])
1013             # if p[1].id == 'print':
1014             #    p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
1015             # else:
1016             #    p[0] = ast.CallFunc(p[1], p[2][1], None, None)
1017         else:
1018             attr = self.try_get_attr(atom, trailer)
1019             if attr is not None:
1020                 return ast.Attribute(atom, attr, ast.Load())
1021             assert trailer[0] == "SUBS"
1022             print("subscript atom", trailer[1])
1023             #raise AssertionError("not implemented %s" % p[2][0])
1024             subs = trailer[1]
1025             if len(subs) == 1:
1026                 idx = subs[0]
1027                 if isinstance(idx, ast.Name) and idx.id in regs + fregs:
1028                     read_regs.add(idx.id)
1029                 if isinstance(idx, ast.Name) and idx.id in regs:
1030                     print("single atom subscript, underscored", idx.id)
1031                     idx = ast.Name("_%s" % idx.id, ast.Load())
1032             else:
1033                 idx = ast.Slice(subs[0], subs[1], None)
1034             # if isinstance(atom, ast.Name) and atom.id == 'CR':
1035                 # atom.id = 'CR' # bad hack
1036                 #print ("apply_trailer Subscript", atom.id, idx)
1037             return ast.Subscript(atom, idx, ast.Load())
1038
1039
1040 _CACHE_DECODER = True
1041 _CACHED_DECODER = None
1042
1043
1044 def _create_cached_decoder():
1045     global _CACHED_DECODER
1046     if _CACHE_DECODER:
1047         if _CACHED_DECODER is None:
1048             _CACHED_DECODER = create_pdecode()
1049         return _CACHED_DECODER
1050     return create_pdecode()
1051
1052
1053 class GardenSnakeParser(PowerParser):
1054     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1055         if form is not None:
1056             self.sd = _create_cached_decoder()
1057         PowerParser.__init__(self, form, incl_carry, helper=helper)
1058         self.debug = debug
1059         self.lexer = IndentLexer(debug=0)
1060         self.tokens = self.lexer.tokens
1061         self.parser = yacc.yacc(module=self, start="file_input_end",
1062                                 debug=debug, write_tables=False)
1063
1064     def parse(self, code):
1065         self.reset()
1066         self.lexer.filename = self.filename
1067         result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
1068         if self.helper:
1069             v = ast.ClassDef()  # pypy workaround
1070             v.name = "ISACallerFnHelper"
1071             v.bases = ["ISACallerHelper"]
1072             v.keywords = []
1073             v.body = result
1074             v.decorator_list = []
1075             result = [v]
1076         return ast.Module(result)
1077
1078
1079 ###### Code generation ######
1080
1081 #from compiler import misc, syntax, pycodegen
1082
1083 _CACHED_PARSERS = {}
1084 _CACHE_PARSERS = True
1085
1086
1087 class GardenSnakeCompiler(object):
1088     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1089         if _CACHE_PARSERS:
1090             try:
1091                 self.parser = _CACHED_PARSERS[debug, form, incl_carry, helper]
1092             except KeyError:
1093                 self.parser = GardenSnakeParser(
1094                     debug=debug, form=form, incl_carry=incl_carry,
1095                     helper=helper)
1096                 _CACHED_PARSERS[debug, form, incl_carry, helper] = self.parser
1097         else:
1098             self.parser = GardenSnakeParser(debug=debug, form=form,
1099                                             incl_carry=incl_carry, helper=helper)
1100
1101     def compile(self, code, mode="exec", filename="<string>"):
1102         if filename in ["string", "<string>"]:
1103             raise ValueError("missing filename")
1104         self.parser.filename = filename
1105         self.parser.input_text = code
1106         try:
1107             tree = self.parser.parse(code)
1108         except SyntaxError2 as e:
1109             e.raise_syntax_error()
1110         print("snake")
1111         pprint(tree)
1112         return tree
1113         #misc.set_filename(filename, tree)
1114         return compile(tree, mode="exec", filename="<string>")
1115         # syntax.check(tree)
1116         gen = pycodegen.ModuleCodeGenerator(tree)
1117         code = gen.getCode()
1118         return code