src/openpower/decoder/pseudo/parser.py

   1 # Based on GardenSnake - a parser generator demonstration program
   2 # GardenSnake was released into the Public Domain by Andrew Dalke.
   3
   4 # Portions of this work are derived from Python's Grammar definition
   5 # and may be covered under the Python copyright and license
   6 #
   7 #          Andrew Dalke / Dalke Scientific Software, LLC
   8 #             30 August 2006 / Cape Town, South Africa
   9
  10 # Modifications for inclusion in PLY distribution
  11 from pprint import pprint
  12 from ply import lex, yacc
  13 import astor
  14 from copy import deepcopy
  15
  16 from openpower.decoder.power_decoder import create_pdecode
  17 from openpower.decoder.pseudo.lexer import (
  18     IndentLexer, raise_syntax_error, SyntaxError2)
  19 from openpower.decoder.orderedset import OrderedSet
  20 from openpower.decoder.power_enums import BFP_FLAG_NAMES
  21
  22 # I use the Python AST
  23 #from compiler import ast
  24 import ast
  25
  26 # Helper function
  27
  28 regs = ['RA', 'RS', 'RB', 'RC', 'RT']
  29 fregs = ['FRA', 'FRS', 'FRB', 'FRC', 'FRT', 'FRS']
  30 SPECIAL_HELPERS = {'concat', 'MEM', 'GPR', 'FPR', 'SPR'}
  31
  32
  33 # I implemented INDENT / DEDENT generation as a post-processing filter
  34
  35 # The original lex token stream contains WS and NEWLINE characters.
  36 # WS will only occur before any other tokens on a line.
  37
  38 # I have three filters.  One tags tokens by adding two attributes.
  39 # "must_indent" is True if the token must be indented from the
  40 # previous code.  The other is "at_line_start" which is True for WS
  41 # and the first non-WS/non-NEWLINE on a line.  It flags the check so
  42 # see if the new line has changed indication level.
  43
  44
  45 # No using Python's approach because Ply supports precedence
  46
  47 # comparison: expr (comp_op expr)*
  48 # arith_expr: term (('+'|'-') term)*
  49 # term: factor (('*'|'/'|'%'|'//') factor)*
  50 # factor: ('+'|'-'|'~') factor | power
  51 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
  52
  53 def make_le_compare(arg):
  54     (left, right) = arg
  55     return ast.Call(ast.Name("le", ast.Load()), (left, right), [])
  56
  57
  58 def make_ge_compare(arg):
  59     (left, right) = arg
  60     return ast.Call(ast.Name("ge", ast.Load()), (left, right), [])
  61
  62
  63 def make_lt_compare(arg):
  64     (left, right) = arg
  65     return ast.Call(ast.Name("lt", ast.Load()), (left, right), [])
  66
  67
  68 def make_gt_compare(arg):
  69     (left, right) = arg
  70     return ast.Call(ast.Name("gt", ast.Load()), (left, right), [])
  71
  72
  73 def make_eq_compare(arg):
  74     (left, right) = arg
  75     return ast.Call(ast.Name("eq", ast.Load()), (left, right), [])
  76
  77
  78 def make_ne_compare(arg):
  79     (left, right) = arg
  80     return ast.Call(ast.Name("ne", ast.Load()), (left, right), [])
  81
  82
  83 binary_ops = {
  84     "^": ast.BitXor(),
  85     "&": ast.BitAnd(),
  86     "|": ast.BitOr(),
  87     "+": ast.Add(),
  88     "-": ast.Sub(),
  89     "*": ast.Mult(),
  90     "/": ast.FloorDiv(),
  91     "%": ast.Mod(),
  92     "<=": make_le_compare,
  93     ">=": make_ge_compare,
  94     "<": make_lt_compare,
  95     ">": make_gt_compare,
  96     "=": make_eq_compare,
  97     "!=": make_ne_compare,
  98 }
  99 unary_ops = {
 100     "+": ast.UAdd(),
 101     "-": ast.USub(),
 102     "¬": ast.Invert(),
 103 }
 104
 105
 106 def check_concat(node):  # checks if the comparison is already a concat
 107     print("check concat", node)
 108     if not isinstance(node, ast.Call):
 109         return [node]
 110     node_func_id = getattr(node.func, "id", None)
 111     print("func", node_func_id)
 112     if node_func_id != 'concat':
 113         return [node]
 114     if node.keywords:  # a repeated list-constant, don't optimise
 115         return [node]
 116     return node.args
 117
 118
 119 # identify SelectableInt pattern [something] * N
 120 # must return concat(something, repeat=N)
 121 # here is a TEMPORARY hack to support minimal expressions
 122 # such as (XLEN-16), looking for ast.BinOp
 123 # have to keep an eye on this
 124 def identify_sint_mul_pattern(p):
 125     """here we are looking for patterns of this type:
 126         [item] * something
 127     these must specifically be returned as concat(item, repeat=something)
 128     """
 129     #print ("identify_sint_mul_pattern")
 130     # for pat in p:
 131     #    print("    ", astor.dump_tree(pat))
 132     if p[2] != '*':  # multiply
 133         return False
 134     if (not isinstance(p[3], ast.Constant) and  # rhs = Num
 135         not isinstance(p[3], ast.BinOp) and     # rhs = (XLEN-something)
 136             (not isinstance(p[3], ast.Name) and  # rhs = {a variable}
 137              not isinstance(p[3], ast.Attribute))):   # rhs = XLEN
 138         return False
 139     if not isinstance(p[1], ast.List):  # lhs is a list
 140         return False
 141     l = p[1].elts
 142     if len(l) != 1:  # lhs is a list of length 1
 143         return False
 144     return True  # yippee!
 145
 146
 147 ##########   Parser (tokens -> AST) ######
 148
 149 # also part of Ply
 150 #import yacc
 151
 152 # https://www.mathcs.emory.edu/~valerie/courses/fall10/155/resources/op_precedence.html
 153 # python operator precedence
 154 # Highest precedence at top, lowest at bottom.
 155 # Operators in the same box evaluate left to right.
 156 #
 157 # Operator Description
 158 # ()                                                     Parentheses (grouping)
 159 # f(args...)                                             Function call
 160 # x[index:index]                                         Slicing
 161 # x[index]                                               Subscription
 162 # x.attribute                                            Attribute reference
 163 # **                                                     Exponentiation
 164 # ~x                                                     Bitwise not
 165 # +x, -x                                                 Positive, negative
 166 # *, /, %                                                mul, div, remainder
 167 # +, -                                                   Addition, subtraction
 168 # <<, >>                                                 Bitwise shifts
 169 # &                                                      Bitwise AND
 170 # ^                                                      Bitwise XOR
 171 # |                                                      Bitwise OR
 172 # in, not in, is, is not, <, <=,  >,  >=, <>, !=, ==     comp, membership, ident
 173 # not x                                                  Boolean NOT
 174 # and                                                    Boolean AND
 175 # or                                                     Boolean OR
 176 # lambda                                                 Lambda expression
 177
 178
 179 class PowerParser:
 180
 181     precedence = (
 182         ("left", "EQ", "NE", "GT", "LT", "LE", "GE", "LTU", "GTU"),
 183         ("left", "BITOR"),
 184         ("left", "BITXOR"),
 185         ("left", "BITAND"),
 186         ("left", "PLUS", "MINUS"),
 187         ("left", "MULT", "DIV", "MOD"),
 188         ("left", "INVERT"),
 189     )
 190
 191     def reset(self):
 192         self.gprs = {}
 193         if self.form is not None:
 194             form = self.sd.sigforms[self.form]
 195             print(form)
 196             formkeys = form._asdict().keys()
 197         else:
 198             formkeys = []
 199         self.declared_vars = set()
 200         self.fnparm_vars = set()
 201         for rname in regs + fregs:
 202             self.gprs[rname] = None
 203             self.declared_vars.add(rname)
 204         self.available_op_fields = set()
 205         for k in formkeys:
 206             if k not in self.gprs:
 207                 if k == 'SPR':  # sigh, lower-case to not conflict
 208                     k = k.lower()
 209                 self.available_op_fields.add(k)
 210         self.op_fields = OrderedSet()
 211         self.read_regs = OrderedSet()
 212         self.uninit_regs = OrderedSet()
 213         self.write_regs = OrderedSet()
 214         self.special_regs = OrderedSet()  # see p_atom_name
 215
 216     def __init__(self, form, include_carry_in_write=False, helper=False):
 217         self.include_ca_in_write = include_carry_in_write
 218         self.helper = helper
 219         self.form = form
 220         self.filename = None
 221         self.input_text = None
 222         self.reset()
 223
 224     # The grammar comments come from Python's Grammar/Grammar file
 225
 226     # NB: compound_stmt in single_input is followed by extra NEWLINE!
 227     # file_input: (NEWLINE | stmt)* ENDMARKER
 228
 229     def p_file_input_end(self, p):
 230         """file_input_end : file_input ENDMARKER"""
 231         print("end", p[1])
 232         p[0] = p[1]
 233
 234     def p_file_input(self, p):
 235         """file_input : file_input NEWLINE
 236                       | file_input stmt
 237                       | NEWLINE
 238                       | stmt"""
 239         if isinstance(p[len(p)-1], str):
 240             if len(p) == 3:
 241                 p[0] = p[1]
 242             else:
 243                 p[0] = []  # p == 2 --> only a blank line
 244         else:
 245             if len(p) == 3:
 246                 p[0] = p[1] + p[2]
 247             else:
 248                 p[0] = p[1]
 249
 250     # funcdef: [decorators] 'def' NAME parameters ':' suite
 251     # ignoring decorators
 252
 253     def p_funcdef(self, p):
 254         "funcdef : DEF NAME parameters COLON suite"
 255         p[0] = ast.FunctionDef(p[2], p[3], p[5], (), lineno=p.lineno(2))
 256         # reset function parameters after suite is identified
 257         self.fnparm_vars = set()
 258
 259     # parameters: '(' [varargslist] ')'
 260     def p_parameters(self, p):
 261         """parameters : LPAR RPAR
 262                       | LPAR varargslist RPAR"""
 263         args = []
 264         if self.helper:
 265             args.append("self")
 266         if len(p) != 3:
 267             args += p[2]
 268         p[0] = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[])
 269         # during the time between parameters identified and suite is not
 270         # there is a window of opportunity to declare the function parameters
 271         # in-scope, for use to not overwrite them with auto-assign
 272         self.fnparm_vars = set()
 273         for arg in args:
 274             print("adding fn parm", arg)
 275             self.fnparm_vars.add(arg)
 276
 277     # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] |
 278     # '**' NAME) |
 279     # highly simplified
 280
 281     def p_varargslist(self, p):
 282         """varargslist : varargslist COMMA NAME
 283                        | NAME"""
 284         if len(p) == 4:
 285             print(p[1], p[3])
 286             p[0] = p[1] + [p[3]]
 287         else:
 288             p[0] = [p[1]]
 289
 290     # stmt: simple_stmt | compound_stmt
 291     def p_stmt_simple(self, p):
 292         """stmt : simple_stmt"""
 293         # simple_stmt is a list
 294         p[0] = p[1]
 295
 296     def p_stmt_compound(self, p):
 297         """stmt : compound_stmt"""
 298         p[0] = [p[1]]
 299
 300     # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 301     def p_simple_stmt(self, p):
 302         """simple_stmt : small_stmts NEWLINE
 303                        | small_stmts SEMICOLON NEWLINE"""
 304         p[0] = p[1]
 305
 306     def p_small_stmts(self, p):
 307         """small_stmts : small_stmts SEMICOLON small_stmt
 308                        | small_stmt"""
 309         if len(p) == 4:
 310             p[0] = p[1] + [p[3]]
 311         elif isinstance(p[1], list):
 312             p[0] = p[1]
 313         else:
 314             p[0] = [p[1]]
 315
 316     # small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
 317     #    import_stmt | global_stmt | exec_stmt | assert_stmt
 318     def p_small_stmt(self, p):
 319         """small_stmt : flow_stmt
 320                       | break_stmt
 321                       | expr_stmt"""
 322         if isinstance(p[1], ast.Call):
 323             p[0] = ast.Expr(p[1])
 324         elif isinstance(p[1], ast.Name) and p[1].id not in SPECIAL_HELPERS:
 325             fname = p[1].id
 326             name = ast.Name("self", ast.Load())
 327             name = ast.Attribute(name, fname, ast.Load())
 328             p[0] = ast.Call(name, [], [])
 329         else:
 330             p[0] = p[1]
 331
 332     # expr_stmt: testlist (augassign (yield_expr|testlist) |
 333     #                      ('=' (yield_expr|testlist))*)
 334     # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
 335     #             '<<=' | '>>=' | '**=' | '//=')
 336     def p_expr_stmt(self, p):
 337         """expr_stmt : testlist ASSIGNEA testlist
 338                      | testlist ASSIGN testlist
 339                      | testlist """
 340         print("expr_stmt", p)
 341         if len(p) == 2:
 342             # a list of expressions
 343             #p[0] = ast.Discard(p[1])
 344             p[0] = p[1]
 345         else:
 346             iea_mode = p[2] == '<-iea'
 347             name = None
 348             autoassign = False
 349             if isinstance(p[1], ast.Name):
 350                 name = p[1].id
 351             elif isinstance(p[1], ast.Subscript):
 352                 print("assign subscript", p[1].value,
 353                       self.declared_vars,
 354                       self.fnparm_vars,
 355                       self.special_regs)
 356                 print(astor.dump_tree(p[1]))
 357                 if isinstance(p[1].value, ast.Name):
 358                     name = p[1].value.id
 359                     print("assign subscript value to name", name)
 360                     if name in self.gprs:
 361                         # add to list of uninitialised
 362                         self.uninit_regs.add(name)
 363                     # work out if this is an ininitialised variable
 364                     # that should be auto-assigned simply by being "sliced"
 365                     autoassign = (name not in self.declared_vars and
 366                                   name not in self.fnparm_vars and
 367                                   name not in self.special_regs)
 368             elif isinstance(p[1], (ast.Attribute, ast.Tuple)):
 369                 pass
 370             elif isinstance(p[1], ast.Call) and p[1].func.id in \
 371                     ['GPR', 'FPR', 'SPR']:
 372                 print(astor.dump_tree(p[1]))
 373                 # replace GPR(x) with GPR[x]
 374                 idx = p[1].args[0].id
 375                 if idx in regs + fregs:
 376                     ridx = ast.Name("_%s" % idx, ast.Load())
 377                 else:
 378                     ridx = ast.Name(idx, ast.Load())
 379                 p[1] = ast.Subscript(p[1].func, ridx, ast.Load())
 380                 if idx in self.gprs:
 381                     self.read_regs.add(idx)  # add to list of regs to read
 382             elif isinstance(p[1], ast.Call) and p[1].func.id == 'MEM':
 383                 print("mem assign")
 384                 print(astor.dump_tree(p[1]))
 385                 p[1].func.id = "memassign"  # change function name to set
 386                 p[1].args.append(p[3])
 387                 p[0] = p[1]
 388                 print("mem rewrite")
 389                 print(astor.dump_tree(p[0]))
 390                 return
 391             else:
 392                 print(astor.dump_tree(p[1]))
 393                 raise_syntax_error("help, help", self.filename,
 394                                    p.slice[2].lineno, p.slice[2].lexpos,
 395                                    self.input_text)
 396             print("expr assign", name, p[1], "to", p[3])
 397             if isinstance(p[3], ast.Name):
 398                 toname = p[3].id
 399                 if toname in self.gprs:
 400                     self.read_regs.add(toname)
 401             if name and name in self.gprs:
 402                 self.write_regs.add(name)  # add to list of regs to write
 403
 404             # copy rhs -- see openpower.decoder.helpers.copy_assign_rhs()'s
 405             # documentation for why we need this
 406             copy_fn = ast.Name("copy_assign_rhs", ast.Load())
 407             rhs = ast.Call(copy_fn, (p[3],), [])
 408             p[0] = self.Assign(autoassign, name, p[1], rhs, iea_mode,
 409                                p.slice[2])
 410             if name:
 411                 self.declared_vars.add(name)
 412
 413     def p_flow_stmt(self, p):
 414         "flow_stmt : return_stmt"
 415         p[0] = p[1]
 416
 417     # return_stmt: 'return' [testlist]
 418     def p_return_stmt(self, p):
 419         "return_stmt : RETURN testlist"
 420         p[0] = ast.Return(p[2])
 421
 422     def p_compound_stmt(self, p):
 423         """compound_stmt : if_stmt
 424                          | while_stmt
 425                          | switch_stmt
 426                          | for_stmt
 427                          | funcdef
 428         """
 429         p[0] = p[1]
 430
 431     def p_break_stmt(self, p):
 432         """break_stmt : BREAK
 433         """
 434         p[0] = ast.Break()
 435
 436     def p_for_stmt(self, p):
 437         """for_stmt : FOR atom EQ comparison TO comparison COLON suite
 438                     | DO atom EQ comparison TO comparison COLON suite
 439         """
 440         start = p[4]
 441         end = p[6]
 442         it = ast.Call(ast.Name("RANGE", ast.Load()), (start, end), [])
 443         p[0] = ast.For(p[2], it, p[8], [])
 444
 445     def p_while_stmt(self, p):
 446         """while_stmt : DO WHILE test COLON suite ELSE COLON suite
 447                       | DO WHILE test COLON suite
 448         """
 449         if len(p) == 6:
 450             p[0] = ast.While(p[3], p[5], [])
 451         else:
 452             p[0] = ast.While(p[3], p[5], p[8])
 453
 454     def p_switch_smt(self, p):
 455         """switch_stmt : SWITCH LPAR atom RPAR COLON NEWLINE INDENT switches DEDENT
 456         """
 457         switchon = p[3]
 458         print("switch stmt")
 459         print(astor.dump_tree(p[1]))
 460
 461         cases = []
 462         current_cases = []  # for deferral
 463         for (case, suite) in p[8]:
 464             print("for", case, suite)
 465             if suite is None:
 466                 for c in case:
 467                     current_cases.append(ast.Num(c))
 468                 continue
 469             if case == 'default':  # last
 470                 break
 471             for c in case:
 472                 current_cases.append(ast.Num(c))
 473             print("cases", current_cases)
 474             compare = ast.Compare(switchon, [ast.In()],
 475                                   [ast.List(current_cases, ast.Load())])
 476             current_cases = []
 477             cases.append((compare, suite))
 478
 479         print("ended", case, current_cases)
 480         if case == 'default':
 481             if current_cases:
 482                 compare = ast.Compare(switchon, [ast.In()],
 483                                       [ast.List(current_cases, ast.Load())])
 484                 cases.append((compare, suite))
 485             cases.append((None, suite))
 486
 487         cases.reverse()
 488         res = []
 489         for compare, suite in cases:
 490             print("after rev", compare, suite)
 491             if compare is None:
 492                 assert len(res) == 0, "last case should be default"
 493                 res = suite
 494             else:
 495                 if not isinstance(res, list):
 496                     res = [res]
 497                 res = ast.If(compare, suite, res)
 498         p[0] = res
 499
 500     def p_switches(self, p):
 501         """switches : switch_list switch_default
 502                     | switch_default
 503         """
 504         if len(p) == 3:
 505             p[0] = p[1] + [p[2]]
 506         else:
 507             p[0] = [p[1]]
 508
 509     def p_switch_list(self, p):
 510         """switch_list : switch_case switch_list
 511                        | switch_case
 512         """
 513         if len(p) == 3:
 514             p[0] = [p[1]] + p[2]
 515         else:
 516             p[0] = [p[1]]
 517
 518     def p_switch_case(self, p):
 519         """switch_case : CASE LPAR atomlist RPAR COLON suite
 520         """
 521         # XXX bad hack
 522         if isinstance(p[6][0], ast.Name) and p[6][0].id == 'fallthrough':
 523             p[6] = None
 524         p[0] = (p[3], p[6])
 525
 526     def p_switch_default(self, p):
 527         """switch_default : DEFAULT COLON suite
 528         """
 529         p[0] = ('default', p[3])
 530
 531     def p_atomlist(self, p):
 532         """atomlist : atom COMMA atomlist
 533                     | atom
 534         """
 535         assert isinstance(p[1], ast.Constant), "case must be numbers"
 536         if len(p) == 4:
 537             p[0] = [p[1].value] + p[3]
 538         else:
 539             p[0] = [p[1].value]
 540
 541     def p_if_stmt(self, p):
 542         """if_stmt : IF test COLON suite ELSE COLON if_stmt
 543                    | IF test COLON suite ELSE COLON suite
 544                    | IF test COLON suite
 545         """
 546         if len(p) == 8 and isinstance(p[7], ast.If):
 547             p[0] = ast.If(p[2], p[4], [p[7]])
 548         elif len(p) == 5:
 549             p[0] = ast.If(p[2], p[4], [])
 550         else:
 551             p[0] = ast.If(p[2], p[4], p[7])
 552
 553     def p_suite(self, p):
 554         """suite : simple_stmt
 555                  | NEWLINE INDENT stmts DEDENT"""
 556         if len(p) == 2:
 557             p[0] = p[1]
 558         else:
 559             p[0] = p[3]
 560
 561     def p_stmts(self, p):
 562         """stmts : stmts stmt
 563                  | stmt"""
 564         if len(p) == 3:
 565             p[0] = p[1] + p[2]
 566         else:
 567             p[0] = p[1]
 568
 569     def p_comparison(self, p):
 570         """comparison : comparison PLUS comparison
 571                       | comparison MINUS comparison
 572                       | comparison MULT comparison
 573                       | comparison DIV comparison
 574                       | comparison MOD comparison
 575                       | comparison EQ comparison
 576                       | comparison NE comparison
 577                       | comparison LE comparison
 578                       | comparison GE comparison
 579                       | comparison LTU comparison
 580                       | comparison GTU comparison
 581                       | comparison LT comparison
 582                       | comparison GT comparison
 583                       | comparison BITOR comparison
 584                       | comparison BITXOR comparison
 585                       | comparison BITAND comparison
 586                       | PLUS comparison
 587                       | MINUS comparison
 588                       | INVERT comparison
 589                       | comparison APPEND comparison
 590                       | power"""
 591         if len(p) == 4:
 592             print(list(p))
 593             if p[2] == '<u':
 594                 p[0] = ast.Call(ast.Name("ltu", ast.Load()), (p[1], p[3]), [])
 595             elif p[2] == '>u':
 596                 p[0] = ast.Call(ast.Name("gtu", ast.Load()), (p[1], p[3]), [])
 597             elif p[2] == '||':
 598                 l = check_concat(p[1]) + check_concat(p[3])
 599                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, [])
 600             elif p[2] in ['/', '%']:
 601                 # bad hack: if % or / used anywhere other than div/mod ops,
 602                 # do % or /.  however if the argument names are "dividend"
 603                 # we must call the special trunc_divs and trunc_rems functions
 604                 l, r = p[1], p[3]
 605                 # actual call will be "dividend / divisor" - just check
 606                 # LHS name
 607                 # XXX DISABLE BAD HACK (False)
 608                 if False and isinstance(l, ast.Name) and l.id == 'dividend':
 609                     if p[2] == '/':
 610                         fn = 'trunc_divs'
 611                     else:
 612                         fn = 'trunc_rems'
 613                     # return "function trunc_xxx(l, r)"
 614                     p[0] = ast.Call(ast.Name(fn, ast.Load()), (l, r), [])
 615                 else:
 616                     # return "l {binop} r"
 617                     p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 618             elif p[2] in ['<', '>', '=', '<=', '>=', '!=']:
 619                 p[0] = binary_ops[p[2]]((p[1], p[3]))
 620             elif identify_sint_mul_pattern(p):
 621                 keywords = [ast.keyword(arg='repeat', value=p[3])]
 622                 l = p[1].elts
 623                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 624             else:
 625                 p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 626                 # HORRENDOUS hack, add brackets around the bin-op by
 627                 # creating a function call with a *blank* function name!
 628                 # XXX argh doesn't work because of analysis of
 629                 # identify_sint_pattern
 630                 #p[0] = ast.Call(ast.Name("", ast.Load()), [p[0]], [])
 631         elif len(p) == 3:
 632             if isinstance(p[2], str) and p[2] == '-':
 633                 p[0] = ast.UnaryOp(unary_ops[p[2]], p[1])
 634             else:
 635                 p[0] = ast.UnaryOp(unary_ops[p[1]], p[2])
 636         else:
 637             p[0] = p[1]
 638
 639     # power: atom trailer* ['**' factor]
 640     # trailers enables function calls (and subscripts).
 641     # so this is 'trailerlist'
 642     def p_power(self, p):
 643         """power : atom
 644                  | atom trailerlist"""
 645         if len(p) == 2:
 646             print("power dump atom notrailer")
 647             print(astor.dump_tree(p[1]))
 648             p[0] = p[1]
 649         else:
 650             print("power dump atom")
 651             print(astor.dump_tree(p[1]))
 652             print("power dump trailerlist")
 653             print(astor.dump_tree(p[2]))
 654             p[0] = self.apply_trailer(p[1], p[2], self.read_regs)
 655             if isinstance(p[1], ast.Name):
 656                 name = p[1].id
 657                 if name in regs + fregs:
 658                     self.read_regs.add(name)
 659
 660     def p_atom_name(self, p):
 661         """atom : NAME"""
 662         name = p[1]
 663         if name in self.available_op_fields:
 664             self.op_fields.add(name)
 665         if name in ['overflow', 'CR0']:
 666             self.write_regs.add(name)
 667         if self.include_ca_in_write:
 668             if name in ['CA', 'CA32']:
 669                 self.write_regs.add(name)
 670         if name in ['CR', 'LR', 'CTR', 'TAR', 'FPSCR', 'MSR',
 671                     'SVSTATE', 'SVREMAP',
 672                     'SVSHAPE0', 'SVSHAPE1', 'SVSHAPE2', 'SVSHAPE3']:
 673             self.special_regs.add(name)
 674             self.write_regs.add(name)  # and add to list to write
 675         if name in ('XLEN', 'FPSCR') or name in BFP_FLAG_NAMES:
 676             attr = ast.Name("self", ast.Load())
 677             p[0] = ast.Attribute(attr, name, ast.Load(), lineno=p.lineno(1))
 678         else:
 679             p[0] = ast.Name(id=name, ctx=ast.Load(), lineno=p.lineno(1))
 680
 681     def p_atom_number(self, p):
 682         """atom : BINARY
 683                 | NUMBER
 684                 | HEX
 685                 | STRING"""
 686         p[0] = ast.Constant(p[1])
 687
 688     # '[' [listmaker] ']' |
 689
 690     def p_atom_listmaker(self, p):
 691         """atom : LBRACK listmaker RBRACK"""
 692         p[0] = p[2]
 693
 694     def p_listmaker(self, p):
 695         """listmaker : test COMMA listmaker
 696                      | test
 697         """
 698         if len(p) == 2:
 699             p[0] = ast.List([p[1]], ast.Load())
 700         else:
 701             p[0] = ast.List([p[1]] + p[3].nodes, ast.Load())
 702
 703     def p_atom_tuple(self, p):
 704         """atom : LPAR testlist RPAR"""
 705         print("tuple", p[2])
 706         print("astor dump")
 707         print(astor.dump_tree(p[2]))
 708
 709         if isinstance(p[2], ast.Name):
 710             name = p[2].id
 711             print("tuple name", name)
 712             if name in self.gprs:
 713                 self.read_regs.add(name)  # add to list of regs to read
 714                 #p[0] = ast.Subscript(ast.Name("GPR", ast.Load()), ast.Str(p[2].id))
 715                 # return
 716             p[0] = p[2]
 717         elif isinstance(p[2], ast.BinOp):
 718             if isinstance(p[2].left, ast.Name) and \
 719                isinstance(p[2].right, ast.Constant) and \
 720                     p[2].right.value == 0 and \
 721                     p[2].left.id in self.gprs:
 722                 rid = p[2].left.id
 723                 self.read_regs.add(rid)  # add to list of regs to read
 724                 # create special call to GPR.getz or FPR.getz
 725                 if rid in fregs:
 726                     gprz = ast.Name("FPR", ast.Load())
 727                 else:
 728                     gprz = ast.Name("GPR", ast.Load())
 729                 # get testzero function
 730                 gprz = ast.Attribute(gprz, "getz", ast.Load())
 731                 # *sigh* see class GPR.  we need index itself not reg value
 732                 ridx = ast.Name("_%s" % rid, ast.Load())
 733                 p[0] = ast.Call(gprz, [ridx], [])
 734                 print("tree", astor.dump_tree(p[0]))
 735             else:
 736                 p[0] = p[2]
 737         else:
 738             p[0] = p[2]
 739
 740     def p_trailerlist(self, p):
 741         """trailerlist : trailer trailerlist
 742                        | trailer
 743         """
 744         if len(p) == 2:
 745             p[0] = p[1]
 746         else:
 747             p[0] = ("TLIST", p[1], p[2])
 748
 749     # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 750     def p_trailer(self, p):
 751         """trailer : trailer_arglist
 752                    | trailer_subscript
 753                    | trailer_attr
 754         """
 755         p[0] = p[1]
 756
 757     def p_trailer_arglist(self, p):
 758         """trailer_arglist : LPAR arglist RPAR
 759                            | LPAR RPAR
 760         """
 761         args = [] if len(p) == 3 else p[2]
 762         p[0] = ("CALL", args, p.slice[1])
 763
 764     def p_trailer_subscript(self, p):
 765         "trailer_subscript : LBRACK subscript RBRACK"
 766         p[0] = ("SUBS", p[2])
 767
 768     def p_trailer_attr(self, p):
 769         "trailer_attr : PERIOD NAME"
 770         p[0] = ("ATTR", p[2])
 771
 772     # subscript: '.' '.' '.' | test | [test] ':' [test]
 773
 774     def p_subscript(self, p):
 775         """subscript : test COLON test
 776                      | test
 777         """
 778         if len(p) == 4:
 779             # add one to end
 780             if isinstance(p[3], ast.Constant):
 781                 end = ast.Constant(p[3].value+1)
 782             else:
 783                 end = ast.BinOp(p[3], ast.Add(), ast.Constant(1))
 784             p[0] = [p[1], end]
 785         else:
 786             p[0] = [p[1]]
 787
 788     # testlist: test (',' test)* [',']
 789     # Contains shift/reduce error
 790
 791     def p_testlist(self, p):
 792         """testlist : testlist_multi COMMA
 793                     | testlist_multi """
 794         if len(p) == 2:
 795             p[0] = p[1]
 796         else:
 797             # May need to promote singleton to tuple
 798             if isinstance(p[1], list):
 799                 p[0] = p[1]
 800             else:
 801                 p[0] = [p[1]]
 802         # Convert into a tuple?
 803         if isinstance(p[0], list):
 804             p[0] = ast.Tuple(p[0])
 805
 806     def p_testlist_multi(self, p):
 807         """testlist_multi : testlist_multi COMMA test
 808                           | test"""
 809         if len(p) == 2:
 810             # singleton
 811             p[0] = p[1]
 812         else:
 813             if isinstance(p[1], list):
 814                 p[0] = p[1] + [p[3]]
 815             else:
 816                 # singleton -> tuple
 817                 p[0] = [p[1], p[3]]
 818
 819     # test: or_test ['if' or_test 'else' test] | lambdef
 820     #  as I don't support 'and', 'or', and 'not' this works down to 'comparison'
 821
 822     def p_test(self, p):
 823         "test : comparison"
 824         p[0] = p[1]
 825
 826     # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
 827     # | '**' test)
 828     # XXX INCOMPLETE: this doesn't allow the trailing comma
 829
 830     def p_arglist(self, p):
 831         """arglist : arglist COMMA argument
 832                    | argument"""
 833         if len(p) == 4:
 834             p[0] = p[1] + [p[3]]
 835         else:
 836             p[0] = [p[1]]
 837
 838     # argument: test [gen_for] | test '=' test  # Really [keyword '='] test
 839     def p_argument(self, p):
 840         "argument : test"
 841         p[0] = p[1]
 842
 843     def p_error(self, p):
 844         # print "Error!", repr(p)
 845         raise_syntax_error(str(p), self.filename, p.lineno, p.lexpos,
 846                            self.input_text)
 847
 848     def Assign(self, autoassign, assignname, left, right, iea_mode, eq_tok):
 849         names = []
 850         print("Assign", autoassign, assignname, left, right)
 851         if isinstance(left, ast.Name):
 852             # Single assignment on left
 853             # XXX when doing IntClass, which will have an "eq" function,
 854             # this is how to access it
 855             #   eq = ast.Attribute(left, "eq")   # get eq fn
 856             #   return ast.Call(eq, [right], []) # now call left.eq(right)
 857             return ast.Assign([ast.Name(left.id, ast.Store())], right)
 858         elif isinstance(left, ast.Tuple):
 859             # List of things - make sure they are Name nodes
 860             names = []
 861             for child in left.elts:
 862                 if not isinstance(child, ast.Name):
 863                     raise_syntax_error(
 864                         "that assignment not supported", self.filename,
 865                         eq_tok.lineno, eq_tok.lexpos, self.input_text)
 866                 names.append(child.id)
 867             ass_list = [ast.Name(name, ast.Store()) for name in names]
 868             return ast.Assign([ast.Tuple(ass_list)], right)
 869         elif isinstance(left, ast.Attribute):
 870             return ast.Assign([
 871                 ast.Attribute(left.value, left.attr, ast.Store())], right)
 872         elif isinstance(left, ast.Subscript):
 873             ls = left.slice
 874             # XXX changing meaning of "undefined" to a function
 875             # if (isinstance(ls, ast.Slice) and isinstance(right, ast.Name) and
 876             #        right.id == 'undefined'):
 877             #    # undefined needs to be copied the exact same slice
 878             #    right = ast.Subscript(right, ls, ast.Load())
 879             #    return ast.Assign([left], right)
 880             res = ast.Assign([left], right)
 881             if autoassign and isinstance(ls, ast.Slice):
 882                 # hack to create a variable pre-declared based on a slice.
 883                 # dividend[0:32] = (RA)[0:32] will create
 884                 #       dividend = [0] * 32
 885                 #       dividend[0:32] = (RA)[0:32]
 886                 # the declaration makes the slice-assignment "work"
 887                 lower, upper, step = ls.lower, ls.upper, ls.step
 888                 print("lower, upper, step", repr(lower), repr(upper), step)
 889                 # XXX relax constraint that indices on auto-assign have
 890                 # to be constants x[0:32]
 891                 # if not isinstance(lower, ast.Constant) or \
 892                 #   not isinstance(upper, ast.Constant):
 893                 #    return res
 894                 qty = ast.BinOp(upper, binary_ops['-'], lower)
 895                 keywords = [ast.keyword(arg='repeat', value=qty)]
 896                 l = [ast.Num(0)]
 897                 right = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 898                 declare = ast.Assign(
 899                     [ast.Name(assignname, ast.Store())], right)
 900                 return [declare, res]
 901             return res
 902             # XXX HMMM probably not needed...
 903             ls = left.slice
 904             if isinstance(ls, ast.Slice):
 905                 lower, upper, step = ls.lower, ls.upper, ls.step
 906                 print("slice assign", lower, upper, step)
 907                 if step is None:
 908                     ls = (lower, upper, None)
 909                 else:
 910                     ls = (lower, upper, step)
 911                 ls = ast.Tuple(ls)
 912             return ast.Call(ast.Name("selectassign", ast.Load()),
 913                             [left.value, ls, right], [])
 914         else:
 915             print("Assign fail")
 916             raise_syntax_error("Can't do that yet", self.filename,
 917                                eq_tok.lineno, eq_tok.lexpos, self.input_text)
 918
 919     def try_extract_name(self, atom):
 920         if isinstance(atom, ast.Attribute):
 921             if isinstance(atom.value, ast.Name) and atom.value.id == "self":
 922                 return atom.attr
 923             return None
 924         if isinstance(atom, ast.Name):
 925             return atom.id
 926         return None
 927
 928     def try_extract_uppercase_name(self, atom):
 929         name = self.try_extract_name(atom)
 930         if name is None:
 931             return None
 932         # we want to filter out names like _123 that have no letters with
 933         # casing at all, hence the lower() check
 934         if name.upper() != name or name.lower() == name:
 935             return None
 936         return name
 937
 938     def try_get_attr(self, atom, trailer):
 939         if trailer[0] == "ATTR":
 940             return trailer[1]
 941         if trailer[0] != "SUBS":
 942             return None
 943         base_name = self.try_extract_uppercase_name(atom)
 944         if base_name is None or base_name in SPECIAL_HELPERS:
 945             return None
 946         if len(trailer[1]) != 1:
 947             return None
 948         return self.try_extract_uppercase_name(trailer[1][0])
 949
 950     def apply_trailer(self, atom, trailer, read_regs):
 951         if trailer[0] == "TLIST":
 952             # assume depth of one
 953             atom = self.apply_trailer(atom, trailer[1], read_regs)
 954             trailer = trailer[2]
 955         if trailer[0] == "CALL":
 956             #p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
 957             for arg in trailer[1]:
 958                 if isinstance(arg, ast.Name):
 959                     name = arg.id
 960                     if name in regs + fregs:
 961                         read_regs.add(name)
 962             if atom.id == "SetFX":
 963                 if len(trailer[1]) != 1 or \
 964                         not isinstance(trailer[1][0], ast.Attribute):
 965                     raise_syntax_error(
 966                         "SetFX call must have only one argument that is an "
 967                         "attribute access -- like `SetFX(FPSCR.VXSNAN)`",
 968                         self.filename, trailer[2].lineno, trailer[2].lexpos,
 969                         self.input_text)
 970                 arg = trailer[1][0]
 971                 args = [arg.value, ast.Str(arg.attr)]
 972                 name = ast.Name("self", ast.Load())
 973                 atom = ast.Attribute(name, atom, ast.Load())
 974                 return ast.Call(atom, args, [])
 975             # special-case, these functions must NOT be made "self.xxxx"
 976             if atom.id not in SPECIAL_HELPERS:
 977                 name = ast.Name("self", ast.Load())
 978                 atom = ast.Attribute(name, atom, ast.Load())
 979             return ast.Call(atom, trailer[1], [])
 980             # if p[1].id == 'print':
 981             #    p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
 982             # else:
 983             #    p[0] = ast.CallFunc(p[1], p[2][1], None, None)
 984         else:
 985             attr = self.try_get_attr(atom, trailer)
 986             if attr is not None:
 987                 return ast.Attribute(atom, attr, ast.Load())
 988             assert trailer[0] == "SUBS"
 989             print("subscript atom", trailer[1])
 990             #raise AssertionError("not implemented %s" % p[2][0])
 991             subs = trailer[1]
 992             if len(subs) == 1:
 993                 idx = subs[0]
 994                 if isinstance(idx, ast.Name) and idx.id in regs + fregs:
 995                     read_regs.add(idx.id)
 996                 if isinstance(idx, ast.Name) and idx.id in regs:
 997                     print("single atom subscript, underscored", idx.id)
 998                     idx = ast.Name("_%s" % idx.id, ast.Load())
 999             else:
1000                 idx = ast.Slice(subs[0], subs[1], None)
1001             # if isinstance(atom, ast.Name) and atom.id == 'CR':
1002                 # atom.id = 'CR' # bad hack
1003                 #print ("apply_trailer Subscript", atom.id, idx)
1004             return ast.Subscript(atom, idx, ast.Load())
1005
1006
1007 _CACHE_DECODER = True
1008 _CACHED_DECODER = None
1009
1010
1011 def _create_cached_decoder():
1012     global _CACHED_DECODER
1013     if _CACHE_DECODER:
1014         if _CACHED_DECODER is None:
1015             _CACHED_DECODER = create_pdecode()
1016         return _CACHED_DECODER
1017     return create_pdecode()
1018
1019
1020 class GardenSnakeParser(PowerParser):
1021     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1022         if form is not None:
1023             self.sd = _create_cached_decoder()
1024         PowerParser.__init__(self, form, incl_carry, helper=helper)
1025         self.debug = debug
1026         self.lexer = IndentLexer(debug=0)
1027         self.tokens = self.lexer.tokens
1028         self.parser = yacc.yacc(module=self, start="file_input_end",
1029                                 debug=debug, write_tables=False)
1030
1031     def parse(self, code):
1032         self.reset()
1033         self.lexer.filename = self.filename
1034         result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
1035         if self.helper:
1036             result = [ast.ClassDef("ISACallerFnHelper", [
1037                                    "ISACallerHelper"], [], result, decorator_list=[])]
1038         return ast.Module(result)
1039
1040
1041 ###### Code generation ######
1042
1043 #from compiler import misc, syntax, pycodegen
1044
1045 _CACHED_PARSERS = {}
1046 _CACHE_PARSERS = True
1047
1048
1049 class GardenSnakeCompiler(object):
1050     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1051         if _CACHE_PARSERS:
1052             try:
1053                 self.parser = _CACHED_PARSERS[debug, form, incl_carry, helper]
1054             except KeyError:
1055                 self.parser = GardenSnakeParser(
1056                     debug=debug, form=form, incl_carry=incl_carry,
1057                     helper=helper)
1058                 _CACHED_PARSERS[debug, form, incl_carry, helper] = self.parser
1059         else:
1060             self.parser = GardenSnakeParser(debug=debug, form=form,
1061                                             incl_carry=incl_carry, helper=helper)
1062
1063     def compile(self, code, mode="exec", filename="<string>"):
1064         if filename in ["string", "<string>"]:
1065             raise ValueError("missing filename")
1066         self.parser.filename = filename
1067         self.parser.input_text = code
1068         try:
1069             tree = self.parser.parse(code)
1070         except SyntaxError2 as e:
1071             e.raise_syntax_error()
1072         print("snake")
1073         pprint(tree)
1074         return tree
1075         #misc.set_filename(filename, tree)
1076         return compile(tree, mode="exec", filename="<string>")
1077         # syntax.check(tree)
1078         gen = pycodegen.ModuleCodeGenerator(tree)
1079         code = gen.getCode()
1080         return code