src/openpower/decoder/pseudo/parser.py

   1 # Based on GardenSnake - a parser generator demonstration program
   2 # GardenSnake was released into the Public Domain by Andrew Dalke.
   3
   4 # Portions of this work are derived from Python's Grammar definition
   5 # and may be covered under the Python copyright and license
   6 #
   7 #          Andrew Dalke / Dalke Scientific Software, LLC
   8 #             30 August 2006 / Cape Town, South Africa
   9
  10 # Modifications for inclusion in PLY distribution
  11 from pprint import pprint
  12 from ply import lex, yacc
  13 import astor
  14 from copy import deepcopy
  15
  16 from openpower.decoder.power_decoder import create_pdecode
  17 from openpower.decoder.pseudo.lexer import (
  18     IndentLexer, raise_syntax_error, SyntaxError2)
  19 from openpower.decoder.orderedset import OrderedSet
  20 from openpower.decoder.power_enums import BFP_FLAG_NAMES
  21
  22 # I use the Python AST
  23 #from compiler import ast
  24 import ast
  25
  26 # Helper function
  27
  28 regs = ['RA', 'RS', 'RB', 'RC', 'RT']
  29 fregs = ['FRA', 'FRS', 'FRB', 'FRC', 'FRT', 'FRS']
  30 SPECIAL_HELPERS = {'concat', 'MEM', 'GPR', 'FPR', 'SPR', 'pow'}
  31 SUBS_TO_ATTR_EXCEPTIONS = SPECIAL_HELPERS | {'CR'}
  32
  33
  34 # I implemented INDENT / DEDENT generation as a post-processing filter
  35
  36 # The original lex token stream contains WS and NEWLINE characters.
  37 # WS will only occur before any other tokens on a line.
  38
  39 # I have three filters.  One tags tokens by adding two attributes.
  40 # "must_indent" is True if the token must be indented from the
  41 # previous code.  The other is "at_line_start" which is True for WS
  42 # and the first non-WS/non-NEWLINE on a line.  It flags the check so
  43 # see if the new line has changed indication level.
  44
  45
  46 # No using Python's approach because Ply supports precedence
  47
  48 # comparison: expr (comp_op expr)*
  49 # arith_expr: term (('+'|'-') term)*
  50 # term: factor (('*'|'/'|'%'|'//') factor)*
  51 # factor: ('+'|'-'|'~') factor | power
  52 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
  53
  54 def make_le_compare(arg):
  55     (left, right) = arg
  56     return ast.Call(ast.Name("le", ast.Load()), (left, right), [])
  57
  58
  59 def make_ge_compare(arg):
  60     (left, right) = arg
  61     return ast.Call(ast.Name("ge", ast.Load()), (left, right), [])
  62
  63
  64 def make_lt_compare(arg):
  65     (left, right) = arg
  66     return ast.Call(ast.Name("lt", ast.Load()), (left, right), [])
  67
  68
  69 def make_gt_compare(arg):
  70     (left, right) = arg
  71     return ast.Call(ast.Name("gt", ast.Load()), (left, right), [])
  72
  73
  74 def make_eq_compare(arg):
  75     (left, right) = arg
  76     return ast.Call(ast.Name("eq", ast.Load()), (left, right), [])
  77
  78
  79 def make_ne_compare(arg):
  80     (left, right) = arg
  81     return ast.Call(ast.Name("ne", ast.Load()), (left, right), [])
  82
  83
  84 binary_ops = {
  85     "^": ast.BitXor(),
  86     "&": ast.BitAnd(),
  87     "|": ast.BitOr(),
  88     "+": ast.Add(),
  89     "-": ast.Sub(),
  90     "*": ast.Mult(),
  91     "/": ast.FloorDiv(),
  92     "%": ast.Mod(),
  93     "<=": make_le_compare,
  94     ">=": make_ge_compare,
  95     "<": make_lt_compare,
  96     ">": make_gt_compare,
  97     "=": make_eq_compare,
  98     "!=": make_ne_compare,
  99 }
 100 unary_ops = {
 101     "+": ast.UAdd(),
 102     "-": ast.USub(),
 103     "¬": ast.Invert(),
 104 }
 105
 106
 107 def check_concat(node):  # checks if the comparison is already a concat
 108     print("check concat", node)
 109     if not isinstance(node, ast.Call):
 110         return [node]
 111     node_func_id = getattr(node.func, "id", None)
 112     print("func", node_func_id)
 113     if node_func_id != 'concat':
 114         return [node]
 115     if node.keywords:  # a repeated list-constant, don't optimise
 116         return [node]
 117     return node.args
 118
 119
 120 # identify SelectableInt pattern [something] * N
 121 # must return concat(something, repeat=N)
 122 # here is a TEMPORARY hack to support minimal expressions
 123 # such as (XLEN-16), looking for ast.BinOp
 124 # have to keep an eye on this
 125 def identify_sint_mul_pattern(p):
 126     """here we are looking for patterns of this type:
 127         [item] * something
 128     these must specifically be returned as concat(item, repeat=something)
 129     """
 130     #print ("identify_sint_mul_pattern")
 131     # for pat in p:
 132     #    print("    ", astor.dump_tree(pat))
 133     if p[2] != '*':  # multiply
 134         return False
 135     if (not isinstance(p[3], ast.Constant) and  # rhs = Num
 136         not isinstance(p[3], ast.BinOp) and     # rhs = (XLEN-something)
 137             (not isinstance(p[3], ast.Name) and  # rhs = {a variable}
 138              not isinstance(p[3], ast.Attribute))):   # rhs = XLEN
 139         return False
 140     if not isinstance(p[1], ast.List):  # lhs is a list
 141         return False
 142     l = p[1].elts
 143     if len(l) != 1:  # lhs is a list of length 1
 144         return False
 145     return True  # yippee!
 146
 147
 148 ##########   Parser (tokens -> AST) ######
 149
 150 # also part of Ply
 151 #import yacc
 152
 153 # https://www.mathcs.emory.edu/~valerie/courses/fall10/155/resources/op_precedence.html
 154 # python operator precedence
 155 # Highest precedence at top, lowest at bottom.
 156 # Operators in the same box evaluate left to right.
 157 #
 158 # Operator Description
 159 # ()                                                     Parentheses (grouping)
 160 # f(args...)                                             Function call
 161 # x[index:index]                                         Slicing
 162 # x[index]                                               Subscription
 163 # x.attribute                                            Attribute reference
 164 # **                                                     Exponentiation
 165 # ~x                                                     Bitwise not
 166 # +x, -x                                                 Positive, negative
 167 # *, /, %                                                mul, div, remainder
 168 # +, -                                                   Addition, subtraction
 169 # <<, >>                                                 Bitwise shifts
 170 # &                                                      Bitwise AND
 171 # ^                                                      Bitwise XOR
 172 # |                                                      Bitwise OR
 173 # in, not in, is, is not, <, <=,  >,  >=, <>, !=, ==     comp, membership, ident
 174 # not x                                                  Boolean NOT
 175 # and                                                    Boolean AND
 176 # or                                                     Boolean OR
 177 # lambda                                                 Lambda expression
 178
 179
 180 class PowerParser:
 181
 182     precedence = (
 183         ("left", "EQ", "NE", "GT", "LT", "LE", "GE", "LTU", "GTU"),
 184         ("left", "BITOR"),
 185         ("left", "BITXOR"),
 186         ("left", "BITAND"),
 187         ("left", "PLUS", "MINUS"),
 188         ("left", "MULT", "DIV", "MOD"),
 189         ("left", "INVERT"),
 190     )
 191
 192     def reset(self):
 193         self.gprs = {}
 194         if self.form is not None:
 195             form = self.sd.sigforms[self.form]
 196             print(form)
 197             formkeys = form._asdict().keys()
 198         else:
 199             formkeys = []
 200         self.declared_vars = set()
 201         self.fnparm_vars = set()
 202         for rname in regs + fregs:
 203             self.gprs[rname] = None
 204             self.declared_vars.add(rname)
 205         self.available_op_fields = set()
 206         for k in formkeys:
 207             if k not in self.gprs:
 208                 if k == 'SPR':  # sigh, lower-case to not conflict
 209                     k = k.lower()
 210                 self.available_op_fields.add(k)
 211         self.op_fields = OrderedSet()
 212         self.read_regs = OrderedSet()
 213         self.uninit_regs = OrderedSet()
 214         self.write_regs = OrderedSet()
 215         self.special_regs = OrderedSet()  # see p_atom_name
 216
 217     def __init__(self, form, include_carry_in_write=False, helper=False):
 218         self.include_ca_in_write = include_carry_in_write
 219         self.helper = helper
 220         self.form = form
 221         self.filename = None
 222         self.input_text = None
 223         self.reset()
 224
 225     # The grammar comments come from Python's Grammar/Grammar file
 226
 227     # NB: compound_stmt in single_input is followed by extra NEWLINE!
 228     # file_input: (NEWLINE | stmt)* ENDMARKER
 229
 230     def p_file_input_end(self, p):
 231         """file_input_end : file_input ENDMARKER"""
 232         print("end", p[1])
 233         p[0] = p[1]
 234
 235     def p_file_input(self, p):
 236         """file_input : file_input NEWLINE
 237                       | file_input stmt
 238                       | NEWLINE
 239                       | stmt"""
 240         if isinstance(p[len(p)-1], str):
 241             if len(p) == 3:
 242                 p[0] = p[1]
 243             else:
 244                 p[0] = []  # p == 2 --> only a blank line
 245         else:
 246             if len(p) == 3:
 247                 p[0] = p[1] + p[2]
 248             else:
 249                 p[0] = p[1]
 250
 251     # funcdef: [decorators] 'def' NAME parameters ':' suite
 252     # ignoring decorators
 253
 254     def p_funcdef(self, p):
 255         "funcdef : DEF NAME parameters COLON suite"
 256         p[0] = ast.FunctionDef(p[2], p[3], p[5], (), lineno=p.lineno(2))
 257         # reset function parameters after suite is identified
 258         self.fnparm_vars = set()
 259
 260     # parameters: '(' [varargslist] ')'
 261     def p_parameters(self, p):
 262         """parameters : LPAR RPAR
 263                       | LPAR varargslist RPAR"""
 264         args = []
 265         if self.helper:
 266             args.append("self")
 267         if len(p) != 3:
 268             args += p[2]
 269         p[0] = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[])
 270         # during the time between parameters identified and suite is not
 271         # there is a window of opportunity to declare the function parameters
 272         # in-scope, for use to not overwrite them with auto-assign
 273         self.fnparm_vars = set()
 274         for arg in args:
 275             print("adding fn parm", arg)
 276             self.fnparm_vars.add(arg)
 277
 278     # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] |
 279     # '**' NAME) |
 280     # highly simplified
 281
 282     def p_varargslist(self, p):
 283         """varargslist : varargslist COMMA NAME
 284                        | NAME"""
 285         if len(p) == 4:
 286             print(p[1], p[3])
 287             p[0] = p[1] + [p[3]]
 288         else:
 289             p[0] = [p[1]]
 290
 291     # stmt: simple_stmt | compound_stmt
 292     def p_stmt_simple(self, p):
 293         """stmt : simple_stmt"""
 294         # simple_stmt is a list
 295         p[0] = p[1]
 296
 297     def p_stmt_compound(self, p):
 298         """stmt : compound_stmt"""
 299         p[0] = [p[1]]
 300
 301     # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 302     def p_simple_stmt(self, p):
 303         """simple_stmt : small_stmts NEWLINE
 304                        | small_stmts SEMICOLON NEWLINE"""
 305         p[0] = p[1]
 306
 307     def p_small_stmts(self, p):
 308         """small_stmts : small_stmts SEMICOLON small_stmt
 309                        | small_stmt"""
 310         if len(p) == 4:
 311             p[0] = p[1] + [p[3]]
 312         elif isinstance(p[1], list):
 313             p[0] = p[1]
 314         else:
 315             p[0] = [p[1]]
 316
 317     # small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
 318     #    import_stmt | global_stmt | exec_stmt | assert_stmt
 319     def p_small_stmt(self, p):
 320         """small_stmt : flow_stmt
 321                       | break_stmt
 322                       | expr_stmt"""
 323         if isinstance(p[1], ast.Call):
 324             p[0] = ast.Expr(p[1])
 325         elif isinstance(p[1], ast.Name) and p[1].id not in SPECIAL_HELPERS:
 326             fname = p[1].id
 327             name = ast.Name("self", ast.Load())
 328             name = ast.Attribute(name, fname, ast.Load())
 329             p[0] = ast.Call(name, [], [])
 330         else:
 331             p[0] = p[1]
 332
 333     # expr_stmt: testlist (augassign (yield_expr|testlist) |
 334     #                      ('=' (yield_expr|testlist))*)
 335     # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
 336     #             '<<=' | '>>=' | '**=' | '//=')
 337     def p_expr_stmt(self, p):
 338         """expr_stmt : testlist ASSIGNEA testlist
 339                      | testlist ASSIGN testlist
 340                      | testlist """
 341         print("expr_stmt", p)
 342         if len(p) == 2:
 343             # a list of expressions
 344             #p[0] = ast.Discard(p[1])
 345             p[0] = p[1]
 346         else:
 347             iea_mode = p[2] == '<-iea'
 348             if iea_mode:
 349                 print ("iea_mode")
 350             name = None
 351             autoassign = False
 352             if isinstance(p[1], ast.Name):
 353                 name = p[1].id
 354             elif isinstance(p[1], ast.Subscript):
 355                 print("assign subscript", p[1].value,
 356                       self.declared_vars,
 357                       self.fnparm_vars,
 358                       self.special_regs)
 359                 print(astor.dump_tree(p[1]))
 360                 print(astor.dump_tree(p[3]))
 361                 if isinstance(p[1].value, ast.Name):
 362                     name = p[1].value.id
 363                     print("assign subscript value to name", name)
 364                     if name in self.gprs:
 365                         # add to list of uninitialised
 366                         self.uninit_regs.add(name)
 367                     # work out if this is an ininitialised variable
 368                     # that should be auto-assigned simply by being "sliced"
 369                     autoassign = (name not in self.declared_vars and
 370                                   name not in self.fnparm_vars and
 371                                   name not in self.special_regs)
 372             elif isinstance(p[1], (ast.Attribute, ast.Tuple)):
 373                 pass
 374             elif isinstance(p[1], ast.Call) and p[1].func.id in \
 375                     ['GPR', 'FPR', 'SPR']:
 376                 print(astor.dump_tree(p[1]))
 377                 # replace GPR(x) with GPR[x]
 378                 idx = p[1].args[0].id
 379                 if idx in regs + fregs:
 380                     ridx = ast.Name("_%s" % idx, ast.Load())
 381                 else:
 382                     ridx = ast.Name(idx, ast.Load())
 383                 p[1] = ast.Subscript(p[1].func, ridx, ast.Load())
 384                 if idx in self.gprs:
 385                     self.read_regs.add(idx)  # add to list of regs to read
 386             elif isinstance(p[1], ast.Call) and p[1].func.id == 'MEM':
 387                 print("mem assign")
 388                 print(astor.dump_tree(p[1]))
 389                 p[1].func.id = "memassign"  # change function name to set
 390                 p[1].args.append(p[3])
 391                 p[0] = p[1]
 392                 print("mem rewrite")
 393                 print(astor.dump_tree(p[0]))
 394                 return
 395             else:
 396                 print(astor.dump_tree(p[1]))
 397                 raise_syntax_error("help, help", self.filename,
 398                                    p.slice[2].lineno, p.slice[2].lexpos,
 399                                    self.input_text)
 400             print("expr assign", name, p[1], "to", p[3])
 401             print(astor.dump_tree(p[3]))
 402             if isinstance(p[3], ast.Name):
 403                 toname = p[3].id
 404                 if toname in self.gprs:
 405                     self.read_regs.add(toname)
 406             if name and name in self.gprs:
 407                 self.write_regs.add(name)  # add to list of regs to write
 408             p[0] = self.Assign(autoassign, name, p[1], p[3], iea_mode,
 409                                p.slice[2])
 410             if name:
 411                 self.declared_vars.add(name)
 412
 413     def p_flow_stmt(self, p):
 414         "flow_stmt : return_stmt"
 415         p[0] = p[1]
 416
 417     # return_stmt: 'return' [testlist]
 418     def p_return_stmt(self, p):
 419         "return_stmt : RETURN testlist"
 420         p[0] = ast.Return(p[2])
 421
 422     def p_compound_stmt(self, p):
 423         """compound_stmt : if_stmt
 424                          | while_stmt
 425                          | switch_stmt
 426                          | for_stmt
 427                          | funcdef
 428         """
 429         p[0] = p[1]
 430
 431     def p_break_stmt(self, p):
 432         """break_stmt : BREAK
 433         """
 434         p[0] = ast.Break()
 435
 436     def p_for_stmt(self, p):
 437         """for_stmt : FOR atom EQ comparison TO comparison COLON suite
 438                     | DO atom EQ comparison TO comparison COLON suite
 439         """
 440         start = p[4]
 441         end = p[6]
 442         it = ast.Call(ast.Name("RANGE", ast.Load()), (start, end), [])
 443         p[0] = ast.For(p[2], it, p[8], [])
 444
 445     def p_while_stmt(self, p):
 446         """while_stmt : DO WHILE test COLON suite ELSE COLON suite
 447                       | DO WHILE test COLON suite
 448         """
 449         if len(p) == 6:
 450             p[0] = ast.While(p[3], p[5], [])
 451         else:
 452             p[0] = ast.While(p[3], p[5], p[8])
 453
 454     def p_switch_smt(self, p):
 455         """switch_stmt : SWITCH LPAR atom RPAR COLON NEWLINE INDENT switches DEDENT
 456         """
 457         switchon = p[3]
 458         print("switch stmt")
 459         print(astor.dump_tree(p[1]))
 460
 461         cases = []
 462         current_cases = []  # for deferral
 463         for (case, suite) in p[8]:
 464             print("for", case, suite)
 465             if suite is None:
 466                 for c in case:
 467                     current_cases.append(ast.Num(c))
 468                 continue
 469             if case == 'default':  # last
 470                 break
 471             for c in case:
 472                 current_cases.append(ast.Num(c))
 473             print("cases", current_cases)
 474             compare = ast.Compare(switchon, [ast.In()],
 475                                   [ast.List(current_cases, ast.Load())])
 476             current_cases = []
 477             cases.append((compare, suite))
 478
 479         print("ended", case, current_cases)
 480         if case == 'default':
 481             if current_cases:
 482                 compare = ast.Compare(switchon, [ast.In()],
 483                                       [ast.List(current_cases, ast.Load())])
 484                 cases.append((compare, suite))
 485             cases.append((None, suite))
 486
 487         cases.reverse()
 488         res = []
 489         for compare, suite in cases:
 490             print("after rev", compare, suite)
 491             if compare is None:
 492                 assert len(res) == 0, "last case should be default"
 493                 res = suite
 494             else:
 495                 if not isinstance(res, list):
 496                     res = [res]
 497                 res = ast.If(compare, suite, res)
 498         p[0] = res
 499
 500     def p_switches(self, p):
 501         """switches : switch_list switch_default
 502                     | switch_default
 503         """
 504         if len(p) == 3:
 505             p[0] = p[1] + [p[2]]
 506         else:
 507             p[0] = [p[1]]
 508
 509     def p_switch_list(self, p):
 510         """switch_list : switch_case switch_list
 511                        | switch_case
 512         """
 513         if len(p) == 3:
 514             p[0] = [p[1]] + p[2]
 515         else:
 516             p[0] = [p[1]]
 517
 518     def p_switch_case(self, p):
 519         """switch_case : CASE LPAR atomlist RPAR COLON suite
 520         """
 521         # XXX bad hack
 522         if isinstance(p[6][0], ast.Name) and p[6][0].id == 'fallthrough':
 523             p[6] = None
 524         p[0] = (p[3], p[6])
 525
 526     def p_switch_default(self, p):
 527         """switch_default : DEFAULT COLON suite
 528         """
 529         p[0] = ('default', p[3])
 530
 531     def p_atomlist(self, p):
 532         """atomlist : atom COMMA atomlist
 533                     | atom
 534         """
 535         assert isinstance(p[1], ast.Constant), "case must be numbers"
 536         if len(p) == 4:
 537             p[0] = [p[1].value] + p[3]
 538         else:
 539             p[0] = [p[1].value]
 540
 541     def p_if_stmt(self, p):
 542         """if_stmt : IF test COLON suite ELSE COLON if_stmt
 543                    | IF test COLON suite ELSE COLON suite
 544                    | IF test COLON suite
 545         """
 546         if len(p) == 8 and isinstance(p[7], ast.If):
 547             p[0] = ast.If(p[2], p[4], [p[7]])
 548         elif len(p) == 5:
 549             p[0] = ast.If(p[2], p[4], [])
 550         else:
 551             p[0] = ast.If(p[2], p[4], p[7])
 552
 553     def p_suite(self, p):
 554         """suite : simple_stmt
 555                  | NEWLINE INDENT stmts DEDENT"""
 556         if len(p) == 2:
 557             p[0] = p[1]
 558         else:
 559             p[0] = p[3]
 560
 561     def p_stmts(self, p):
 562         """stmts : stmts stmt
 563                  | stmt"""
 564         if len(p) == 3:
 565             p[0] = p[1] + p[2]
 566         else:
 567             p[0] = p[1]
 568
 569     def p_comparison(self, p):
 570         """comparison : comparison PLUS comparison
 571                       | comparison MINUS comparison
 572                       | comparison MULT comparison
 573                       | comparison DIV comparison
 574                       | comparison MOD comparison
 575                       | comparison EQ comparison
 576                       | comparison NE comparison
 577                       | comparison LE comparison
 578                       | comparison GE comparison
 579                       | comparison LTU comparison
 580                       | comparison GTU comparison
 581                       | comparison LT comparison
 582                       | comparison GT comparison
 583                       | comparison BITOR comparison
 584                       | comparison BITXOR comparison
 585                       | comparison BITAND comparison
 586                       | PLUS comparison
 587                       | MINUS comparison
 588                       | INVERT comparison
 589                       | comparison APPEND comparison
 590                       | power"""
 591         if len(p) == 4:
 592             print(list(p))
 593             if p[2] == '<u':
 594                 p[0] = ast.Call(ast.Name("ltu", ast.Load()), (p[1], p[3]), [])
 595             elif p[2] == '>u':
 596                 p[0] = ast.Call(ast.Name("gtu", ast.Load()), (p[1], p[3]), [])
 597             elif p[2] == '||':
 598                 l = check_concat(p[1]) + check_concat(p[3])
 599                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, [])
 600             elif p[2] in ['/', '%']:
 601                 # bad hack: if % or / used anywhere other than div/mod ops,
 602                 # do % or /.  however if the argument names are "dividend"
 603                 # we must call the special trunc_divs and trunc_rems functions
 604                 l, r = p[1], p[3]
 605                 # actual call will be "dividend / divisor" - just check
 606                 # LHS name
 607                 # XXX DISABLE BAD HACK (False)
 608                 if False and isinstance(l, ast.Name) and l.id == 'dividend':
 609                     if p[2] == '/':
 610                         fn = 'trunc_divs'
 611                     else:
 612                         fn = 'trunc_rems'
 613                     # return "function trunc_xxx(l, r)"
 614                     p[0] = ast.Call(ast.Name(fn, ast.Load()), (l, r), [])
 615                 else:
 616                     # return "l {binop} r"
 617                     p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 618             elif p[2] in ['<', '>', '=', '<=', '>=', '!=']:
 619                 p[0] = binary_ops[p[2]]((p[1], p[3]))
 620             elif identify_sint_mul_pattern(p):
 621                 keywords = [ast.keyword(arg='repeat', value=p[3])]
 622                 l = p[1].elts
 623                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 624             else:
 625                 p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 626                 # HORRENDOUS hack, add brackets around the bin-op by
 627                 # creating a function call with a *blank* function name!
 628                 # XXX argh doesn't work because of analysis of
 629                 # identify_sint_pattern
 630                 #p[0] = ast.Call(ast.Name("", ast.Load()), [p[0]], [])
 631         elif len(p) == 3:
 632             if isinstance(p[2], str) and p[2] == '-':
 633                 p[0] = ast.UnaryOp(unary_ops[p[2]], p[1])
 634             else:
 635                 p[0] = ast.UnaryOp(unary_ops[p[1]], p[2])
 636         else:
 637             p[0] = p[1]
 638
 639     # power: atom trailer* ['**' factor]
 640     # trailers enables function calls (and subscripts).
 641     # so this is 'trailerlist'
 642     def p_power(self, p):
 643         """power : atom
 644                  | atom trailerlist"""
 645         if len(p) == 2:
 646             print("power dump atom notrailer")
 647             print(astor.dump_tree(p[1]))
 648             p[0] = p[1]
 649         else:
 650             print("power dump atom")
 651             print(astor.dump_tree(p[1]))
 652             print("power dump trailerlist")
 653             print(astor.dump_tree(p[2]))
 654             p[0] = self.apply_trailer(p[1], p[2], self.read_regs)
 655             if isinstance(p[1], ast.Name):
 656                 name = p[1].id
 657                 if name in regs + fregs:
 658                     self.read_regs.add(name)
 659
 660     def p_atom_name(self, p):
 661         """atom : NAME"""
 662         name = p[1]
 663         if name in self.available_op_fields:
 664             self.op_fields.add(name)
 665         if name in ['overflow', 'CR0']:
 666             self.write_regs.add(name)
 667         if self.include_ca_in_write:
 668             if name in ['CA', 'CA32']:
 669                 self.write_regs.add(name)
 670         if name in ['CR', 'LR', 'CTR', 'TAR', 'FPSCR', 'MSR',
 671                     'SVSTATE', 'SVREMAP', 'SRR0', 'SRR1',
 672                     'SVSHAPE0', 'SVSHAPE1', 'SVSHAPE2', 'SVSHAPE3']:
 673             self.special_regs.add(name)
 674             self.write_regs.add(name)  # and add to list to write
 675         if name in ('XLEN', ) or name in BFP_FLAG_NAMES:
 676             attr = ast.Name("self", ast.Load())
 677             p[0] = ast.Attribute(attr, name, ast.Load(), lineno=p.lineno(1))
 678         else:
 679             p[0] = ast.Name(id=name, ctx=ast.Load(), lineno=p.lineno(1))
 680
 681     def p_atom_number(self, p):
 682         """atom : BINARY
 683                 | NUMBER
 684                 | HEX
 685                 | STRING"""
 686         p[0] = ast.Constant(p[1])
 687
 688     # '[' [listmaker] ']' |
 689
 690     def p_atom_listmaker(self, p):
 691         """atom : LBRACK listmaker RBRACK"""
 692         p[0] = p[2]
 693
 694     def p_listmaker(self, p):
 695         """listmaker : test COMMA listmaker
 696                      | test
 697         """
 698         if len(p) == 2:
 699             p[0] = ast.List([p[1]], ast.Load())
 700         else:
 701             p[0] = ast.List([p[1]] + p[3].nodes, ast.Load())
 702
 703     def p_atom_tuple(self, p):
 704         """atom : LPAR testlist RPAR"""
 705         print("tuple", p[2])
 706         print("astor dump")
 707         print(astor.dump_tree(p[2]))
 708
 709         if isinstance(p[2], ast.Name):
 710             name = p[2].id
 711             print("tuple name", name)
 712             if name in self.gprs:
 713                 self.read_regs.add(name)  # add to list of regs to read
 714                 #p[0] = ast.Subscript(ast.Name("GPR", ast.Load()), ast.Str(p[2].id))
 715                 # return
 716             p[0] = p[2]
 717         elif isinstance(p[2], ast.BinOp):
 718             if isinstance(p[2].left, ast.Name) and \
 719                isinstance(p[2].right, ast.Constant) and \
 720                     p[2].right.value == 0 and \
 721                     p[2].left.id in self.gprs:
 722                 rid = p[2].left.id
 723                 self.read_regs.add(rid)  # add to list of regs to read
 724                 # create special call to GPR.getz or FPR.getz
 725                 if rid in fregs:
 726                     gprz = ast.Name("FPR", ast.Load())
 727                 else:
 728                     gprz = ast.Name("GPR", ast.Load())
 729                 # get testzero function
 730                 gprz = ast.Attribute(gprz, "getz", ast.Load())
 731                 # *sigh* see class GPR.  we need index itself not reg value
 732                 ridx = ast.Name("_%s" % rid, ast.Load())
 733                 p[0] = ast.Call(gprz, [ridx], [])
 734                 print("tree", astor.dump_tree(p[0]))
 735             else:
 736                 p[0] = p[2]
 737         else:
 738             p[0] = p[2]
 739
 740     def p_trailerlist(self, p):
 741         """trailerlist : trailer trailerlist
 742                        | trailer
 743         """
 744         if len(p) == 2:
 745             p[0] = p[1]
 746         else:
 747             p[0] = ("TLIST", p[1], p[2])
 748
 749     # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 750     def p_trailer(self, p):
 751         """trailer : trailer_arglist
 752                    | trailer_subscript
 753                    | trailer_attr
 754         """
 755         p[0] = p[1]
 756
 757     def p_trailer_arglist(self, p):
 758         """trailer_arglist : LPAR arglist RPAR
 759                            | LPAR RPAR
 760         """
 761         args = [] if len(p) == 3 else p[2]
 762         p[0] = ("CALL", args, p.slice[1])
 763
 764     def p_trailer_subscript(self, p):
 765         "trailer_subscript : LBRACK subscript RBRACK"
 766         p[0] = ("SUBS", p[2])
 767
 768     def p_trailer_attr(self, p):
 769         "trailer_attr : PERIOD NAME"
 770         p[0] = ("ATTR", p[2])
 771
 772     # subscript: '.' '.' '.' | test | [test] ':' [test]
 773
 774     def p_subscript(self, p):
 775         """subscript : test COLON test
 776                      | test
 777         """
 778         if len(p) == 4:
 779             # add one to end
 780             if isinstance(p[3], ast.Constant):
 781                 end = ast.Constant(p[3].value+1)
 782             else:
 783                 end = ast.BinOp(p[3], ast.Add(), ast.Constant(1))
 784             p[0] = [p[1], end]
 785         else:
 786             p[0] = [p[1]]
 787
 788     # testlist: test (',' test)* [',']
 789     # Contains shift/reduce error
 790
 791     def p_testlist(self, p):
 792         """testlist : testlist_multi COMMA
 793                     | testlist_multi """
 794         if len(p) == 2:
 795             p[0] = p[1]
 796         else:
 797             # May need to promote singleton to tuple
 798             if isinstance(p[1], list):
 799                 p[0] = p[1]
 800             else:
 801                 p[0] = [p[1]]
 802         # Convert into a tuple?
 803         if isinstance(p[0], list):
 804             p[0] = ast.Tuple(p[0])
 805
 806     def p_testlist_multi(self, p):
 807         """testlist_multi : testlist_multi COMMA test
 808                           | test"""
 809         if len(p) == 2:
 810             # singleton
 811             p[0] = p[1]
 812         else:
 813             if isinstance(p[1], list):
 814                 p[0] = p[1] + [p[3]]
 815             else:
 816                 # singleton -> tuple
 817                 p[0] = [p[1], p[3]]
 818
 819     # test: or_test ['if' or_test 'else' test] | lambdef
 820     #  as I don't support 'and', 'or', and 'not' this works down to 'comparison'
 821
 822     def p_test(self, p):
 823         """test : comparison
 824                 | comparison QMARK test COLON test"""
 825         if len(p) == 2:
 826             p[0] = p[1]
 827         else:
 828             p[0] = ast.IfExp(test=p[1], body=p[3], orelse=p[5])
 829
 830     # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
 831     # | '**' test)
 832     # XXX INCOMPLETE: this doesn't allow the trailing comma
 833
 834     def p_arglist(self, p):
 835         """arglist : arglist COMMA argument
 836                    | argument"""
 837         if len(p) == 4:
 838             p[0] = p[1] + [p[3]]
 839         else:
 840             p[0] = [p[1]]
 841
 842     # argument: test [gen_for] | test '=' test  # Really [keyword '='] test
 843     def p_argument(self, p):
 844         "argument : test"
 845         p[0] = p[1]
 846
 847     def p_error(self, p):
 848         # print "Error!", repr(p)
 849         raise_syntax_error(str(p), self.filename, p.lineno, p.lexpos,
 850                            self.input_text)
 851
 852     def Assign(self, autoassign, assignname, left, right, iea_mode, eq_tok):
 853         names = []
 854         print("Assign", autoassign, assignname, left, right)
 855         if isinstance(left, ast.Name):
 856             # Single assignment on left
 857             # XXX when doing IntClass, which will have an "eq" function,
 858             # this is how to access it
 859             #   eq = ast.Attribute(left, "eq")   # get eq fn
 860             #   return ast.Call(eq, [right], []) # now call left.eq(right)
 861             return ast.Assign([ast.Name(left.id, ast.Store())], right)
 862         elif isinstance(left, ast.Tuple):
 863             # List of things - make sure they are Name nodes
 864             names = []
 865             for child in left.elts:
 866                 if not isinstance(child, ast.Name):
 867                     raise_syntax_error(
 868                         "that assignment not supported", self.filename,
 869                         eq_tok.lineno, eq_tok.lexpos, self.input_text)
 870                 names.append(child.id)
 871             ass_list = [ast.Name(name, ast.Store()) for name in names]
 872             return ast.Assign([ast.Tuple(ass_list)], right)
 873         elif isinstance(left, ast.Attribute):
 874             return ast.Assign([
 875                 ast.Attribute(left.value, left.attr, ast.Store())], right)
 876         elif isinstance(left, ast.Subscript):
 877             ls = left.slice
 878             # XXX changing meaning of "undefined" to a function
 879             # if (isinstance(ls, ast.Slice) and isinstance(right, ast.Name) and
 880             #        right.id == 'undefined'):
 881             #    # undefined needs to be copied the exact same slice
 882             #    right = ast.Subscript(right, ls, ast.Load())
 883             #    return ast.Assign([left], right)
 884             res = ast.Assign([left], right)
 885             if autoassign and isinstance(ls, ast.Slice):
 886                 # hack to create a variable pre-declared based on a slice.
 887                 # dividend[0:32] = (RA)[0:32] will create
 888                 #       dividend = [0] * 32
 889                 #       dividend[0:32] = (RA)[0:32]
 890                 # the declaration makes the slice-assignment "work"
 891                 lower, upper, step = ls.lower, ls.upper, ls.step
 892                 print("lower, upper, step", repr(lower), repr(upper), step)
 893                 # XXX relax constraint that indices on auto-assign have
 894                 # to be constants x[0:32]
 895                 # if not isinstance(lower, ast.Constant) or \
 896                 #   not isinstance(upper, ast.Constant):
 897                 #    return res
 898                 qty = ast.BinOp(upper, binary_ops['-'], lower)
 899                 keywords = [ast.keyword(arg='repeat', value=qty)]
 900                 l = [ast.Num(0)]
 901                 right = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 902                 declare = ast.Assign(
 903                     [ast.Name(assignname, ast.Store())], right)
 904                 return [declare, res]
 905             return res
 906             # XXX HMMM probably not needed...
 907             ls = left.slice
 908             if isinstance(ls, ast.Slice):
 909                 lower, upper, step = ls.lower, ls.upper, ls.step
 910                 print("slice assign", lower, upper, step)
 911                 if step is None:
 912                     ls = (lower, upper, None)
 913                 else:
 914                     ls = (lower, upper, step)
 915                 ls = ast.Tuple(ls)
 916             return ast.Call(ast.Name("selectassign", ast.Load()),
 917                             [left.value, ls, right], [])
 918         else:
 919             print("Assign fail")
 920             raise_syntax_error("Can't do that yet", self.filename,
 921                                eq_tok.lineno, eq_tok.lexpos, self.input_text)
 922
 923     def try_extract_name(self, atom):
 924         if isinstance(atom, ast.Attribute):
 925             if isinstance(atom.value, ast.Name) and atom.value.id == "self":
 926                 return atom.attr
 927             return None
 928         if isinstance(atom, ast.Name):
 929             return atom.id
 930         return None
 931
 932     def try_extract_uppercase_name(self, atom):
 933         name = self.try_extract_name(atom)
 934         if name is None:
 935             return None
 936         # we want to filter out names like _123 that have no letters with
 937         # casing at all, hence the lower() check
 938         if name.upper() != name or name.lower() == name:
 939             return None
 940         return name
 941
 942     def try_get_attr(self, atom, trailer):
 943         if trailer[0] == "ATTR":
 944             return trailer[1]
 945         if trailer[0] != "SUBS":
 946             return None
 947         base_name = self.try_extract_uppercase_name(atom)
 948         if base_name is None or base_name in SUBS_TO_ATTR_EXCEPTIONS:
 949             return None
 950         if len(trailer[1]) != 1:
 951             return None
 952         return self.try_extract_uppercase_name(trailer[1][0])
 953
 954     def apply_trailer(self, atom, trailer, read_regs):
 955         if trailer[0] == "TLIST":
 956             # assume depth of one
 957             atom = self.apply_trailer(atom, trailer[1], read_regs)
 958             trailer = trailer[2]
 959         if trailer[0] == "CALL":
 960             #p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
 961             for arg in trailer[1]:
 962                 if isinstance(arg, ast.Name):
 963                     name = arg.id
 964                     if name in regs + fregs:
 965                         read_regs.add(name)
 966             if atom.id == "SetFX":
 967                 if len(trailer[1]) != 1 or \
 968                         not isinstance(trailer[1][0], ast.Attribute):
 969                     raise_syntax_error(
 970                         "SetFX call must have only one argument that is an "
 971                         "attribute access -- like `SetFX(FPSCR.VXSNAN)`",
 972                         self.filename, trailer[2].lineno, trailer[2].lexpos,
 973                         self.input_text)
 974                 arg = trailer[1][0]
 975                 args = [arg.value, ast.Str(arg.attr)]
 976                 name = ast.Name("self", ast.Load())
 977                 atom = ast.Attribute(name, atom, ast.Load())
 978                 return ast.Call(atom, args, [])
 979             # special-case, these functions must NOT be made "self.xxxx"
 980             if atom.id not in SPECIAL_HELPERS:
 981                 name = ast.Name("self", ast.Load())
 982                 atom = ast.Attribute(name, atom, ast.Load())
 983             return ast.Call(atom, trailer[1], [])
 984             # if p[1].id == 'print':
 985             #    p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
 986             # else:
 987             #    p[0] = ast.CallFunc(p[1], p[2][1], None, None)
 988         else:
 989             attr = self.try_get_attr(atom, trailer)
 990             if attr is not None:
 991                 return ast.Attribute(atom, attr, ast.Load())
 992             assert trailer[0] == "SUBS"
 993             print("subscript atom", trailer[1])
 994             #raise AssertionError("not implemented %s" % p[2][0])
 995             subs = trailer[1]
 996             if len(subs) == 1:
 997                 idx = subs[0]
 998                 if isinstance(idx, ast.Name) and idx.id in regs + fregs:
 999                     read_regs.add(idx.id)
1000                 if isinstance(idx, ast.Name) and idx.id in regs:
1001                     print("single atom subscript, underscored", idx.id)
1002                     idx = ast.Name("_%s" % idx.id, ast.Load())
1003             else:
1004                 idx = ast.Slice(subs[0], subs[1], None)
1005             # if isinstance(atom, ast.Name) and atom.id == 'CR':
1006                 # atom.id = 'CR' # bad hack
1007                 #print ("apply_trailer Subscript", atom.id, idx)
1008             return ast.Subscript(atom, idx, ast.Load())
1009
1010
1011 _CACHE_DECODER = True
1012 _CACHED_DECODER = None
1013
1014
1015 def _create_cached_decoder():
1016     global _CACHED_DECODER
1017     if _CACHE_DECODER:
1018         if _CACHED_DECODER is None:
1019             _CACHED_DECODER = create_pdecode()
1020         return _CACHED_DECODER
1021     return create_pdecode()
1022
1023
1024 class GardenSnakeParser(PowerParser):
1025     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1026         if form is not None:
1027             self.sd = _create_cached_decoder()
1028         PowerParser.__init__(self, form, incl_carry, helper=helper)
1029         self.debug = debug
1030         self.lexer = IndentLexer(debug=0)
1031         self.tokens = self.lexer.tokens
1032         self.parser = yacc.yacc(module=self, start="file_input_end",
1033                                 debug=debug, write_tables=False)
1034
1035     def parse(self, code):
1036         self.reset()
1037         self.lexer.filename = self.filename
1038         result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
1039         if self.helper:
1040             result = [ast.ClassDef("ISACallerFnHelper", [
1041                                    "ISACallerHelper"], [], result, decorator_list=[])]
1042         return ast.Module(result)
1043
1044
1045 ###### Code generation ######
1046
1047 #from compiler import misc, syntax, pycodegen
1048
1049 _CACHED_PARSERS = {}
1050 _CACHE_PARSERS = True
1051
1052
1053 class GardenSnakeCompiler(object):
1054     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1055         if _CACHE_PARSERS:
1056             try:
1057                 self.parser = _CACHED_PARSERS[debug, form, incl_carry, helper]
1058             except KeyError:
1059                 self.parser = GardenSnakeParser(
1060                     debug=debug, form=form, incl_carry=incl_carry,
1061                     helper=helper)
1062                 _CACHED_PARSERS[debug, form, incl_carry, helper] = self.parser
1063         else:
1064             self.parser = GardenSnakeParser(debug=debug, form=form,
1065                                             incl_carry=incl_carry, helper=helper)
1066
1067     def compile(self, code, mode="exec", filename="<string>"):
1068         if filename in ["string", "<string>"]:
1069             raise ValueError("missing filename")
1070         self.parser.filename = filename
1071         self.parser.input_text = code
1072         try:
1073             tree = self.parser.parse(code)
1074         except SyntaxError2 as e:
1075             e.raise_syntax_error()
1076         print("snake")
1077         pprint(tree)
1078         return tree
1079         #misc.set_filename(filename, tree)
1080         return compile(tree, mode="exec", filename="<string>")
1081         # syntax.check(tree)
1082         gen = pycodegen.ModuleCodeGenerator(tree)
1083         code = gen.getCode()
1084         return code