src/openpower/decoder/pseudo/parser.py

   1 # Based on GardenSnake - a parser generator demonstration program
   2 # GardenSnake was released into the Public Domain by Andrew Dalke.
   3
   4 # Portions of this work are derived from Python's Grammar definition
   5 # and may be covered under the Python copyright and license
   6 #
   7 #          Andrew Dalke / Dalke Scientific Software, LLC
   8 #             30 August 2006 / Cape Town, South Africa
   9
  10 # Modifications for inclusion in PLY distribution
  11 from pprint import pprint
  12 from ply import lex, yacc
  13 import astor
  14 from copy import deepcopy
  15
  16 from openpower.decoder.power_decoder import create_pdecode
  17 from openpower.decoder.pseudo.lexer import (
  18     IndentLexer, raise_syntax_error, SyntaxError2)
  19 from openpower.decoder.orderedset import OrderedSet
  20 from openpower.decoder.power_enums import BFP_FLAG_NAMES
  21
  22 # I use the Python AST
  23 #from compiler import ast
  24 import ast
  25
  26 # Helper function
  27
  28 regs = ['RA', 'RS', 'RB', 'RC', 'RT']
  29 fregs = ['FRA', 'FRS', 'FRB', 'FRC', 'FRT', 'FRS']
  30 SPECIAL_HELPERS = {'concat', 'MEM', 'GPR', 'FPR', 'SPR', 'pow'}
  31 SUBS_TO_ATTR_EXCEPTIONS = SPECIAL_HELPERS | {'CR'}
  32
  33
  34 # I implemented INDENT / DEDENT generation as a post-processing filter
  35
  36 # The original lex token stream contains WS and NEWLINE characters.
  37 # WS will only occur before any other tokens on a line.
  38
  39 # I have three filters.  One tags tokens by adding two attributes.
  40 # "must_indent" is True if the token must be indented from the
  41 # previous code.  The other is "at_line_start" which is True for WS
  42 # and the first non-WS/non-NEWLINE on a line.  It flags the check so
  43 # see if the new line has changed indication level.
  44
  45
  46 # No using Python's approach because Ply supports precedence
  47
  48 # comparison: expr (comp_op expr)*
  49 # arith_expr: term (('+'|'-') term)*
  50 # term: factor (('*'|'/'|'%'|'//') factor)*
  51 # factor: ('+'|'-'|'~') factor | power
  52 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
  53
  54 def make_le_compare(arg):
  55     (left, right) = arg
  56     return ast.Call(ast.Name("le", ast.Load()), (left, right), [])
  57
  58
  59 def make_ge_compare(arg):
  60     (left, right) = arg
  61     return ast.Call(ast.Name("ge", ast.Load()), (left, right), [])
  62
  63
  64 def make_lt_compare(arg):
  65     (left, right) = arg
  66     return ast.Call(ast.Name("lt", ast.Load()), (left, right), [])
  67
  68
  69 def make_gt_compare(arg):
  70     (left, right) = arg
  71     return ast.Call(ast.Name("gt", ast.Load()), (left, right), [])
  72
  73
  74 def make_eq_compare(arg):
  75     (left, right) = arg
  76     return ast.Call(ast.Name("eq", ast.Load()), (left, right), [])
  77
  78
  79 def make_ne_compare(arg):
  80     (left, right) = arg
  81     return ast.Call(ast.Name("ne", ast.Load()), (left, right), [])
  82
  83
  84 binary_ops = {
  85     "^": ast.BitXor(),
  86     "&": ast.BitAnd(),
  87     "|": ast.BitOr(),
  88     "+": ast.Add(),
  89     "-": ast.Sub(),
  90     "*": ast.Mult(),
  91     "/": ast.FloorDiv(),
  92     "%": ast.Mod(),
  93     "<=": make_le_compare,
  94     ">=": make_ge_compare,
  95     "<": make_lt_compare,
  96     ">": make_gt_compare,
  97     "=": make_eq_compare,
  98     "!=": make_ne_compare,
  99 }
 100 unary_ops = {
 101     "+": ast.UAdd(),
 102     "-": ast.USub(),
 103     "¬": ast.Invert(),
 104 }
 105
 106
 107 def check_concat(node):  # checks if the comparison is already a concat
 108     print("check concat", node)
 109     if not isinstance(node, ast.Call):
 110         return [node]
 111     node_func_id = getattr(node.func, "id", None)
 112     print("func", node_func_id)
 113     if node_func_id != 'concat':
 114         return [node]
 115     if node.keywords:  # a repeated list-constant, don't optimise
 116         return [node]
 117     return node.args
 118
 119
 120 # identify SelectableInt pattern [something] * N
 121 # must return concat(something, repeat=N)
 122 # here is a TEMPORARY hack to support minimal expressions
 123 # such as (XLEN-16), looking for ast.BinOp
 124 # have to keep an eye on this
 125 def identify_sint_mul_pattern(p):
 126     """here we are looking for patterns of this type:
 127         [item] * something
 128     these must specifically be returned as concat(item, repeat=something)
 129     """
 130     #print ("identify_sint_mul_pattern")
 131     # for pat in p:
 132     #    print("    ", astor.dump_tree(pat))
 133     if p[2] != '*':  # multiply
 134         return False
 135     if (not isinstance(p[3], ast.Constant) and  # rhs = Num
 136         not isinstance(p[3], ast.BinOp) and     # rhs = (XLEN-something)
 137             (not isinstance(p[3], ast.Name) and  # rhs = {a variable}
 138              not isinstance(p[3], ast.Attribute))):   # rhs = XLEN
 139         return False
 140     if not isinstance(p[1], ast.List):  # lhs is a list
 141         return False
 142     l = p[1].elts
 143     if len(l) != 1:  # lhs is a list of length 1
 144         return False
 145     return True  # yippee!
 146
 147
 148 ##########   Parser (tokens -> AST) ######
 149
 150 # also part of Ply
 151 #import yacc
 152
 153 # https://www.mathcs.emory.edu/~valerie/courses/fall10/155/resources/op_precedence.html
 154 # python operator precedence
 155 # Highest precedence at top, lowest at bottom.
 156 # Operators in the same box evaluate left to right.
 157 #
 158 # Operator Description
 159 # ()                                                     Parentheses (grouping)
 160 # f(args...)                                             Function call
 161 # x[index:index]                                         Slicing
 162 # x[index]                                               Subscription
 163 # x.attribute                                            Attribute reference
 164 # **                                                     Exponentiation
 165 # ~x                                                     Bitwise not
 166 # +x, -x                                                 Positive, negative
 167 # *, /, %                                                mul, div, remainder
 168 # +, -                                                   Addition, subtraction
 169 # <<, >>                                                 Bitwise shifts
 170 # &                                                      Bitwise AND
 171 # ^                                                      Bitwise XOR
 172 # |                                                      Bitwise OR
 173 # in, not in, is, is not, <, <=,  >,  >=, <>, !=, ==     comp, membership, ident
 174 # not x                                                  Boolean NOT
 175 # and                                                    Boolean AND
 176 # or                                                     Boolean OR
 177 # lambda                                                 Lambda expression
 178
 179
 180 class PowerParser:
 181
 182     precedence = (
 183         ("left", "EQ", "NE", "GT", "LT", "LE", "GE", "LTU", "GTU"),
 184         ("left", "BITOR"),
 185         ("left", "BITXOR"),
 186         ("left", "BITAND"),
 187         ("left", "PLUS", "MINUS"),
 188         ("left", "MULT", "DIV", "MOD"),
 189         ("left", "INVERT"),
 190     )
 191
 192     def reset(self):
 193         self.gprs = {}
 194         if self.form is not None:
 195             form = self.sd.sigforms[self.form]
 196             print(form)
 197             formkeys = form._asdict().keys()
 198         else:
 199             formkeys = []
 200         self.declared_vars = set()
 201         self.fnparm_vars = set()
 202         for rname in regs + fregs:
 203             self.gprs[rname] = None
 204             self.declared_vars.add(rname)
 205         self.available_op_fields = set()
 206         for k in formkeys:
 207             if k not in self.gprs:
 208                 if k == 'SPR':  # sigh, lower-case to not conflict
 209                     k = k.lower()
 210                 self.available_op_fields.add(k)
 211         self.op_fields = OrderedSet()
 212         self.read_regs = OrderedSet()
 213         self.uninit_regs = OrderedSet()
 214         self.write_regs = OrderedSet()
 215         self.special_regs = OrderedSet()  # see p_atom_name
 216
 217     def __init__(self, form, include_carry_in_write=False, helper=False):
 218         self.include_ca_in_write = include_carry_in_write
 219         self.helper = helper
 220         self.form = form
 221         self.filename = None
 222         self.input_text = None
 223         self.reset()
 224
 225     # The grammar comments come from Python's Grammar/Grammar file
 226
 227     # NB: compound_stmt in single_input is followed by extra NEWLINE!
 228     # file_input: (NEWLINE | stmt)* ENDMARKER
 229
 230     def p_file_input_end(self, p):
 231         """file_input_end : file_input ENDMARKER"""
 232         print("end", p[1])
 233         p[0] = p[1]
 234
 235     def p_file_input(self, p):
 236         """file_input : file_input NEWLINE
 237                       | file_input stmt
 238                       | NEWLINE
 239                       | stmt"""
 240         if isinstance(p[len(p)-1], str):
 241             if len(p) == 3:
 242                 p[0] = p[1]
 243             else:
 244                 p[0] = []  # p == 2 --> only a blank line
 245         else:
 246             if len(p) == 3:
 247                 p[0] = p[1] + p[2]
 248             else:
 249                 p[0] = p[1]
 250
 251     # funcdef: [decorators] 'def' NAME parameters ':' suite
 252     # ignoring decorators
 253
 254     def p_funcdef(self, p):
 255         "funcdef : DEF NAME parameters COLON suite"
 256         p[0] = ast.FunctionDef(p[2], p[3], p[5], (), lineno=p.lineno(2))
 257         # reset function parameters after suite is identified
 258         self.fnparm_vars = set()
 259
 260     # parameters: '(' [varargslist] ')'
 261     def p_parameters(self, p):
 262         """parameters : LPAR RPAR
 263                       | LPAR varargslist RPAR"""
 264         args = []
 265         if self.helper:
 266             args.append("self")
 267         if len(p) != 3:
 268             args += p[2]
 269         p[0] = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[])
 270         # during the time between parameters identified and suite is not
 271         # there is a window of opportunity to declare the function parameters
 272         # in-scope, for use to not overwrite them with auto-assign
 273         self.fnparm_vars = set()
 274         for arg in args:
 275             print("adding fn parm", arg)
 276             self.fnparm_vars.add(arg)
 277
 278     # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] |
 279     # '**' NAME) |
 280     # highly simplified
 281
 282     def p_varargslist(self, p):
 283         """varargslist : varargslist COMMA NAME
 284                        | NAME"""
 285         if len(p) == 4:
 286             print(p[1], p[3])
 287             p[0] = p[1] + [p[3]]
 288         else:
 289             p[0] = [p[1]]
 290
 291     # stmt: simple_stmt | compound_stmt
 292     def p_stmt_simple(self, p):
 293         """stmt : simple_stmt"""
 294         # simple_stmt is a list
 295         p[0] = p[1]
 296
 297     def p_stmt_compound(self, p):
 298         """stmt : compound_stmt"""
 299         p[0] = [p[1]]
 300
 301     # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 302     def p_simple_stmt(self, p):
 303         """simple_stmt : small_stmts NEWLINE
 304                        | small_stmts SEMICOLON NEWLINE"""
 305         p[0] = p[1]
 306
 307     def p_small_stmts(self, p):
 308         """small_stmts : small_stmts SEMICOLON small_stmt
 309                        | small_stmt"""
 310         if len(p) == 4:
 311             p[0] = p[1] + [p[3]]
 312         elif isinstance(p[1], list):
 313             p[0] = p[1]
 314         else:
 315             p[0] = [p[1]]
 316
 317     # small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
 318     #    import_stmt | global_stmt | exec_stmt | assert_stmt
 319     def p_small_stmt(self, p):
 320         """small_stmt : flow_stmt
 321                       | break_stmt
 322                       | expr_stmt"""
 323         if isinstance(p[1], ast.Call):
 324             p[0] = ast.Expr(p[1])
 325         elif isinstance(p[1], ast.Name) and p[1].id not in SPECIAL_HELPERS:
 326             fname = p[1].id
 327             name = ast.Name("self", ast.Load())
 328             name = ast.Attribute(name, fname, ast.Load())
 329             p[0] = ast.Call(name, [], [])
 330         else:
 331             p[0] = p[1]
 332
 333     # expr_stmt: testlist (augassign (yield_expr|testlist) |
 334     #                      ('=' (yield_expr|testlist))*)
 335     # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
 336     #             '<<=' | '>>=' | '**=' | '//=')
 337     def p_expr_stmt(self, p):
 338         """expr_stmt : testlist ASSIGNEA testlist
 339                      | testlist ASSIGN testlist
 340                      | testlist """
 341         print("expr_stmt", p)
 342         if len(p) == 2:
 343             # a list of expressions
 344             #p[0] = ast.Discard(p[1])
 345             p[0] = p[1]
 346         else:
 347             iea_mode = p[2] == '<-iea'
 348             if iea_mode:
 349                 print ("iea_mode")
 350             name = None
 351             autoassign = False
 352             if isinstance(p[1], ast.Name):
 353                 name = p[1].id
 354             elif isinstance(p[1], ast.Subscript):
 355                 print("assign subscript", p[1].value,
 356                       self.declared_vars,
 357                       self.fnparm_vars,
 358                       self.special_regs)
 359                 print(astor.dump_tree(p[1]))
 360                 print(astor.dump_tree(p[3]))
 361                 if isinstance(p[1].value, ast.Name):
 362                     name = p[1].value.id
 363                     print("assign subscript value to name", name)
 364                     if name in self.gprs:
 365                         # add to list of uninitialised
 366                         self.uninit_regs.add(name)
 367                     # work out if this is an ininitialised variable
 368                     # that should be auto-assigned simply by being "sliced"
 369                     autoassign = (name not in self.declared_vars and
 370                                   name not in self.fnparm_vars and
 371                                   name not in self.special_regs)
 372             elif isinstance(p[1], (ast.Attribute, ast.Tuple)):
 373                 pass
 374             elif isinstance(p[1], ast.Call) and p[1].func.id in \
 375                     ['GPR', 'FPR', 'SPR']:
 376                 print(astor.dump_tree(p[1]))
 377                 # replace GPR(x) with GPR[x]
 378                 idx = p[1].args[0].id
 379                 if idx in regs + fregs:
 380                     ridx = ast.Name("_%s" % idx, ast.Load())
 381                 else:
 382                     ridx = ast.Name(idx, ast.Load())
 383                 p[1] = ast.Subscript(p[1].func, ridx, ast.Load())
 384                 if idx in self.gprs:
 385                     self.read_regs.add(idx)  # add to list of regs to read
 386             elif isinstance(p[1], ast.Call) and p[1].func.id == 'MEM':
 387                 print("mem assign")
 388                 print(astor.dump_tree(p[1]))
 389                 p[1].func.id = "memassign"  # change function name to set
 390                 p[1].args.append(p[3])
 391                 p[0] = p[1]
 392                 print("mem rewrite")
 393                 print(astor.dump_tree(p[0]))
 394                 return
 395             else:
 396                 print(astor.dump_tree(p[1]))
 397                 raise_syntax_error("help, help", self.filename,
 398                                    p.slice[2].lineno, p.slice[2].lexpos,
 399                                    self.input_text)
 400             print("expr assign", name, p[1], "to", p[3])
 401             print(astor.dump_tree(p[3]))
 402             if isinstance(p[3], ast.Name):
 403                 toname = p[3].id
 404                 if toname in self.gprs:
 405                     self.read_regs.add(toname)
 406             if name and name in self.gprs:
 407                 self.write_regs.add(name)  # add to list of regs to write
 408             p[0] = Assign(autoassign, name, p[1], p[3], iea_mode)
 409             if name:
 410                 self.declared_vars.add(name)
 411
 412     def p_flow_stmt(self, p):
 413         "flow_stmt : return_stmt"
 414         p[0] = p[1]
 415
 416     # return_stmt: 'return' [testlist]
 417     def p_return_stmt(self, p):
 418         "return_stmt : RETURN testlist"
 419         p[0] = ast.Return(p[2])
 420
 421     def p_compound_stmt(self, p):
 422         """compound_stmt : if_stmt
 423                          | while_stmt
 424                          | switch_stmt
 425                          | for_stmt
 426                          | funcdef
 427         """
 428         p[0] = p[1]
 429
 430     def p_break_stmt(self, p):
 431         """break_stmt : BREAK
 432         """
 433         p[0] = ast.Break()
 434
 435     def p_for_stmt(self, p):
 436         """for_stmt : FOR atom EQ comparison TO comparison COLON suite
 437                     | DO atom EQ comparison TO comparison COLON suite
 438         """
 439         start = p[4]
 440         end = p[6]
 441         it = ast.Call(ast.Name("RANGE", ast.Load()), (start, end), [])
 442         p[0] = ast.For(p[2], it, p[8], [])
 443
 444     def p_while_stmt(self, p):
 445         """while_stmt : DO WHILE test COLON suite ELSE COLON suite
 446                       | DO WHILE test COLON suite
 447         """
 448         if len(p) == 6:
 449             p[0] = ast.While(p[3], p[5], [])
 450         else:
 451             p[0] = ast.While(p[3], p[5], p[8])
 452
 453     def p_switch_smt(self, p):
 454         """switch_stmt : SWITCH LPAR atom RPAR COLON NEWLINE INDENT switches DEDENT
 455         """
 456         switchon = p[3]
 457         print("switch stmt")
 458         print(astor.dump_tree(p[1]))
 459
 460         cases = []
 461         current_cases = []  # for deferral
 462         for (case, suite) in p[8]:
 463             print("for", case, suite)
 464             if suite is None:
 465                 for c in case:
 466                     current_cases.append(ast.Num(c))
 467                 continue
 468             if case == 'default':  # last
 469                 break
 470             for c in case:
 471                 current_cases.append(ast.Num(c))
 472             print("cases", current_cases)
 473             compare = ast.Compare(switchon, [ast.In()],
 474                                   [ast.List(current_cases, ast.Load())])
 475             current_cases = []
 476             cases.append((compare, suite))
 477
 478         print("ended", case, current_cases)
 479         if case == 'default':
 480             if current_cases:
 481                 compare = ast.Compare(switchon, [ast.In()],
 482                                       [ast.List(current_cases, ast.Load())])
 483                 cases.append((compare, suite))
 484             cases.append((None, suite))
 485
 486         cases.reverse()
 487         res = []
 488         for compare, suite in cases:
 489             print("after rev", compare, suite)
 490             if compare is None:
 491                 assert len(res) == 0, "last case should be default"
 492                 res = suite
 493             else:
 494                 if not isinstance(res, list):
 495                     res = [res]
 496                 res = ast.If(compare, suite, res)
 497         p[0] = res
 498
 499     def p_switches(self, p):
 500         """switches : switch_list switch_default
 501                     | switch_default
 502         """
 503         if len(p) == 3:
 504             p[0] = p[1] + [p[2]]
 505         else:
 506             p[0] = [p[1]]
 507
 508     def p_switch_list(self, p):
 509         """switch_list : switch_case switch_list
 510                        | switch_case
 511         """
 512         if len(p) == 3:
 513             p[0] = [p[1]] + p[2]
 514         else:
 515             p[0] = [p[1]]
 516
 517     def p_switch_case(self, p):
 518         """switch_case : CASE LPAR atomlist RPAR COLON suite
 519         """
 520         # XXX bad hack
 521         if isinstance(p[6][0], ast.Name) and p[6][0].id == 'fallthrough':
 522             p[6] = None
 523         p[0] = (p[3], p[6])
 524
 525     def p_switch_default(self, p):
 526         """switch_default : DEFAULT COLON suite
 527         """
 528         p[0] = ('default', p[3])
 529
 530     def p_atomlist(self, p):
 531         """atomlist : atom COMMA atomlist
 532                     | atom
 533         """
 534         assert isinstance(p[1], ast.Constant), "case must be numbers"
 535         if len(p) == 4:
 536             p[0] = [p[1].value] + p[3]
 537         else:
 538             p[0] = [p[1].value]
 539
 540     def p_if_stmt(self, p):
 541         """if_stmt : IF test COLON suite ELSE COLON if_stmt
 542                    | IF test COLON suite ELSE COLON suite
 543                    | IF test COLON suite
 544         """
 545         if len(p) == 8 and isinstance(p[7], ast.If):
 546             p[0] = ast.If(p[2], p[4], [p[7]])
 547         elif len(p) == 5:
 548             p[0] = ast.If(p[2], p[4], [])
 549         else:
 550             p[0] = ast.If(p[2], p[4], p[7])
 551
 552     def p_suite(self, p):
 553         """suite : simple_stmt
 554                  | NEWLINE INDENT stmts DEDENT"""
 555         if len(p) == 2:
 556             p[0] = p[1]
 557         else:
 558             p[0] = p[3]
 559
 560     def p_stmts(self, p):
 561         """stmts : stmts stmt
 562                  | stmt"""
 563         if len(p) == 3:
 564             p[0] = p[1] + p[2]
 565         else:
 566             p[0] = p[1]
 567
 568     def p_comparison(self, p):
 569         """comparison : comparison PLUS comparison
 570                       | comparison MINUS comparison
 571                       | comparison MULT comparison
 572                       | comparison DIV comparison
 573                       | comparison MOD comparison
 574                       | comparison EQ comparison
 575                       | comparison NE comparison
 576                       | comparison LE comparison
 577                       | comparison GE comparison
 578                       | comparison LTU comparison
 579                       | comparison GTU comparison
 580                       | comparison LT comparison
 581                       | comparison GT comparison
 582                       | comparison BITOR comparison
 583                       | comparison BITXOR comparison
 584                       | comparison BITAND comparison
 585                       | PLUS comparison
 586                       | MINUS comparison
 587                       | INVERT comparison
 588                       | comparison APPEND comparison
 589                       | power"""
 590         if len(p) == 4:
 591             print(list(p))
 592             if p[2] == '<u':
 593                 p[0] = ast.Call(ast.Name("ltu", ast.Load()), (p[1], p[3]), [])
 594             elif p[2] == '>u':
 595                 p[0] = ast.Call(ast.Name("gtu", ast.Load()), (p[1], p[3]), [])
 596             elif p[2] == '||':
 597                 l = check_concat(p[1]) + check_concat(p[3])
 598                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, [])
 599             elif p[2] in ['/', '%']:
 600                 # bad hack: if % or / used anywhere other than div/mod ops,
 601                 # do % or /.  however if the argument names are "dividend"
 602                 # we must call the special trunc_divs and trunc_rems functions
 603                 l, r = p[1], p[3]
 604                 # actual call will be "dividend / divisor" - just check
 605                 # LHS name
 606                 # XXX DISABLE BAD HACK (False)
 607                 if False and isinstance(l, ast.Name) and l.id == 'dividend':
 608                     if p[2] == '/':
 609                         fn = 'trunc_divs'
 610                     else:
 611                         fn = 'trunc_rems'
 612                     # return "function trunc_xxx(l, r)"
 613                     p[0] = ast.Call(ast.Name(fn, ast.Load()), (l, r), [])
 614                 else:
 615                     # return "l {binop} r"
 616                     p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 617             elif p[2] in ['<', '>', '=', '<=', '>=', '!=']:
 618                 p[0] = binary_ops[p[2]]((p[1], p[3]))
 619             elif identify_sint_mul_pattern(p):
 620                 keywords = [ast.keyword(arg='repeat', value=p[3])]
 621                 l = p[1].elts
 622                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 623             else:
 624                 p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 625                 # HORRENDOUS hack, add brackets around the bin-op by
 626                 # creating a function call with a *blank* function name!
 627                 # XXX argh doesn't work because of analysis of
 628                 # identify_sint_pattern
 629                 #p[0] = ast.Call(ast.Name("", ast.Load()), [p[0]], [])
 630         elif len(p) == 3:
 631             if isinstance(p[2], str) and p[2] == '-':
 632                 p[0] = ast.UnaryOp(unary_ops[p[2]], p[1])
 633             else:
 634                 p[0] = ast.UnaryOp(unary_ops[p[1]], p[2])
 635         else:
 636             p[0] = p[1]
 637
 638     # power: atom trailer* ['**' factor]
 639     # trailers enables function calls (and subscripts).
 640     # so this is 'trailerlist'
 641     def p_power(self, p):
 642         """power : atom
 643                  | atom trailerlist"""
 644         if len(p) == 2:
 645             print("power dump atom notrailer")
 646             print(astor.dump_tree(p[1]))
 647             p[0] = p[1]
 648         else:
 649             print("power dump atom")
 650             print(astor.dump_tree(p[1]))
 651             print("power dump trailerlist")
 652             print(astor.dump_tree(p[2]))
 653             p[0] = self.apply_trailer(p[1], p[2], self.read_regs)
 654             if isinstance(p[1], ast.Name):
 655                 name = p[1].id
 656                 if name in regs + fregs:
 657                     self.read_regs.add(name)
 658
 659     def p_atom_name(self, p):
 660         """atom : NAME"""
 661         name = p[1]
 662         if name in self.available_op_fields:
 663             self.op_fields.add(name)
 664         if name in ['overflow', 'CR0']:
 665             self.write_regs.add(name)
 666         if self.include_ca_in_write:
 667             if name in ['CA', 'CA32']:
 668                 self.write_regs.add(name)
 669         if name in ['CR', 'LR', 'CTR', 'TAR', 'FPSCR', 'MSR',
 670                     'SVSTATE', 'SVREMAP', 'SRR0', 'SRR1',
 671                     'SVSHAPE0', 'SVSHAPE1', 'SVSHAPE2', 'SVSHAPE3']:
 672             self.special_regs.add(name)
 673             self.write_regs.add(name)  # and add to list to write
 674         if name in ('XLEN', ) or name in BFP_FLAG_NAMES:
 675             attr = ast.Name("self", ast.Load())
 676             p[0] = ast.Attribute(attr, name, ast.Load(), lineno=p.lineno(1))
 677         else:
 678             p[0] = ast.Name(id=name, ctx=ast.Load(), lineno=p.lineno(1))
 679
 680     def p_atom_number(self, p):
 681         """atom : BINARY
 682                 | NUMBER
 683                 | HEX
 684                 | STRING"""
 685         p[0] = ast.Constant(p[1])
 686
 687     # '[' [listmaker] ']' |
 688
 689     def p_atom_listmaker(self, p):
 690         """atom : LBRACK listmaker RBRACK"""
 691         p[0] = p[2]
 692
 693     def p_listmaker(self, p):
 694         """listmaker : test COMMA listmaker
 695                      | test
 696         """
 697         if len(p) == 2:
 698             p[0] = ast.List([p[1]], ast.Load())
 699         else:
 700             p[0] = ast.List([p[1]] + p[3].nodes, ast.Load())
 701
 702     def p_atom_tuple(self, p):
 703         """atom : LPAR testlist RPAR"""
 704         print("tuple", p[2])
 705         print("astor dump")
 706         print(astor.dump_tree(p[2]))
 707
 708         if isinstance(p[2], ast.Name):
 709             name = p[2].id
 710             print("tuple name", name)
 711             if name in self.gprs:
 712                 self.read_regs.add(name)  # add to list of regs to read
 713                 #p[0] = ast.Subscript(ast.Name("GPR", ast.Load()), ast.Str(p[2].id))
 714                 # return
 715             p[0] = p[2]
 716         elif isinstance(p[2], ast.BinOp):
 717             if isinstance(p[2].left, ast.Name) and \
 718                isinstance(p[2].right, ast.Constant) and \
 719                     p[2].right.value == 0 and \
 720                     p[2].left.id in self.gprs:
 721                 rid = p[2].left.id
 722                 self.read_regs.add(rid)  # add to list of regs to read
 723                 # create special call to GPR.getz or FPR.getz
 724                 if rid in fregs:
 725                     gprz = ast.Name("FPR", ast.Load())
 726                 else:
 727                     gprz = ast.Name("GPR", ast.Load())
 728                 # get testzero function
 729                 gprz = ast.Attribute(gprz, "getz", ast.Load())
 730                 # *sigh* see class GPR.  we need index itself not reg value
 731                 ridx = ast.Name("_%s" % rid, ast.Load())
 732                 p[0] = ast.Call(gprz, [ridx], [])
 733                 print("tree", astor.dump_tree(p[0]))
 734             else:
 735                 p[0] = p[2]
 736         else:
 737             p[0] = p[2]
 738
 739     def p_trailerlist(self, p):
 740         """trailerlist : trailer trailerlist
 741                        | trailer
 742         """
 743         if len(p) == 2:
 744             p[0] = p[1]
 745         else:
 746             p[0] = ("TLIST", p[1], p[2])
 747
 748     # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 749     def p_trailer(self, p):
 750         """trailer : trailer_arglist
 751                    | trailer_subscript
 752                    | trailer_attr
 753         """
 754         p[0] = p[1]
 755
 756     def p_trailer_arglist(self, p):
 757         """trailer_arglist : LPAR arglist RPAR
 758                            | LPAR RPAR
 759         """
 760         args = [] if len(p) == 3 else p[2]
 761         p[0] = ("CALL", args, p.slice[1])
 762
 763     def p_trailer_subscript(self, p):
 764         "trailer_subscript : LBRACK subscript RBRACK"
 765         p[0] = ("SUBS", p[2])
 766
 767     def p_trailer_attr(self, p):
 768         "trailer_attr : PERIOD NAME"
 769         p[0] = ("ATTR", p[2])
 770
 771     # subscript: '.' '.' '.' | test | [test] ':' [test]
 772
 773     def p_subscript(self, p):
 774         """subscript : test COLON test
 775                      | test
 776         """
 777         if len(p) == 4:
 778             # add one to end
 779             if isinstance(p[3], ast.Constant):
 780                 end = ast.Constant(p[3].value+1)
 781             else:
 782                 end = ast.BinOp(p[3], ast.Add(), ast.Constant(1))
 783             p[0] = [p[1], end]
 784         else:
 785             p[0] = [p[1]]
 786
 787     # testlist: test (',' test)* [',']
 788     # Contains shift/reduce error
 789
 790     def p_testlist(self, p):
 791         """testlist : testlist_multi COMMA
 792                     | testlist_multi """
 793         if len(p) == 2:
 794             p[0] = p[1]
 795         else:
 796             # May need to promote singleton to tuple
 797             if isinstance(p[1], list):
 798                 p[0] = p[1]
 799             else:
 800                 p[0] = [p[1]]
 801         # Convert into a tuple?
 802         if isinstance(p[0], list):
 803             p[0] = ast.Tuple(p[0])
 804
 805     def p_testlist_multi(self, p):
 806         """testlist_multi : testlist_multi COMMA test
 807                           | test"""
 808         if len(p) == 2:
 809             # singleton
 810             p[0] = p[1]
 811         else:
 812             if isinstance(p[1], list):
 813                 p[0] = p[1] + [p[3]]
 814             else:
 815                 # singleton -> tuple
 816                 p[0] = [p[1], p[3]]
 817
 818     # test: or_test ['if' or_test 'else' test] | lambdef
 819     #  as I don't support 'and', 'or', and 'not' this works down to 'comparison'
 820
 821     def p_test(self, p):
 822         """test : comparison
 823                 | comparison QMARK test COLON test"""
 824         if len(p) == 2:
 825             p[0] = p[1]
 826         else:
 827             p[0] = ast.IfExp(test=p[1], body=p[3], orelse=p[5])
 828
 829     # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
 830     # | '**' test)
 831     # XXX INCOMPLETE: this doesn't allow the trailing comma
 832
 833     def p_arglist(self, p):
 834         """arglist : arglist COMMA argument
 835                    | argument"""
 836         if len(p) == 4:
 837             p[0] = p[1] + [p[3]]
 838         else:
 839             p[0] = [p[1]]
 840
 841     # argument: test [gen_for] | test '=' test  # Really [keyword '='] test
 842     def p_argument(self, p):
 843         "argument : test"
 844         p[0] = p[1]
 845
 846     def p_error(self, p):
 847         # print "Error!", repr(p)
 848         raise_syntax_error(str(p), self.filename, p.lineno, p.lexpos,
 849                            self.input_text)
 850
 851     def Assign(self, autoassign, assignname, left, right, iea_mode, eq_tok):
 852         names = []
 853         print("Assign", autoassign, assignname, left, right)
 854         if isinstance(left, ast.Name):
 855             # Single assignment on left
 856             # XXX when doing IntClass, which will have an "eq" function,
 857             # this is how to access it
 858             #   eq = ast.Attribute(left, "eq")   # get eq fn
 859             #   return ast.Call(eq, [right], []) # now call left.eq(right)
 860             return ast.Assign([ast.Name(left.id, ast.Store())], right)
 861         elif isinstance(left, ast.Tuple):
 862             # List of things - make sure they are Name nodes
 863             names = []
 864             for child in left.elts:
 865                 if not isinstance(child, ast.Name):
 866                     raise_syntax_error(
 867                         "that assignment not supported", self.filename,
 868                         eq_tok.lineno, eq_tok.lexpos, self.input_text)
 869                 names.append(child.id)
 870             ass_list = [ast.Name(name, ast.Store()) for name in names]
 871             return ast.Assign([ast.Tuple(ass_list)], right)
 872         elif isinstance(left, ast.Attribute):
 873             return ast.Assign([
 874                 ast.Attribute(left.value, left.attr, ast.Store())], right)
 875         elif isinstance(left, ast.Subscript):
 876             ls = left.slice
 877             # XXX changing meaning of "undefined" to a function
 878             # if (isinstance(ls, ast.Slice) and isinstance(right, ast.Name) and
 879             #        right.id == 'undefined'):
 880             #    # undefined needs to be copied the exact same slice
 881             #    right = ast.Subscript(right, ls, ast.Load())
 882             #    return ast.Assign([left], right)
 883             res = ast.Assign([left], right)
 884             if autoassign and isinstance(ls, ast.Slice):
 885                 # hack to create a variable pre-declared based on a slice.
 886                 # dividend[0:32] = (RA)[0:32] will create
 887                 #       dividend = [0] * 32
 888                 #       dividend[0:32] = (RA)[0:32]
 889                 # the declaration makes the slice-assignment "work"
 890                 lower, upper, step = ls.lower, ls.upper, ls.step
 891                 print("lower, upper, step", repr(lower), repr(upper), step)
 892                 # XXX relax constraint that indices on auto-assign have
 893                 # to be constants x[0:32]
 894                 # if not isinstance(lower, ast.Constant) or \
 895                 #   not isinstance(upper, ast.Constant):
 896                 #    return res
 897                 qty = ast.BinOp(upper, binary_ops['-'], lower)
 898                 keywords = [ast.keyword(arg='repeat', value=qty)]
 899                 l = [ast.Num(0)]
 900                 right = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 901                 declare = ast.Assign(
 902                     [ast.Name(assignname, ast.Store())], right)
 903                 return [declare, res]
 904             return res
 905             # XXX HMMM probably not needed...
 906             ls = left.slice
 907             if isinstance(ls, ast.Slice):
 908                 lower, upper, step = ls.lower, ls.upper, ls.step
 909                 print("slice assign", lower, upper, step)
 910                 if step is None:
 911                     ls = (lower, upper, None)
 912                 else:
 913                     ls = (lower, upper, step)
 914                 ls = ast.Tuple(ls)
 915             return ast.Call(ast.Name("selectassign", ast.Load()),
 916                             [left.value, ls, right], [])
 917         else:
 918             print("Assign fail")
 919             raise_syntax_error("Can't do that yet", self.filename,
 920                                eq_tok.lineno, eq_tok.lexpos, self.input_text)
 921
 922     def try_extract_name(self, atom):
 923         if isinstance(atom, ast.Attribute):
 924             if isinstance(atom.value, ast.Name) and atom.value.id == "self":
 925                 return atom.attr
 926             return None
 927         if isinstance(atom, ast.Name):
 928             return atom.id
 929         return None
 930
 931     def try_extract_uppercase_name(self, atom):
 932         name = self.try_extract_name(atom)
 933         if name is None:
 934             return None
 935         # we want to filter out names like _123 that have no letters with
 936         # casing at all, hence the lower() check
 937         if name.upper() != name or name.lower() == name:
 938             return None
 939         return name
 940
 941     def try_get_attr(self, atom, trailer):
 942         if trailer[0] == "ATTR":
 943             return trailer[1]
 944         if trailer[0] != "SUBS":
 945             return None
 946         base_name = self.try_extract_uppercase_name(atom)
 947         if base_name is None or base_name in SUBS_TO_ATTR_EXCEPTIONS:
 948             return None
 949         if len(trailer[1]) != 1:
 950             return None
 951         return self.try_extract_uppercase_name(trailer[1][0])
 952
 953     def apply_trailer(self, atom, trailer, read_regs):
 954         if trailer[0] == "TLIST":
 955             # assume depth of one
 956             atom = self.apply_trailer(atom, trailer[1], read_regs)
 957             trailer = trailer[2]
 958         if trailer[0] == "CALL":
 959             #p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
 960             for arg in trailer[1]:
 961                 if isinstance(arg, ast.Name):
 962                     name = arg.id
 963                     if name in regs + fregs:
 964                         read_regs.add(name)
 965             if atom.id == "SetFX":
 966                 if len(trailer[1]) != 1 or \
 967                         not isinstance(trailer[1][0], ast.Attribute):
 968                     raise_syntax_error(
 969                         "SetFX call must have only one argument that is an "
 970                         "attribute access -- like `SetFX(FPSCR.VXSNAN)`",
 971                         self.filename, trailer[2].lineno, trailer[2].lexpos,
 972                         self.input_text)
 973                 arg = trailer[1][0]
 974                 args = [arg.value, ast.Str(arg.attr)]
 975                 name = ast.Name("self", ast.Load())
 976                 atom = ast.Attribute(name, atom, ast.Load())
 977                 return ast.Call(atom, args, [])
 978             # special-case, these functions must NOT be made "self.xxxx"
 979             if atom.id not in SPECIAL_HELPERS:
 980                 name = ast.Name("self", ast.Load())
 981                 atom = ast.Attribute(name, atom, ast.Load())
 982             return ast.Call(atom, trailer[1], [])
 983             # if p[1].id == 'print':
 984             #    p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
 985             # else:
 986             #    p[0] = ast.CallFunc(p[1], p[2][1], None, None)
 987         else:
 988             attr = self.try_get_attr(atom, trailer)
 989             if attr is not None:
 990                 return ast.Attribute(atom, attr, ast.Load())
 991             assert trailer[0] == "SUBS"
 992             print("subscript atom", trailer[1])
 993             #raise AssertionError("not implemented %s" % p[2][0])
 994             subs = trailer[1]
 995             if len(subs) == 1:
 996                 idx = subs[0]
 997                 if isinstance(idx, ast.Name) and idx.id in regs + fregs:
 998                     read_regs.add(idx.id)
 999                 if isinstance(idx, ast.Name) and idx.id in regs:
1000                     print("single atom subscript, underscored", idx.id)
1001                     idx = ast.Name("_%s" % idx.id, ast.Load())
1002             else:
1003                 idx = ast.Slice(subs[0], subs[1], None)
1004             # if isinstance(atom, ast.Name) and atom.id == 'CR':
1005                 # atom.id = 'CR' # bad hack
1006                 #print ("apply_trailer Subscript", atom.id, idx)
1007             return ast.Subscript(atom, idx, ast.Load())
1008
1009
1010 _CACHE_DECODER = True
1011 _CACHED_DECODER = None
1012
1013
1014 def _create_cached_decoder():
1015     global _CACHED_DECODER
1016     if _CACHE_DECODER:
1017         if _CACHED_DECODER is None:
1018             _CACHED_DECODER = create_pdecode()
1019         return _CACHED_DECODER
1020     return create_pdecode()
1021
1022
1023 class GardenSnakeParser(PowerParser):
1024     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1025         if form is not None:
1026             self.sd = _create_cached_decoder()
1027         PowerParser.__init__(self, form, incl_carry, helper=helper)
1028         self.debug = debug
1029         self.lexer = IndentLexer(debug=0)
1030         self.tokens = self.lexer.tokens
1031         self.parser = yacc.yacc(module=self, start="file_input_end",
1032                                 debug=debug, write_tables=False)
1033
1034     def parse(self, code):
1035         self.reset()
1036         self.lexer.filename = self.filename
1037         result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
1038         if self.helper:
1039             result = [ast.ClassDef("ISACallerFnHelper", [
1040                                    "ISACallerHelper"], [], result, decorator_list=[])]
1041         return ast.Module(result)
1042
1043
1044 ###### Code generation ######
1045
1046 #from compiler import misc, syntax, pycodegen
1047
1048 _CACHED_PARSERS = {}
1049 _CACHE_PARSERS = True
1050
1051
1052 class GardenSnakeCompiler(object):
1053     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1054         if _CACHE_PARSERS:
1055             try:
1056                 self.parser = _CACHED_PARSERS[debug, form, incl_carry, helper]
1057             except KeyError:
1058                 self.parser = GardenSnakeParser(
1059                     debug=debug, form=form, incl_carry=incl_carry,
1060                     helper=helper)
1061                 _CACHED_PARSERS[debug, form, incl_carry, helper] = self.parser
1062         else:
1063             self.parser = GardenSnakeParser(debug=debug, form=form,
1064                                             incl_carry=incl_carry, helper=helper)
1065
1066     def compile(self, code, mode="exec", filename="<string>"):
1067         if filename in ["string", "<string>"]:
1068             raise ValueError("missing filename")
1069         self.parser.filename = filename
1070         self.parser.input_text = code
1071         try:
1072             tree = self.parser.parse(code)
1073         except SyntaxError2 as e:
1074             e.raise_syntax_error()
1075         print("snake")
1076         pprint(tree)
1077         return tree
1078         #misc.set_filename(filename, tree)
1079         return compile(tree, mode="exec", filename="<string>")
1080         # syntax.check(tree)
1081         gen = pycodegen.ModuleCodeGenerator(tree)
1082         code = gen.getCode()
1083         return code