src/openpower/decoder/pseudo/parser.py

   1 # Based on GardenSnake - a parser generator demonstration program
   2 # GardenSnake was released into the Public Domain by Andrew Dalke.
   3
   4 # Portions of this work are derived from Python's Grammar definition
   5 # and may be covered under the Python copyright and license
   6 #
   7 #          Andrew Dalke / Dalke Scientific Software, LLC
   8 #             30 August 2006 / Cape Town, South Africa
   9
  10 # Modifications for inclusion in PLY distribution
  11 from pprint import pprint
  12 from ply import lex, yacc
  13 import astor
  14 from copy import deepcopy
  15
  16 from openpower.decoder.power_decoder import create_pdecode
  17 from openpower.decoder.pseudo.lexer import (
  18     IndentLexer, raise_syntax_error, SyntaxError2)
  19 from openpower.decoder.orderedset import OrderedSet
  20 from openpower.decoder.power_enums import BFP_FLAG_NAMES
  21
  22 # I use the Python AST
  23 #from compiler import ast
  24 import ast
  25
  26 # Helper function
  27
  28 regs = ['RA', 'RS', 'RB', 'RC', 'RT']
  29 fregs = ['FRA', 'FRS', 'FRB', 'FRC', 'FRT', 'FRS']
  30 SPECIAL_HELPERS = {'concat', 'MEM', 'GPR', 'FPR', 'SPR', 'pow'}
  31 SUBS_TO_ATTR_EXCEPTIONS = SPECIAL_HELPERS | {'CR'}
  32
  33
  34 # I implemented INDENT / DEDENT generation as a post-processing filter
  35
  36 # The original lex token stream contains WS and NEWLINE characters.
  37 # WS will only occur before any other tokens on a line.
  38
  39 # I have three filters.  One tags tokens by adding two attributes.
  40 # "must_indent" is True if the token must be indented from the
  41 # previous code.  The other is "at_line_start" which is True for WS
  42 # and the first non-WS/non-NEWLINE on a line.  It flags the check so
  43 # see if the new line has changed indication level.
  44
  45
  46 # No using Python's approach because Ply supports precedence
  47
  48 # comparison: expr (comp_op expr)*
  49 # arith_expr: term (('+'|'-') term)*
  50 # term: factor (('*'|'/'|'%'|'//') factor)*
  51 # factor: ('+'|'-'|'~') factor | power
  52 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
  53
  54 def make_le_compare(arg):
  55     (left, right) = arg
  56     return ast.Call(ast.Name("le", ast.Load()), (left, right), [])
  57
  58
  59 def make_ge_compare(arg):
  60     (left, right) = arg
  61     return ast.Call(ast.Name("ge", ast.Load()), (left, right), [])
  62
  63
  64 def make_lt_compare(arg):
  65     (left, right) = arg
  66     return ast.Call(ast.Name("lt", ast.Load()), (left, right), [])
  67
  68
  69 def make_gt_compare(arg):
  70     (left, right) = arg
  71     return ast.Call(ast.Name("gt", ast.Load()), (left, right), [])
  72
  73
  74 def make_eq_compare(arg):
  75     (left, right) = arg
  76     return ast.Call(ast.Name("eq", ast.Load()), (left, right), [])
  77
  78
  79 def make_ne_compare(arg):
  80     (left, right) = arg
  81     return ast.Call(ast.Name("ne", ast.Load()), (left, right), [])
  82
  83
  84 binary_ops = {
  85     "^": ast.BitXor(),
  86     "&": ast.BitAnd(),
  87     "|": ast.BitOr(),
  88     "+": ast.Add(),
  89     "-": ast.Sub(),
  90     "*": ast.Mult(),
  91     "/": ast.FloorDiv(),
  92     "%": ast.Mod(),
  93     "<=": make_le_compare,
  94     ">=": make_ge_compare,
  95     "<": make_lt_compare,
  96     ">": make_gt_compare,
  97     "=": make_eq_compare,
  98     "!=": make_ne_compare,
  99 }
 100 unary_ops = {
 101     "+": ast.UAdd(),
 102     "-": ast.USub(),
 103     "¬": ast.Invert(),
 104 }
 105
 106
 107 def check_concat(node):  # checks if the comparison is already a concat
 108     print("check concat", node)
 109     if not isinstance(node, ast.Call):
 110         return [node]
 111     node_func_id = getattr(node.func, "id", None)
 112     print("func", node_func_id)
 113     if node_func_id != 'concat':
 114         return [node]
 115     if node.keywords:  # a repeated list-constant, don't optimise
 116         return [node]
 117     return node.args
 118
 119
 120 # identify SelectableInt pattern [something] * N
 121 # must return concat(something, repeat=N)
 122 # here is a TEMPORARY hack to support minimal expressions
 123 # such as (XLEN-16), looking for ast.BinOp
 124 # have to keep an eye on this
 125 def identify_sint_mul_pattern(p):
 126     """here we are looking for patterns of this type:
 127         [item] * something
 128     these must specifically be returned as concat(item, repeat=something)
 129     """
 130     #print ("identify_sint_mul_pattern")
 131     # for pat in p:
 132     #    print("    ", astor.dump_tree(pat))
 133     if p[2] != '*':  # multiply
 134         return False
 135     if (not isinstance(p[3], ast.Constant) and  # rhs = Num
 136         not isinstance(p[3], ast.BinOp) and     # rhs = (XLEN-something)
 137             (not isinstance(p[3], ast.Name) and  # rhs = {a variable}
 138              not isinstance(p[3], ast.Attribute))):   # rhs = XLEN
 139         return False
 140     if not isinstance(p[1], ast.List):  # lhs is a list
 141         return False
 142     l = p[1].elts
 143     if len(l) != 1:  # lhs is a list of length 1
 144         return False
 145     return True  # yippee!
 146
 147
 148 ##########   Parser (tokens -> AST) ######
 149
 150 # also part of Ply
 151 #import yacc
 152
 153 # https://www.mathcs.emory.edu/~valerie/courses/fall10/155/resources/op_precedence.html
 154 # python operator precedence
 155 # Highest precedence at top, lowest at bottom.
 156 # Operators in the same box evaluate left to right.
 157 #
 158 # Operator Description
 159 # ()                                                     Parentheses (grouping)
 160 # f(args...)                                             Function call
 161 # x[index:index]                                         Slicing
 162 # x[index]                                               Subscription
 163 # x.attribute                                            Attribute reference
 164 # **                                                     Exponentiation
 165 # ~x                                                     Bitwise not
 166 # +x, -x                                                 Positive, negative
 167 # *, /, %                                                mul, div, remainder
 168 # +, -                                                   Addition, subtraction
 169 # <<, >>                                                 Bitwise shifts
 170 # &                                                      Bitwise AND
 171 # ^                                                      Bitwise XOR
 172 # |                                                      Bitwise OR
 173 # in, not in, is, is not, <, <=,  >,  >=, <>, !=, ==     comp, membership, ident
 174 # not x                                                  Boolean NOT
 175 # and                                                    Boolean AND
 176 # or                                                     Boolean OR
 177 # lambda                                                 Lambda expression
 178
 179
 180 class PowerParser:
 181
 182     precedence = (
 183         ("left", "EQ", "NE", "GT", "LT", "LE", "GE", "LTU", "GTU"),
 184         ("left", "BITOR"),
 185         ("left", "BITXOR"),
 186         ("left", "BITAND"),
 187         ("left", "PLUS", "MINUS"),
 188         ("left", "MULT", "DIV", "MOD"),
 189         ("left", "INVERT"),
 190     )
 191
 192     def reset(self):
 193         self.gprs = {}
 194         if self.form is not None:
 195             form = self.sd.sigforms[self.form]
 196             print(form)
 197             formkeys = form._asdict().keys()
 198         else:
 199             formkeys = []
 200         self.declared_vars = set()
 201         self.fnparm_vars = set()
 202         for rname in regs + fregs:
 203             self.gprs[rname] = None
 204             self.declared_vars.add(rname)
 205         self.available_op_fields = set()
 206         for k in formkeys:
 207             if k not in self.gprs:
 208                 if k == 'SPR':  # sigh, lower-case to not conflict
 209                     k = k.lower()
 210                 self.available_op_fields.add(k)
 211         self.op_fields = OrderedSet()
 212         self.read_regs = OrderedSet()
 213         self.uninit_regs = OrderedSet()
 214         self.write_regs = OrderedSet()
 215         self.special_regs = OrderedSet()  # see p_atom_name
 216
 217     def __init__(self, form, include_carry_in_write=False, helper=False):
 218         self.include_ca_in_write = include_carry_in_write
 219         self.helper = helper
 220         self.form = form
 221         self.filename = None
 222         self.input_text = None
 223         self.reset()
 224
 225     # The grammar comments come from Python's Grammar/Grammar file
 226
 227     # NB: compound_stmt in single_input is followed by extra NEWLINE!
 228     # file_input: (NEWLINE | stmt)* ENDMARKER
 229
 230     def p_file_input_end(self, p):
 231         """file_input_end : file_input ENDMARKER"""
 232         print("end", p[1])
 233         p[0] = p[1]
 234
 235     def p_file_input(self, p):
 236         """file_input : file_input NEWLINE
 237                       | file_input stmt
 238                       | NEWLINE
 239                       | stmt"""
 240         if isinstance(p[len(p)-1], str):
 241             if len(p) == 3:
 242                 p[0] = p[1]
 243             else:
 244                 p[0] = []  # p == 2 --> only a blank line
 245         else:
 246             if len(p) == 3:
 247                 p[0] = p[1] + p[2]
 248             else:
 249                 p[0] = p[1]
 250
 251     # funcdef: [decorators] 'def' NAME parameters ':' suite
 252     # ignoring decorators
 253
 254     def p_funcdef(self, p):
 255         "funcdef : DEF NAME parameters COLON suite"
 256         p[0] = ast.FunctionDef(p[2], p[3], p[5], (), lineno=p.lineno(2))
 257         # reset function parameters after suite is identified
 258         self.fnparm_vars = set()
 259
 260     # parameters: '(' [varargslist] ')'
 261     def p_parameters(self, p):
 262         """parameters : LPAR RPAR
 263                       | LPAR varargslist RPAR"""
 264         args = []
 265         if self.helper:
 266             args.append("self")
 267         if len(p) != 3:
 268             args += p[2]
 269         p[0] = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[])
 270         # during the time between parameters identified and suite is not
 271         # there is a window of opportunity to declare the function parameters
 272         # in-scope, for use to not overwrite them with auto-assign
 273         self.fnparm_vars = set()
 274         for arg in args:
 275             print("adding fn parm", arg)
 276             self.fnparm_vars.add(arg)
 277
 278     # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] |
 279     # '**' NAME) |
 280     # highly simplified
 281
 282     def p_varargslist(self, p):
 283         """varargslist : varargslist COMMA NAME
 284                        | NAME"""
 285         if len(p) == 4:
 286             print(p[1], p[3])
 287             p[0] = p[1] + [p[3]]
 288         else:
 289             p[0] = [p[1]]
 290
 291     # stmt: simple_stmt | compound_stmt
 292     def p_stmt_simple(self, p):
 293         """stmt : simple_stmt"""
 294         # simple_stmt is a list
 295         p[0] = p[1]
 296
 297     def p_stmt_compound(self, p):
 298         """stmt : compound_stmt"""
 299         p[0] = [p[1]]
 300
 301     # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 302     def p_simple_stmt(self, p):
 303         """simple_stmt : small_stmts NEWLINE
 304                        | small_stmts SEMICOLON NEWLINE"""
 305         p[0] = p[1]
 306
 307     def p_small_stmts(self, p):
 308         """small_stmts : small_stmts SEMICOLON small_stmt
 309                        | small_stmt"""
 310         if len(p) == 4:
 311             p[0] = p[1] + [p[3]]
 312         elif isinstance(p[1], list):
 313             p[0] = p[1]
 314         else:
 315             p[0] = [p[1]]
 316
 317     # small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
 318     #    import_stmt | global_stmt | exec_stmt | assert_stmt
 319     def p_small_stmt(self, p):
 320         """small_stmt : flow_stmt
 321                       | break_stmt
 322                       | expr_stmt"""
 323         if isinstance(p[1], ast.Call):
 324             p[0] = ast.Expr(p[1])
 325         elif isinstance(p[1], ast.Name) and p[1].id not in SPECIAL_HELPERS:
 326             fname = p[1].id
 327             name = ast.Name("self", ast.Load())
 328             name = ast.Attribute(name, fname, ast.Load())
 329             p[0] = ast.Call(name, [], [])
 330         else:
 331             p[0] = p[1]
 332
 333     # expr_stmt: testlist (augassign (yield_expr|testlist) |
 334     #                      ('=' (yield_expr|testlist))*)
 335     # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
 336     #             '<<=' | '>>=' | '**=' | '//=')
 337     def p_expr_stmt(self, p):
 338         """expr_stmt : testlist ASSIGNEA testlist
 339                      | testlist ASSIGN testlist
 340                      | testlist """
 341         print("expr_stmt", p)
 342         if len(p) == 2:
 343             # a list of expressions
 344             #p[0] = ast.Discard(p[1])
 345             p[0] = p[1]
 346         else:
 347             iea_mode = p[2] == '<-iea'
 348             name = None
 349             autoassign = False
 350             if isinstance(p[1], ast.Name):
 351                 name = p[1].id
 352             elif isinstance(p[1], ast.Subscript):
 353                 print("assign subscript", p[1].value,
 354                       self.declared_vars,
 355                       self.fnparm_vars,
 356                       self.special_regs)
 357                 print(astor.dump_tree(p[1]))
 358                 if isinstance(p[1].value, ast.Name):
 359                     name = p[1].value.id
 360                     print("assign subscript value to name", name)
 361                     if name in self.gprs:
 362                         # add to list of uninitialised
 363                         self.uninit_regs.add(name)
 364                     # work out if this is an ininitialised variable
 365                     # that should be auto-assigned simply by being "sliced"
 366                     autoassign = (name not in self.declared_vars and
 367                                   name not in self.fnparm_vars and
 368                                   name not in self.special_regs)
 369             elif isinstance(p[1], (ast.Attribute, ast.Tuple)):
 370                 pass
 371             elif isinstance(p[1], ast.Call) and p[1].func.id in \
 372                     ['GPR', 'FPR', 'SPR']:
 373                 print(astor.dump_tree(p[1]))
 374                 # replace GPR(x) with GPR[x]
 375                 idx = p[1].args[0].id
 376                 if idx in regs + fregs:
 377                     ridx = ast.Name("_%s" % idx, ast.Load())
 378                 else:
 379                     ridx = ast.Name(idx, ast.Load())
 380                 p[1] = ast.Subscript(p[1].func, ridx, ast.Load())
 381                 if idx in self.gprs:
 382                     self.read_regs.add(idx)  # add to list of regs to read
 383             elif isinstance(p[1], ast.Call) and p[1].func.id == 'MEM':
 384                 print("mem assign")
 385                 print(astor.dump_tree(p[1]))
 386                 p[1].func.id = "memassign"  # change function name to set
 387                 p[1].args.append(p[3])
 388                 p[0] = p[1]
 389                 print("mem rewrite")
 390                 print(astor.dump_tree(p[0]))
 391                 return
 392             else:
 393                 print(astor.dump_tree(p[1]))
 394                 raise_syntax_error("help, help", self.filename,
 395                                    p.slice[2].lineno, p.slice[2].lexpos,
 396                                    self.input_text)
 397             print("expr assign", name, p[1], "to", p[3])
 398             if isinstance(p[3], ast.Name):
 399                 toname = p[3].id
 400                 if toname in self.gprs:
 401                     self.read_regs.add(toname)
 402             if name and name in self.gprs:
 403                 self.write_regs.add(name)  # add to list of regs to write
 404
 405             # copy rhs -- see openpower.decoder.helpers.copy_assign_rhs()'s
 406             # documentation for why we need this
 407             copy_fn = ast.Name("copy_assign_rhs", ast.Load())
 408             rhs = ast.Call(copy_fn, (p[3],), [])
 409             p[0] = self.Assign(autoassign, name, p[1], rhs, iea_mode,
 410                                p.slice[2])
 411             if name:
 412                 self.declared_vars.add(name)
 413
 414     def p_flow_stmt(self, p):
 415         "flow_stmt : return_stmt"
 416         p[0] = p[1]
 417
 418     # return_stmt: 'return' [testlist]
 419     def p_return_stmt(self, p):
 420         "return_stmt : RETURN testlist"
 421         p[0] = ast.Return(p[2])
 422
 423     def p_compound_stmt(self, p):
 424         """compound_stmt : if_stmt
 425                          | while_stmt
 426                          | switch_stmt
 427                          | for_stmt
 428                          | funcdef
 429         """
 430         p[0] = p[1]
 431
 432     def p_break_stmt(self, p):
 433         """break_stmt : BREAK
 434         """
 435         p[0] = ast.Break()
 436
 437     def p_for_stmt(self, p):
 438         """for_stmt : FOR atom EQ comparison TO comparison COLON suite
 439                     | DO atom EQ comparison TO comparison COLON suite
 440         """
 441         start = p[4]
 442         end = p[6]
 443         it = ast.Call(ast.Name("RANGE", ast.Load()), (start, end), [])
 444         p[0] = ast.For(p[2], it, p[8], [])
 445
 446     def p_while_stmt(self, p):
 447         """while_stmt : DO WHILE test COLON suite ELSE COLON suite
 448                       | DO WHILE test COLON suite
 449         """
 450         if len(p) == 6:
 451             p[0] = ast.While(p[3], p[5], [])
 452         else:
 453             p[0] = ast.While(p[3], p[5], p[8])
 454
 455     def p_switch_smt(self, p):
 456         """switch_stmt : SWITCH LPAR atom RPAR COLON NEWLINE INDENT switches DEDENT
 457         """
 458         switchon = p[3]
 459         print("switch stmt")
 460         print(astor.dump_tree(p[1]))
 461
 462         cases = []
 463         current_cases = []  # for deferral
 464         for (case, suite) in p[8]:
 465             print("for", case, suite)
 466             if suite is None:
 467                 for c in case:
 468                     current_cases.append(ast.Num(c))
 469                 continue
 470             if case == 'default':  # last
 471                 break
 472             for c in case:
 473                 current_cases.append(ast.Num(c))
 474             print("cases", current_cases)
 475             compare = ast.Compare(switchon, [ast.In()],
 476                                   [ast.List(current_cases, ast.Load())])
 477             current_cases = []
 478             cases.append((compare, suite))
 479
 480         print("ended", case, current_cases)
 481         if case == 'default':
 482             if current_cases:
 483                 compare = ast.Compare(switchon, [ast.In()],
 484                                       [ast.List(current_cases, ast.Load())])
 485                 cases.append((compare, suite))
 486             cases.append((None, suite))
 487
 488         cases.reverse()
 489         res = []
 490         for compare, suite in cases:
 491             print("after rev", compare, suite)
 492             if compare is None:
 493                 assert len(res) == 0, "last case should be default"
 494                 res = suite
 495             else:
 496                 if not isinstance(res, list):
 497                     res = [res]
 498                 res = ast.If(compare, suite, res)
 499         p[0] = res
 500
 501     def p_switches(self, p):
 502         """switches : switch_list switch_default
 503                     | switch_default
 504         """
 505         if len(p) == 3:
 506             p[0] = p[1] + [p[2]]
 507         else:
 508             p[0] = [p[1]]
 509
 510     def p_switch_list(self, p):
 511         """switch_list : switch_case switch_list
 512                        | switch_case
 513         """
 514         if len(p) == 3:
 515             p[0] = [p[1]] + p[2]
 516         else:
 517             p[0] = [p[1]]
 518
 519     def p_switch_case(self, p):
 520         """switch_case : CASE LPAR atomlist RPAR COLON suite
 521         """
 522         # XXX bad hack
 523         if isinstance(p[6][0], ast.Name) and p[6][0].id == 'fallthrough':
 524             p[6] = None
 525         p[0] = (p[3], p[6])
 526
 527     def p_switch_default(self, p):
 528         """switch_default : DEFAULT COLON suite
 529         """
 530         p[0] = ('default', p[3])
 531
 532     def p_atomlist(self, p):
 533         """atomlist : atom COMMA atomlist
 534                     | atom
 535         """
 536         assert isinstance(p[1], ast.Constant), "case must be numbers"
 537         if len(p) == 4:
 538             p[0] = [p[1].value] + p[3]
 539         else:
 540             p[0] = [p[1].value]
 541
 542     def p_if_stmt(self, p):
 543         """if_stmt : IF test COLON suite ELSE COLON if_stmt
 544                    | IF test COLON suite ELSE COLON suite
 545                    | IF test COLON suite
 546         """
 547         if len(p) == 8 and isinstance(p[7], ast.If):
 548             p[0] = ast.If(p[2], p[4], [p[7]])
 549         elif len(p) == 5:
 550             p[0] = ast.If(p[2], p[4], [])
 551         else:
 552             p[0] = ast.If(p[2], p[4], p[7])
 553
 554     def p_suite(self, p):
 555         """suite : simple_stmt
 556                  | NEWLINE INDENT stmts DEDENT"""
 557         if len(p) == 2:
 558             p[0] = p[1]
 559         else:
 560             p[0] = p[3]
 561
 562     def p_stmts(self, p):
 563         """stmts : stmts stmt
 564                  | stmt"""
 565         if len(p) == 3:
 566             p[0] = p[1] + p[2]
 567         else:
 568             p[0] = p[1]
 569
 570     def p_comparison(self, p):
 571         """comparison : comparison PLUS comparison
 572                       | comparison MINUS comparison
 573                       | comparison MULT comparison
 574                       | comparison DIV comparison
 575                       | comparison MOD comparison
 576                       | comparison EQ comparison
 577                       | comparison NE comparison
 578                       | comparison LE comparison
 579                       | comparison GE comparison
 580                       | comparison LTU comparison
 581                       | comparison GTU comparison
 582                       | comparison LT comparison
 583                       | comparison GT comparison
 584                       | comparison BITOR comparison
 585                       | comparison BITXOR comparison
 586                       | comparison BITAND comparison
 587                       | PLUS comparison
 588                       | MINUS comparison
 589                       | INVERT comparison
 590                       | comparison APPEND comparison
 591                       | power"""
 592         if len(p) == 4:
 593             print(list(p))
 594             if p[2] == '<u':
 595                 p[0] = ast.Call(ast.Name("ltu", ast.Load()), (p[1], p[3]), [])
 596             elif p[2] == '>u':
 597                 p[0] = ast.Call(ast.Name("gtu", ast.Load()), (p[1], p[3]), [])
 598             elif p[2] == '||':
 599                 l = check_concat(p[1]) + check_concat(p[3])
 600                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, [])
 601             elif p[2] in ['/', '%']:
 602                 # bad hack: if % or / used anywhere other than div/mod ops,
 603                 # do % or /.  however if the argument names are "dividend"
 604                 # we must call the special trunc_divs and trunc_rems functions
 605                 l, r = p[1], p[3]
 606                 # actual call will be "dividend / divisor" - just check
 607                 # LHS name
 608                 # XXX DISABLE BAD HACK (False)
 609                 if False and isinstance(l, ast.Name) and l.id == 'dividend':
 610                     if p[2] == '/':
 611                         fn = 'trunc_divs'
 612                     else:
 613                         fn = 'trunc_rems'
 614                     # return "function trunc_xxx(l, r)"
 615                     p[0] = ast.Call(ast.Name(fn, ast.Load()), (l, r), [])
 616                 else:
 617                     # return "l {binop} r"
 618                     p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 619             elif p[2] in ['<', '>', '=', '<=', '>=', '!=']:
 620                 p[0] = binary_ops[p[2]]((p[1], p[3]))
 621             elif identify_sint_mul_pattern(p):
 622                 keywords = [ast.keyword(arg='repeat', value=p[3])]
 623                 l = p[1].elts
 624                 p[0] = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 625             else:
 626                 p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
 627                 # HORRENDOUS hack, add brackets around the bin-op by
 628                 # creating a function call with a *blank* function name!
 629                 # XXX argh doesn't work because of analysis of
 630                 # identify_sint_pattern
 631                 #p[0] = ast.Call(ast.Name("", ast.Load()), [p[0]], [])
 632         elif len(p) == 3:
 633             if isinstance(p[2], str) and p[2] == '-':
 634                 p[0] = ast.UnaryOp(unary_ops[p[2]], p[1])
 635             else:
 636                 p[0] = ast.UnaryOp(unary_ops[p[1]], p[2])
 637         else:
 638             p[0] = p[1]
 639
 640     # power: atom trailer* ['**' factor]
 641     # trailers enables function calls (and subscripts).
 642     # so this is 'trailerlist'
 643     def p_power(self, p):
 644         """power : atom
 645                  | atom trailerlist"""
 646         if len(p) == 2:
 647             print("power dump atom notrailer")
 648             print(astor.dump_tree(p[1]))
 649             p[0] = p[1]
 650         else:
 651             print("power dump atom")
 652             print(astor.dump_tree(p[1]))
 653             print("power dump trailerlist")
 654             print(astor.dump_tree(p[2]))
 655             p[0] = self.apply_trailer(p[1], p[2], self.read_regs)
 656             if isinstance(p[1], ast.Name):
 657                 name = p[1].id
 658                 if name in regs + fregs:
 659                     self.read_regs.add(name)
 660
 661     def p_atom_name(self, p):
 662         """atom : NAME"""
 663         name = p[1]
 664         if name in self.available_op_fields:
 665             self.op_fields.add(name)
 666         if name in ['overflow', 'CR0']:
 667             self.write_regs.add(name)
 668         if self.include_ca_in_write:
 669             if name in ['CA', 'CA32']:
 670                 self.write_regs.add(name)
 671         if name in ['CR', 'LR', 'CTR', 'TAR', 'FPSCR', 'MSR',
 672                     'SVSTATE', 'SVREMAP',
 673                     'SVSHAPE0', 'SVSHAPE1', 'SVSHAPE2', 'SVSHAPE3']:
 674             self.special_regs.add(name)
 675             self.write_regs.add(name)  # and add to list to write
 676         if name in ('XLEN', ) or name in BFP_FLAG_NAMES:
 677             attr = ast.Name("self", ast.Load())
 678             p[0] = ast.Attribute(attr, name, ast.Load(), lineno=p.lineno(1))
 679         else:
 680             p[0] = ast.Name(id=name, ctx=ast.Load(), lineno=p.lineno(1))
 681
 682     def p_atom_number(self, p):
 683         """atom : BINARY
 684                 | NUMBER
 685                 | HEX
 686                 | STRING"""
 687         p[0] = ast.Constant(p[1])
 688
 689     # '[' [listmaker] ']' |
 690
 691     def p_atom_listmaker(self, p):
 692         """atom : LBRACK listmaker RBRACK"""
 693         p[0] = p[2]
 694
 695     def p_listmaker(self, p):
 696         """listmaker : test COMMA listmaker
 697                      | test
 698         """
 699         if len(p) == 2:
 700             p[0] = ast.List([p[1]], ast.Load())
 701         else:
 702             p[0] = ast.List([p[1]] + p[3].nodes, ast.Load())
 703
 704     def p_atom_tuple(self, p):
 705         """atom : LPAR testlist RPAR"""
 706         print("tuple", p[2])
 707         print("astor dump")
 708         print(astor.dump_tree(p[2]))
 709
 710         if isinstance(p[2], ast.Name):
 711             name = p[2].id
 712             print("tuple name", name)
 713             if name in self.gprs:
 714                 self.read_regs.add(name)  # add to list of regs to read
 715                 #p[0] = ast.Subscript(ast.Name("GPR", ast.Load()), ast.Str(p[2].id))
 716                 # return
 717             p[0] = p[2]
 718         elif isinstance(p[2], ast.BinOp):
 719             if isinstance(p[2].left, ast.Name) and \
 720                isinstance(p[2].right, ast.Constant) and \
 721                     p[2].right.value == 0 and \
 722                     p[2].left.id in self.gprs:
 723                 rid = p[2].left.id
 724                 self.read_regs.add(rid)  # add to list of regs to read
 725                 # create special call to GPR.getz or FPR.getz
 726                 if rid in fregs:
 727                     gprz = ast.Name("FPR", ast.Load())
 728                 else:
 729                     gprz = ast.Name("GPR", ast.Load())
 730                 # get testzero function
 731                 gprz = ast.Attribute(gprz, "getz", ast.Load())
 732                 # *sigh* see class GPR.  we need index itself not reg value
 733                 ridx = ast.Name("_%s" % rid, ast.Load())
 734                 p[0] = ast.Call(gprz, [ridx], [])
 735                 print("tree", astor.dump_tree(p[0]))
 736             else:
 737                 p[0] = p[2]
 738         else:
 739             p[0] = p[2]
 740
 741     def p_trailerlist(self, p):
 742         """trailerlist : trailer trailerlist
 743                        | trailer
 744         """
 745         if len(p) == 2:
 746             p[0] = p[1]
 747         else:
 748             p[0] = ("TLIST", p[1], p[2])
 749
 750     # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 751     def p_trailer(self, p):
 752         """trailer : trailer_arglist
 753                    | trailer_subscript
 754                    | trailer_attr
 755         """
 756         p[0] = p[1]
 757
 758     def p_trailer_arglist(self, p):
 759         """trailer_arglist : LPAR arglist RPAR
 760                            | LPAR RPAR
 761         """
 762         args = [] if len(p) == 3 else p[2]
 763         p[0] = ("CALL", args, p.slice[1])
 764
 765     def p_trailer_subscript(self, p):
 766         "trailer_subscript : LBRACK subscript RBRACK"
 767         p[0] = ("SUBS", p[2])
 768
 769     def p_trailer_attr(self, p):
 770         "trailer_attr : PERIOD NAME"
 771         p[0] = ("ATTR", p[2])
 772
 773     # subscript: '.' '.' '.' | test | [test] ':' [test]
 774
 775     def p_subscript(self, p):
 776         """subscript : test COLON test
 777                      | test
 778         """
 779         if len(p) == 4:
 780             # add one to end
 781             if isinstance(p[3], ast.Constant):
 782                 end = ast.Constant(p[3].value+1)
 783             else:
 784                 end = ast.BinOp(p[3], ast.Add(), ast.Constant(1))
 785             p[0] = [p[1], end]
 786         else:
 787             p[0] = [p[1]]
 788
 789     # testlist: test (',' test)* [',']
 790     # Contains shift/reduce error
 791
 792     def p_testlist(self, p):
 793         """testlist : testlist_multi COMMA
 794                     | testlist_multi """
 795         if len(p) == 2:
 796             p[0] = p[1]
 797         else:
 798             # May need to promote singleton to tuple
 799             if isinstance(p[1], list):
 800                 p[0] = p[1]
 801             else:
 802                 p[0] = [p[1]]
 803         # Convert into a tuple?
 804         if isinstance(p[0], list):
 805             p[0] = ast.Tuple(p[0])
 806
 807     def p_testlist_multi(self, p):
 808         """testlist_multi : testlist_multi COMMA test
 809                           | test"""
 810         if len(p) == 2:
 811             # singleton
 812             p[0] = p[1]
 813         else:
 814             if isinstance(p[1], list):
 815                 p[0] = p[1] + [p[3]]
 816             else:
 817                 # singleton -> tuple
 818                 p[0] = [p[1], p[3]]
 819
 820     # test: or_test ['if' or_test 'else' test] | lambdef
 821     #  as I don't support 'and', 'or', and 'not' this works down to 'comparison'
 822
 823     def p_test(self, p):
 824         """test : comparison
 825                 | comparison QMARK test COLON test"""
 826         if len(p) == 2:
 827             p[0] = p[1]
 828         else:
 829             p[0] = ast.IfExp(test=p[1], body=p[3], orelse=p[5])
 830
 831     # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
 832     # | '**' test)
 833     # XXX INCOMPLETE: this doesn't allow the trailing comma
 834
 835     def p_arglist(self, p):
 836         """arglist : arglist COMMA argument
 837                    | argument"""
 838         if len(p) == 4:
 839             p[0] = p[1] + [p[3]]
 840         else:
 841             p[0] = [p[1]]
 842
 843     # argument: test [gen_for] | test '=' test  # Really [keyword '='] test
 844     def p_argument(self, p):
 845         "argument : test"
 846         p[0] = p[1]
 847
 848     def p_error(self, p):
 849         # print "Error!", repr(p)
 850         raise_syntax_error(str(p), self.filename, p.lineno, p.lexpos,
 851                            self.input_text)
 852
 853     def Assign(self, autoassign, assignname, left, right, iea_mode, eq_tok):
 854         names = []
 855         print("Assign", autoassign, assignname, left, right)
 856         if isinstance(left, ast.Name):
 857             # Single assignment on left
 858             # XXX when doing IntClass, which will have an "eq" function,
 859             # this is how to access it
 860             #   eq = ast.Attribute(left, "eq")   # get eq fn
 861             #   return ast.Call(eq, [right], []) # now call left.eq(right)
 862             return ast.Assign([ast.Name(left.id, ast.Store())], right)
 863         elif isinstance(left, ast.Tuple):
 864             # List of things - make sure they are Name nodes
 865             names = []
 866             for child in left.elts:
 867                 if not isinstance(child, ast.Name):
 868                     raise_syntax_error(
 869                         "that assignment not supported", self.filename,
 870                         eq_tok.lineno, eq_tok.lexpos, self.input_text)
 871                 names.append(child.id)
 872             ass_list = [ast.Name(name, ast.Store()) for name in names]
 873             return ast.Assign([ast.Tuple(ass_list)], right)
 874         elif isinstance(left, ast.Attribute):
 875             return ast.Assign([
 876                 ast.Attribute(left.value, left.attr, ast.Store())], right)
 877         elif isinstance(left, ast.Subscript):
 878             ls = left.slice
 879             # XXX changing meaning of "undefined" to a function
 880             # if (isinstance(ls, ast.Slice) and isinstance(right, ast.Name) and
 881             #        right.id == 'undefined'):
 882             #    # undefined needs to be copied the exact same slice
 883             #    right = ast.Subscript(right, ls, ast.Load())
 884             #    return ast.Assign([left], right)
 885             res = ast.Assign([left], right)
 886             if autoassign and isinstance(ls, ast.Slice):
 887                 # hack to create a variable pre-declared based on a slice.
 888                 # dividend[0:32] = (RA)[0:32] will create
 889                 #       dividend = [0] * 32
 890                 #       dividend[0:32] = (RA)[0:32]
 891                 # the declaration makes the slice-assignment "work"
 892                 lower, upper, step = ls.lower, ls.upper, ls.step
 893                 print("lower, upper, step", repr(lower), repr(upper), step)
 894                 # XXX relax constraint that indices on auto-assign have
 895                 # to be constants x[0:32]
 896                 # if not isinstance(lower, ast.Constant) or \
 897                 #   not isinstance(upper, ast.Constant):
 898                 #    return res
 899                 qty = ast.BinOp(upper, binary_ops['-'], lower)
 900                 keywords = [ast.keyword(arg='repeat', value=qty)]
 901                 l = [ast.Num(0)]
 902                 right = ast.Call(ast.Name("concat", ast.Load()), l, keywords)
 903                 declare = ast.Assign(
 904                     [ast.Name(assignname, ast.Store())], right)
 905                 return [declare, res]
 906             return res
 907             # XXX HMMM probably not needed...
 908             ls = left.slice
 909             if isinstance(ls, ast.Slice):
 910                 lower, upper, step = ls.lower, ls.upper, ls.step
 911                 print("slice assign", lower, upper, step)
 912                 if step is None:
 913                     ls = (lower, upper, None)
 914                 else:
 915                     ls = (lower, upper, step)
 916                 ls = ast.Tuple(ls)
 917             return ast.Call(ast.Name("selectassign", ast.Load()),
 918                             [left.value, ls, right], [])
 919         else:
 920             print("Assign fail")
 921             raise_syntax_error("Can't do that yet", self.filename,
 922                                eq_tok.lineno, eq_tok.lexpos, self.input_text)
 923
 924     def try_extract_name(self, atom):
 925         if isinstance(atom, ast.Attribute):
 926             if isinstance(atom.value, ast.Name) and atom.value.id == "self":
 927                 return atom.attr
 928             return None
 929         if isinstance(atom, ast.Name):
 930             return atom.id
 931         return None
 932
 933     def try_extract_uppercase_name(self, atom):
 934         name = self.try_extract_name(atom)
 935         if name is None:
 936             return None
 937         # we want to filter out names like _123 that have no letters with
 938         # casing at all, hence the lower() check
 939         if name.upper() != name or name.lower() == name:
 940             return None
 941         return name
 942
 943     def try_get_attr(self, atom, trailer):
 944         if trailer[0] == "ATTR":
 945             return trailer[1]
 946         if trailer[0] != "SUBS":
 947             return None
 948         base_name = self.try_extract_uppercase_name(atom)
 949         if base_name is None or base_name in SUBS_TO_ATTR_EXCEPTIONS:
 950             return None
 951         if len(trailer[1]) != 1:
 952             return None
 953         return self.try_extract_uppercase_name(trailer[1][0])
 954
 955     def apply_trailer(self, atom, trailer, read_regs):
 956         if trailer[0] == "TLIST":
 957             # assume depth of one
 958             atom = self.apply_trailer(atom, trailer[1], read_regs)
 959             trailer = trailer[2]
 960         if trailer[0] == "CALL":
 961             #p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
 962             for arg in trailer[1]:
 963                 if isinstance(arg, ast.Name):
 964                     name = arg.id
 965                     if name in regs + fregs:
 966                         read_regs.add(name)
 967             if atom.id == "SetFX":
 968                 if len(trailer[1]) != 1 or \
 969                         not isinstance(trailer[1][0], ast.Attribute):
 970                     raise_syntax_error(
 971                         "SetFX call must have only one argument that is an "
 972                         "attribute access -- like `SetFX(FPSCR.VXSNAN)`",
 973                         self.filename, trailer[2].lineno, trailer[2].lexpos,
 974                         self.input_text)
 975                 arg = trailer[1][0]
 976                 args = [arg.value, ast.Str(arg.attr)]
 977                 name = ast.Name("self", ast.Load())
 978                 atom = ast.Attribute(name, atom, ast.Load())
 979                 return ast.Call(atom, args, [])
 980             # special-case, these functions must NOT be made "self.xxxx"
 981             if atom.id not in SPECIAL_HELPERS:
 982                 name = ast.Name("self", ast.Load())
 983                 atom = ast.Attribute(name, atom, ast.Load())
 984             return ast.Call(atom, trailer[1], [])
 985             # if p[1].id == 'print':
 986             #    p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
 987             # else:
 988             #    p[0] = ast.CallFunc(p[1], p[2][1], None, None)
 989         else:
 990             attr = self.try_get_attr(atom, trailer)
 991             if attr is not None:
 992                 return ast.Attribute(atom, attr, ast.Load())
 993             assert trailer[0] == "SUBS"
 994             print("subscript atom", trailer[1])
 995             #raise AssertionError("not implemented %s" % p[2][0])
 996             subs = trailer[1]
 997             if len(subs) == 1:
 998                 idx = subs[0]
 999                 if isinstance(idx, ast.Name) and idx.id in regs + fregs:
1000                     read_regs.add(idx.id)
1001                 if isinstance(idx, ast.Name) and idx.id in regs:
1002                     print("single atom subscript, underscored", idx.id)
1003                     idx = ast.Name("_%s" % idx.id, ast.Load())
1004             else:
1005                 idx = ast.Slice(subs[0], subs[1], None)
1006             # if isinstance(atom, ast.Name) and atom.id == 'CR':
1007                 # atom.id = 'CR' # bad hack
1008                 #print ("apply_trailer Subscript", atom.id, idx)
1009             return ast.Subscript(atom, idx, ast.Load())
1010
1011
1012 _CACHE_DECODER = True
1013 _CACHED_DECODER = None
1014
1015
1016 def _create_cached_decoder():
1017     global _CACHED_DECODER
1018     if _CACHE_DECODER:
1019         if _CACHED_DECODER is None:
1020             _CACHED_DECODER = create_pdecode()
1021         return _CACHED_DECODER
1022     return create_pdecode()
1023
1024
1025 class GardenSnakeParser(PowerParser):
1026     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1027         if form is not None:
1028             self.sd = _create_cached_decoder()
1029         PowerParser.__init__(self, form, incl_carry, helper=helper)
1030         self.debug = debug
1031         self.lexer = IndentLexer(debug=0)
1032         self.tokens = self.lexer.tokens
1033         self.parser = yacc.yacc(module=self, start="file_input_end",
1034                                 debug=debug, write_tables=False)
1035
1036     def parse(self, code):
1037         self.reset()
1038         self.lexer.filename = self.filename
1039         result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
1040         if self.helper:
1041             result = [ast.ClassDef("ISACallerFnHelper", [
1042                                    "ISACallerHelper"], [], result, decorator_list=[])]
1043         return ast.Module(result)
1044
1045
1046 ###### Code generation ######
1047
1048 #from compiler import misc, syntax, pycodegen
1049
1050 _CACHED_PARSERS = {}
1051 _CACHE_PARSERS = True
1052
1053
1054 class GardenSnakeCompiler(object):
1055     def __init__(self, debug=False, form=None, incl_carry=False, helper=False):
1056         if _CACHE_PARSERS:
1057             try:
1058                 self.parser = _CACHED_PARSERS[debug, form, incl_carry, helper]
1059             except KeyError:
1060                 self.parser = GardenSnakeParser(
1061                     debug=debug, form=form, incl_carry=incl_carry,
1062                     helper=helper)
1063                 _CACHED_PARSERS[debug, form, incl_carry, helper] = self.parser
1064         else:
1065             self.parser = GardenSnakeParser(debug=debug, form=form,
1066                                             incl_carry=incl_carry, helper=helper)
1067
1068     def compile(self, code, mode="exec", filename="<string>"):
1069         if filename in ["string", "<string>"]:
1070             raise ValueError("missing filename")
1071         self.parser.filename = filename
1072         self.parser.input_text = code
1073         try:
1074             tree = self.parser.parse(code)
1075         except SyntaxError2 as e:
1076             e.raise_syntax_error()
1077         print("snake")
1078         pprint(tree)
1079         return tree
1080         #misc.set_filename(filename, tree)
1081         return compile(tree, mode="exec", filename="<string>")
1082         # syntax.check(tree)
1083         gen = pycodegen.ModuleCodeGenerator(tree)
1084         code = gen.getCode()
1085         return code