From 8e0f6ea81295acbf72a14d8310fb3cd48ba01ba5 Mon Sep 17 00:00:00 2001
From: Jacob Lifshay <programmerjake@gmail.com>
Date: Wed, 3 May 2023 21:15:45 -0700
Subject: [PATCH] show actual mdwn source location in backtrace when parser
 raises SyntaxError

---
 src/openpower/decoder/power_pseudo.py         |  8 +++---
 .../decoder/pseudo/functionreader.py          |  4 +--
 src/openpower/decoder/pseudo/lexer.py         | 13 +++++++++-
 src/openpower/decoder/pseudo/pagereader.py    | 26 ++++++++++++++++---
 src/openpower/decoder/pseudo/parser.py        | 23 +++++++++++-----
 src/openpower/decoder/pseudo/pyfnwriter.py    |  5 ++--
 src/openpower/decoder/pseudo/pywriter.py      |  6 +++--
 7 files changed, 65 insertions(+), 20 deletions(-)

diff --git a/src/openpower/decoder/power_pseudo.py b/src/openpower/decoder/power_pseudo.py
index f8d87dce..637826bc 100644
--- a/src/openpower/decoder/power_pseudo.py
+++ b/src/openpower/decoder/power_pseudo.py
@@ -258,21 +258,21 @@ def get_reg_hex(reg):
     return hex(reg.value)
 
 
-def convert_to_pure_python(pcode, helper=False):
+def convert_to_pure_python(pcode, helper=False, filename="string"):
 
     gsc = GardenSnakeCompiler(form=None, incl_carry=False, helper=helper)
 
-    tree = gsc.compile(pcode, mode="exec", filename="string")
+    tree = gsc.compile(pcode, mode="exec", filename=filename)
     tree = ast.fix_missing_locations(tree)
     return astor.to_source(tree)
 
 
-def convert_to_python(pcode, form, incl_carry, helper=False):
+def convert_to_python(pcode, form, incl_carry, helper=False, filename="string"):
 
     print("form", form)
     gsc = GardenSnakeCompiler(form=form, incl_carry=incl_carry, helper=helper)
 
-    tree = gsc.compile(pcode, mode="exec", filename="string")
+    tree = gsc.compile(pcode, mode="exec", filename=filename)
     tree = ast.fix_missing_locations(tree)
     regsused = {'read_regs': gsc.parser.read_regs,
                 'write_regs': gsc.parser.write_regs,
diff --git a/src/openpower/decoder/pseudo/functionreader.py b/src/openpower/decoder/pseudo/functionreader.py
index b87cf53e..4e065239 100644
--- a/src/openpower/decoder/pseudo/functionreader.py
+++ b/src/openpower/decoder/pseudo/functionreader.py
@@ -54,7 +54,7 @@ class ISAFunctions:
             # so please put ending of comments on one line:
             # <!-- line 1 comment -->
             # <!-- line 2 comment -->
-            if l.startswith('<!--'):
+            if l.strip().startswith('<!--'):
                 # print ("skipping comment", l)
                 l = lines.pop(0).rstrip()  # get next line
                 prefix_lines += 1
@@ -92,7 +92,7 @@ class ISAFunctions:
                 if len(l) == 0:
                     li.append(l)
                     continue
-                if l.startswith('<!--'):
+                if l.strip().startswith('<!--'):
                     li.append("")
                     continue
                 assert l.startswith('    '), ("4spcs not found in line %s" % l)
diff --git a/src/openpower/decoder/pseudo/lexer.py b/src/openpower/decoder/pseudo/lexer.py
index 861e56a3..876ec87d 100644
--- a/src/openpower/decoder/pseudo/lexer.py
+++ b/src/openpower/decoder/pseudo/lexer.py
@@ -12,6 +12,14 @@ from copy import copy
 from ply import lex
 from openpower.decoder.selectable_int import SelectableInt
 
+
+def raise_syntax_error(msg, filename, lineno, lexpos, input_text):
+    line_start = input_text.rfind('\n', 0, lexpos) + 1
+    line_end = input_text.find('\n', line_start)
+    col = (lexpos - line_start) + 1
+    raise SyntaxError(str(msg), (filename, lineno, col,
+                                 input_text[line_start:line_end]))
+
 # I implemented INDENT / DEDENT generation as a post-processing filter
 
 # The original lex token stream contains WS and NEWLINE characters.
@@ -332,6 +340,7 @@ class PowerLexer:
     # Build the lexer
     def build(self, **kwargs):
         self.lexer = lex.lex(module=self, **kwargs)
+        self.filename = None
 
     def t_HEX(self, t):
         r"""0x[0-9a-fA-F_]+"""
@@ -454,7 +463,9 @@ class PowerLexer:
     #t_ignore = " "
 
     def t_error(self, t):
-        raise SyntaxError("Unknown symbol %r" % (t.value[0],))
+        raise_syntax_error("Unknown symbol %r" % (t.value[0],),
+                           self.filename, t.lexer.lineno,
+                           t.lexer.lexpos, t.lexer.lexdata)
         print("Skipping", repr(t.value[0]))
         t.lexer.skip(1)
 
diff --git a/src/openpower/decoder/pseudo/pagereader.py b/src/openpower/decoder/pseudo/pagereader.py
index debeccec..71301f2f 100644
--- a/src/openpower/decoder/pseudo/pagereader.py
+++ b/src/openpower/decoder/pseudo/pagereader.py
@@ -218,6 +218,7 @@ class ISA:
         # all sections are mandatory so no need for a full LALR parser.
 
         l = lines.pop(0).rstrip()  # get first line
+        prefix_lines = 0
         while lines:
             if self.verbose:
                 print(l)
@@ -229,11 +230,13 @@ class ISA:
             if l.strip().startswith('<!--'):
                 # print ("skipping comment", l)
                 l = lines.pop(0).rstrip()  # get next line
+                prefix_lines += 1
                 continue
 
             # Ignore blank lines before the first #
             if len(l) == 0:
                 l = lines.pop(0).rstrip()  # get next line
+                prefix_lines += 1
                 continue
 
             # expect get heading
@@ -242,23 +245,27 @@ class ISA:
 
             # whitespace expected
             l = lines.pop(0).strip()
+            prefix_lines += 1
             if self.verbose:
                 print(repr(l))
             assert len(l) == 0, ("blank line not found %s" % l)
 
             # Form expected
             l = lines.pop(0).strip()
+            prefix_lines += 1
             assert l.endswith('-Form'), ("line with -Form expected %s" % l)
             d['form'] = l.split('-')[0]
 
             # whitespace expected
             l = lines.pop(0).strip()
+            prefix_lines += 1
             assert len(l) == 0, ("blank line not found %s" % l)
 
             # get list of opcodes
             opcodes = []
             while True:
                 l = lines.pop(0).strip()
+                prefix_lines += 1
                 if len(l) == 0:
                     break
                 assert l.startswith('*'), ("* not found in line %s" % l)
@@ -285,22 +292,31 @@ class ISA:
 
             # "Pseudocode" expected
             l = lines.pop(0).rstrip()
+            prefix_lines += 1
             assert l.startswith("Pseudo-code:"), ("pseudocode found %s" % l)
 
             # whitespace expected
             l = lines.pop(0).strip()
+            prefix_lines += 1
             if self.verbose:
                 print(repr(l))
             assert len(l) == 0, ("blank line not found %s" % l)
 
             # get pseudocode
-            li = []
+
+            # fix parser line numbers by prepending the right number of
+            # blank lines to the parser input
+            li = [""] * prefix_lines
+            li += [l[4:]]  # first line detected with 4-space
             while True:
                 l = lines.pop(0).rstrip()
-                if l.strip().startswith('<!--'):
-                    continue
+                prefix_lines += 1
                 if len(l) == 0:
+                    li.append(l)
                     break
+                if l.strip().startswith('<!--'):
+                    li.append("")
+                    continue
                 assert l.startswith('    '), ("4spcs not found in line %s" % l)
                 l = l[4:]  # lose 4 spaces
                 li.append(l)
@@ -308,16 +324,19 @@ class ISA:
 
             # "Special Registers Altered" expected
             l = lines.pop(0).rstrip()
+            prefix_lines += 1
             assert l.startswith("Special"), ("special not found %s" % l)
 
             # whitespace expected
             l = lines.pop(0).strip()
+            prefix_lines += 1
             assert len(l) == 0, ("blank line not found %s" % l)
 
             # get special regs
             li = []
             while lines:
                 l = lines.pop(0).rstrip()
+                prefix_lines += 1
                 if len(l) == 0:
                     break
                 assert l.startswith('    '), ("4spcs not found in line %s" % l)
@@ -332,6 +351,7 @@ class ISA:
             # expect and drop whitespace and comments
             while lines:
                 l = lines.pop(0).rstrip()
+                prefix_lines += 1
                 if len(l) != 0 and not l.strip().startswith('<!--'):
                     break
 
diff --git a/src/openpower/decoder/pseudo/parser.py b/src/openpower/decoder/pseudo/parser.py
index 902dee8f..d8c0399e 100644
--- a/src/openpower/decoder/pseudo/parser.py
+++ b/src/openpower/decoder/pseudo/parser.py
@@ -14,7 +14,7 @@ import astor
 from copy import deepcopy
 
 from openpower.decoder.power_decoder import create_pdecode
-from openpower.decoder.pseudo.lexer import IndentLexer
+from openpower.decoder.pseudo.lexer import IndentLexer, raise_syntax_error
 from openpower.decoder.orderedset import OrderedSet
 
 # I use the Python AST
@@ -254,6 +254,8 @@ class PowerParser:
         self.include_ca_in_write = include_carry_in_write
         self.helper = helper
         self.form = form
+        self.filename = None
+        self.input_text = None
         self.reset()
 
     # The grammar comments come from Python's Grammar/Grammar file
@@ -436,7 +438,7 @@ class PowerParser:
             # documentation for why we need this
             copy_fn = ast.Name("copy_assign_rhs", ast.Load())
             rhs = ast.Call(copy_fn, (p[3],), [])
-            p[0] = self.Assign(autoassign, name, p[1], rhs, iea_mode)
+            p[0] = self.Assign(autoassign, name, p[1], rhs, iea_mode, p[2])
             if name:
                 self.declared_vars.add(name)
 
@@ -864,9 +866,10 @@ class PowerParser:
 
     def p_error(self, p):
         # print "Error!", repr(p)
-        raise SyntaxError(p)
+        raise_syntax_error(str(p), self.filename, p.lineno, p.lexpos,
+                           self.input_text)
 
-    def Assign(self, autoassign, assignname, left, right, iea_mode):
+    def Assign(self, autoassign, assignname, left, right, iea_mode, eq_tok):
         names = []
         print("Assign", autoassign, assignname, left, right)
         if isinstance(left, ast.Name):
@@ -881,7 +884,9 @@ class PowerParser:
             names = []
             for child in left.elts:
                 if not isinstance(child, ast.Name):
-                    raise SyntaxError("that assignment not supported")
+                    raise_syntax_error(
+                        "that assignment not supported", self.filename,
+                        eq_tok.lineno, eq_tok.lexpos, self.input_text)
                 names.append(child.id)
             ass_list = [ast.Name(name, ast.Store()) for name in names]
             return ast.Assign([ast.Tuple(ass_list)], right)
@@ -929,7 +934,8 @@ class PowerParser:
                             [left.value, ls, right], [])
         else:
             print("Assign fail")
-            raise SyntaxError("Can't do that yet")
+            raise_syntax_error("Can't do that yet", self.filename,
+                               eq_tok.lineno, eq_tok.lexpos, self.input_text)
 
 
 _CACHE_DECODER = True
@@ -958,6 +964,7 @@ class GardenSnakeParser(PowerParser):
 
     def parse(self, code):
         self.reset()
+        self.lexer.filename = self.filename
         result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
         if self.helper:
             result = [ast.ClassDef("ISACallerFnHelper", [
@@ -988,6 +995,10 @@ class GardenSnakeCompiler(object):
                                             incl_carry=incl_carry, helper=helper)
 
     def compile(self, code, mode="exec", filename="<string>"):
+        if filename in ["string", "<string>"]:
+            raise ValueError("missing filename")
+        self.parser.filename = filename
+        self.parser.input_text = code
         tree = self.parser.parse(code)
         print("snake")
         pprint(tree)
diff --git a/src/openpower/decoder/pseudo/pyfnwriter.py b/src/openpower/decoder/pseudo/pyfnwriter.py
index cb170876..611c16ee 100644
--- a/src/openpower/decoder/pseudo/pyfnwriter.py
+++ b/src/openpower/decoder/pseudo/pyfnwriter.py
@@ -4,7 +4,7 @@ import os
 import sys
 import shutil
 import subprocess
-from openpower.decoder.pseudo.functionreader import ISAFunctions
+from openpower.decoder.pseudo.functionreader import ISAFunctions, get_isafn_dir
 from openpower.decoder.power_pseudo import convert_to_pure_python
 
 
@@ -50,7 +50,8 @@ class PyISAFnWriter(ISAFunctions):
             if phash in sourcecache:
                 pycode = sourcecache[phash]
             else:
-                pycode = convert_to_pure_python(pcode, True)
+                filename = os.path.join(get_isafn_dir(), pagename + ".mdwn")
+                pycode = convert_to_pure_python(pcode, True, filename=filename)
                 sourcecache[phash] = pycode
             f.write(pycode)
 
diff --git a/src/openpower/decoder/pseudo/pywriter.py b/src/openpower/decoder/pseudo/pywriter.py
index 353c2aa1..49d51c09 100644
--- a/src/openpower/decoder/pseudo/pywriter.py
+++ b/src/openpower/decoder/pseudo/pywriter.py
@@ -4,7 +4,7 @@ import os
 import sys
 import shutil
 import subprocess
-from openpower.decoder.pseudo.pagereader import ISA
+from openpower.decoder.pseudo.pagereader import ISA, get_isa_dir
 from openpower.decoder.power_pseudo import convert_to_python
 from openpower.decoder.orderedset import OrderedSet
 from openpower.decoder.isa.caller import create_args
@@ -77,7 +77,9 @@ class PyISAWriter(ISA):
                 pcode = '\n'.join(d.pcode) + '\n'
                 print(pcode)
                 incl_carry = pagename == 'fixedshift'
-                pycode, rused = convert_to_python(pcode, d.form, incl_carry)
+                filename = os.path.join(get_isa_dir(), pagename + ".mdwn")
+                pycode, rused = convert_to_python(pcode, d.form, incl_carry,
+                                                  filename=filename)
                 # create list of arguments to call
                 regs = list(rused['read_regs']) + list(rused['uninit_regs'])
                 regs += list(rused['special_regs'])
-- 
2.30.2