1 # Based on GardenSnake - a parser generator demonstration program
2 # GardenSnake was released into the Public Domain by Andrew Dalke.
4 # Portions of this work are derived from Python's Grammar definition
5 # and may be covered under the Python copyright and license
7 # Andrew Dalke / Dalke Scientific Software, LLC
8 # 30 August 2006 / Cape Town, South Africa
10 # Modifications for inclusion in PLY distribution
13 from openpower
.decoder
.selectable_int
import SelectableInt
15 # I implemented INDENT / DEDENT generation as a post-processing filter
17 # The original lex token stream contains WS and NEWLINE characters.
18 # WS will only occur before any other tokens on a line.
20 # I have three filters. One tags tokens by adding two attributes.
21 # "must_indent" is True if the token must be indented from the
22 # previous code. The other is "at_line_start" which is True for WS
23 # and the first non-WS/non-NEWLINE on a line. It flags the check so
24 # see if the new line has changed indication level.
26 # Python's syntax has three INDENT states
27 # 0) no colon hence no need to indent
28 # 1) "if 1: go()" - simple statements have a COLON but no need for an indent
29 # 2) "if 1:\n go()" - complex statements have a COLON NEWLINE and must indent
34 # turn into python-like colon syntax from pseudo-code syntax.
35 # identify tokens which tell us whether a "hidden colon" is needed.
36 # this in turn means that track_tokens_filter "works" without needing
37 # complex grammar rules
40 def python_colonify(lexer
, tokens
):
42 implied_colon_needed
= False
44 #print ("track colon token", token, token.type)
46 if token
.type == 'THEN':
47 # turn then into colon
50 elif token
.type == 'ELSE':
55 elif token
.type in ['DO', 'WHILE', 'FOR', 'SWITCH']:
56 implied_colon_needed
= True
58 elif token
.type == 'NEWLINE':
59 if implied_colon_needed
:
63 implied_colon_needed
= False
69 # only care about whitespace at the start of a line
70 def track_tokens_filter(lexer
, tokens
):
71 oldignore
= lexer
.lexignore
72 lexer
.at_line_start
= at_line_start
= True
76 #print ("track token", token, token.type)
77 token
.at_line_start
= at_line_start
79 if token
.type == "COLON":
82 token
.must_indent
= False
84 elif token
.type == "NEWLINE":
86 if indent
== MAY_INDENT
:
88 token
.must_indent
= False
90 elif token
.type == "WS":
91 assert token
.at_line_start
== True
93 token
.must_indent
= False
96 # A real token; only indent after COLON NEWLINE
97 if indent
== MUST_INDENT
:
98 token
.must_indent
= True
100 token
.must_indent
= False
101 at_line_start
= False
104 # really bad hack that changes ignore lexer state.
105 # when "must indent" is seen (basically "real tokens" seen)
106 # then ignore whitespace.
107 if token
.must_indent
:
108 lexer
.lexignore
= ('ignore', ' ')
110 lexer
.lexignore
= oldignore
112 token
.indent
= indent
114 lexer
.at_line_start
= at_line_start
117 def _new_token(type, lineno
):
125 # Synthesize a DEDENT tag
129 return _new_token("DEDENT", lineno
)
131 # Synthesize an INDENT tag
135 return _new_token("INDENT", lineno
)
139 for i
in range(len(l
)):
145 def annoying_case_hack_filter(code
):
146 """add annoying "silent keyword" (fallthrough)
148 this which tricks the parser into taking the (silent) case statement
149 as a "small expression". it can then be spotted and used to indicate
150 "fall through" to the next case (in the parser)
152 also skips blank lines
154 bugs: any function that starts with the letters "case" or "default"
155 will be detected erroneously. fixing that involves doing a token
156 lexer which spots the fact that "case" and "default" are words,
157 separating them from space, colon, bracket etc.
159 http://bugs.libre-riscv.org/show_bug.cgi?id=280
162 prev_spc_count
= None
163 for l
in code
.split("\n"):
164 spc_count
= count_spaces(l
)
165 nwhite
= l
[spc_count
:]
166 if len(nwhite
) == 0: # skip blank lines
169 if nwhite
.startswith("case") or nwhite
.startswith("default"):
170 #print ("case/default", nwhite, spc_count, prev_spc_count)
171 if (prev_spc_count
is not None and
172 prev_spc_count
== spc_count
and
173 (res
[-1].endswith(":") or res
[-1].endswith(": fallthrough"))):
174 res
[-1] += " fallthrough" # add to previous line
175 prev_spc_count
= spc_count
177 #print ("notstarts", spc_count, nwhite)
178 prev_spc_count
= None
180 return '\n'.join(res
)
183 # Track the indentation level and emit the right INDENT / DEDENT events.
184 def indentation_filter(tokens
):
185 # A stack of indentation levels; will never pop item 0
192 print("Process", depth
, token
.indent
, token
,)
193 if token
.at_line_start
:
194 print("at_line_start",)
195 if token
.must_indent
:
196 print("must_indent",)
199 # WS only occurs at the start of the line
200 # There may be WS followed by NEWLINE so
201 # only track the depth here. Don't indent/dedent
202 # until there's something real.
203 if token
.type == "WS":
205 depth
= len(token
.value
)
207 # WS tokens are never passed to the parser
210 if token
.type == "NEWLINE":
212 if prev_was_ws
or token
.at_line_start
:
215 # pass the other cases on through
219 # then it must be a real token (not WS, not NEWLINE)
220 # which can affect the indentation level
223 if token
.must_indent
:
224 # The current depth must be larger than the previous level
225 if not (depth
> levels
[-1]):
226 raise IndentationError("expected an indented block")
229 yield INDENT(token
.lineno
)
231 elif token
.at_line_start
:
232 # Must be on the same level or one of the previous levels
233 if depth
== levels
[-1]:
236 elif depth
> levels
[-1]:
237 raise IndentationError("indent increase but not in new block")
239 # Back up; but only if it matches a previous level
241 i
= levels
.index(depth
)
243 raise IndentationError("inconsistent indentation")
244 for _
in range(i
+1, len(levels
)):
245 yield DEDENT(token
.lineno
)
250 ### Finished processing ###
252 # Must dedent any remaining levels
254 assert token
is not None
255 for _
in range(1, len(levels
)):
256 yield DEDENT(token
.lineno
)
259 # The top-level filter adds an ENDMARKER, if requested.
260 # Python's grammar uses it.
261 def filter(lexer
, add_endmarker
=True):
263 tokens
= iter(lexer
.token
, None)
264 tokens
= python_colonify(lexer
, tokens
)
265 tokens
= track_tokens_filter(lexer
, tokens
)
266 for token
in indentation_filter(tokens
):
271 if token
is not None:
272 lineno
= token
.lineno
273 yield _new_token("ENDMARKER", lineno
)
291 'NUMBER', # Python decimals
292 'BINARY', # Python binary
293 'STRING', # single quoted strings only; syntax of raw strings
333 def build(self
, **kwargs
):
334 self
.lexer
= lex
.lex(module
=self
, **kwargs
)
337 r
"""0x[0-9a-fA-F_]+"""
338 val
= t
.value
.replace("_", "")
339 t
.value
= SelectableInt(int(val
, 16), (len(val
)-2)*4) # hex = nibble
342 def t_BINARY(self
, t
):
344 t
.value
= SelectableInt(int(t
.value
, 2), len(t
.value
)-2)
348 # taken from decmial.py but without the leading sign
349 def t_NUMBER(self
, t
):
350 r
"""(\d+(\.\d*)?|\.\d+)([eE][-+]? \d+)?"""
351 t
.value
= int(t
.value
)
354 def t_STRING(self
, t
):
355 r
"'([^\\']+|\\'|\\\\)*'" # I think this is right ...
357 t
.value
= t
.value
[1:-1]
362 t_ASSIGNEA
= r
'<-iea'
384 # Ply nicely documented how to do this.
399 "default": "DEFAULT",
403 r
'[a-zA-Z_][a-zA-Z0-9_]*'
404 t
.type = self
.RESERVED
.get(t
.value
, "NAME")
407 # Putting this before t_WS let it consume lines with only comments in
408 # them so the latter code never sees the WS part. Not consuming the
409 # newline. Needed for "if 1: #comment"
410 def t_comment(self
, t
):
411 r
"[ ]*\043[^\n]*" # \043 is '#'
418 if t
.lexer
.at_line_start
and t
.lexer
.paren_count
== 0 and \
419 t
.lexer
.brack_count
== 0:
422 # Don't generate newline tokens when inside of parenthesis, eg
425 def t_newline(self
, t
):
427 t
.lexer
.lineno
+= len(t
.value
)
429 if t
.lexer
.paren_count
== 0 and t
.lexer
.brack_count
== 0:
432 def t_LBRACK(self
, t
):
434 t
.lexer
.brack_count
+= 1
437 def t_RBRACK(self
, t
):
439 # check for underflow? should be the job of the parser
440 t
.lexer
.brack_count
-= 1
445 t
.lexer
.paren_count
+= 1
450 # check for underflow? should be the job of the parser
451 t
.lexer
.paren_count
-= 1
456 def t_error(self
, t
):
457 raise SyntaxError("Unknown symbol %r" % (t
.value
[0],))
458 print("Skipping", repr(t
.value
[0]))
462 # Combine Ply and my filters into a new lexer
464 class IndentLexer(PowerLexer
):
465 def __init__(self
, debug
=0, optimize
=0, lextab
='lextab', reflags
=0):
467 self
.build(debug
=debug
, optimize
=optimize
,
468 lextab
=lextab
, reflags
=reflags
)
469 self
.token_stream
= None
471 def input(self
, s
, add_endmarker
=True):
472 s
= annoying_case_hack_filter(s
)
476 self
.lexer
.paren_count
= 0
477 self
.lexer
.brack_count
= 0
478 self
.lexer
.lineno
= 1
480 self
.token_stream
= filter(self
.lexer
, add_endmarker
)
484 return next(self
.token_stream
)
485 except StopIteration:
507 if (RS)[63-n] = 0b1 then
514 if __name__
== '__main__':
521 lexer
= IndentLexer(debug
=1)
522 # Give the lexer some input
527 tokens
= iter(lexer
.token
, None)