1 # ----------------------------------------------------------------------
4 # Token specifications for symbols in ANSI C and C++. This file is
5 # meant to be used as a library in other tokenizers.
6 # ----------------------------------------------------------------------
11 # Literals (identifier, integer constant, float constant, string constant, char const)
12 'ID', 'TYPEID', 'ICONST', 'FCONST', 'SCONST', 'CCONST',
14 # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=)
15 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',
16 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',
17 'LOR', 'LAND', 'LNOT',
18 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE',
20 # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=)
21 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL',
22 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL',
24 # Increment/decrement (++,--)
25 'PLUSPLUS', 'MINUSMINUS',
27 # Structure dereference (->)
30 # Ternary operator (?)
33 # Delimeters ( ) [ ] { } , . ; :
35 'LBRACKET', 'RBRACKET',
37 'COMMA', 'PERIOD', 'SEMI', 'COLON',
65 # Assignment operators
73 t_LSHIFTEQUAL
= r
'<<='
74 t_RSHIFTEQUAL
= r
'>>='
100 t_ELLIPSIS
= r
'\.\.\.'
103 t_ID
= r
'[A-Za-z_][A-Za-z0-9_]*'
106 t_INTEGER
= r
'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?'
109 t_FLOAT
= r
'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?'
112 t_STRING
= r
'\"([^\\\n]|(\\.))*?\"'
114 # Character constant 'c' or L'c'
115 t_CHARACTER
= r
'(L)?\'([^
\\\n]|
(\\.))*?
\''
120 t.lexer.lineno += t.value.count('\n')
123 # Comment (C++-Style)