src/amd/compiler/tests/glsl_scraper.py

   1 #! /usr/bin/env python3
   2 # Taken from Crucible and modified to parse declarations
   3
   4 import argparse
   5 import io
   6 import os
   7 import re
   8 import shutil
   9 import struct
  10 import subprocess
  11 import sys
  12 import tempfile
  13 from textwrap import dedent
  14
  15 class ShaderCompileError(RuntimeError):
  16     def __init__(self, *args):
  17         super(ShaderCompileError, self).__init__(*args)
  18
  19 target_env_re = re.compile(r'QO_TARGET_ENV\s+(\S+)')
  20
  21 stage_to_glslang_stage = {
  22     'VERTEX': 'vert',
  23     'TESS_CONTROL': 'tesc',
  24     'TESS_EVALUATION': 'tese',
  25     'GEOMETRY': 'geom',
  26     'FRAGMENT': 'frag',
  27     'COMPUTE': 'comp',
  28 }
  29
  30 base_layout_qualifier_id_re = r'({0}\s*=\s*(?P<{0}>\d+))'
  31 id_re = '(?P<name_%d>[^(gl_)]\S+)'
  32 type_re = '(?P<dtype_%d>\S+)'
  33 location_re = base_layout_qualifier_id_re.format('location')
  34 component_re = base_layout_qualifier_id_re.format('component')
  35 binding_re = base_layout_qualifier_id_re.format('binding')
  36 set_re = base_layout_qualifier_id_re.format('set')
  37 unk_re = r'\S+(=\d+)?'
  38 layout_qualifier_re = r'layout\W*\((%s)+\)' % '|'.join([location_re, binding_re, set_re, unk_re, '[, ]+'])
  39 ubo_decl_re = 'uniform\W+%s(\W*{)?(?P<type_ubo>)' % (id_re%0)
  40 ssbo_decl_re = 'buffer\W+%s(\W*{)?(?P<type_ssbo>)' % (id_re%1)
  41 image_buffer_decl_re = r'uniform\W+imageBuffer\w+%s;(?P<type_img_buf>)' % (id_re%2)
  42 image_decl_re = r'uniform\W+image\S+\W+%s;(?P<type_img>)' % (id_re%3)
  43 texture_buffer_decl_re = r'uniform\W+textureBuffer\w+%s;(?P<type_tex_buf>)' % (id_re%4)
  44 combined_texture_sampler_decl_re = r'uniform\W+sampler\S+\W+%s;(?P<type_combined>)' % (id_re%5)
  45 texture_decl_re = r'uniform\W+texture\S+\W+%s;(?P<type_tex>)' % (id_re%6)
  46 sampler_decl_re = r'uniform\W+sampler\w+%s;(?P<type_samp>)' % (id_re%7)
  47 input_re = r'in\W+%s\W+%s;(?P<type_in>)' % (type_re%0, id_re%8)
  48 output_re = r'out\W+%s\W+%s;(?P<type_out>)' % (type_re%1, id_re%9)
  49 match_decl_re = re.compile(layout_qualifier_re + r'\W*((' + r')|('.join([ubo_decl_re, ssbo_decl_re, image_buffer_decl_re, image_decl_re, texture_buffer_decl_re, combined_texture_sampler_decl_re, texture_decl_re, sampler_decl_re, input_re, output_re]) + r'))$')
  50
  51 class Shader:
  52     def __init__(self, stage):
  53         self.glsl = None
  54         self.stream = io.StringIO()
  55         self.stage = stage
  56         self.dwords = None
  57         self.target_env = ""
  58         self.declarations = []
  59
  60     def add_text(self, s):
  61         self.stream.write(s)
  62
  63     def finish_text(self, start_line, end_line):
  64         self.glsl = self.stream.getvalue()
  65         self.stream = None
  66
  67         # Handle the QO_EXTENSION macro
  68         self.glsl = self.glsl.replace('QO_EXTENSION', '#extension')
  69
  70         # Handle the QO_DEFINE macro
  71         self.glsl = self.glsl.replace('QO_DEFINE', '#define')
  72
  73         m = target_env_re.search(self.glsl)
  74         if m:
  75             self.target_env = m.group(1)
  76         self.glsl = self.glsl.replace('QO_TARGET_ENV', '// --target-env')
  77
  78         self.start_line = start_line
  79         self.end_line = end_line
  80
  81     def __run_glslang(self, extra_args=[]):
  82         stage = stage_to_glslang_stage[self.stage]
  83         stage_flags = ['-S', stage]
  84
  85         in_file = tempfile.NamedTemporaryFile(suffix='.'+stage)
  86         src = ('#version 450\n' + self.glsl).encode('utf-8')
  87         in_file.write(src)
  88         in_file.flush()
  89         out_file = tempfile.NamedTemporaryFile(suffix='.spirv')
  90         args = [glslang, '-H'] + extra_args + stage_flags
  91         if self.target_env:
  92             args += ['--target-env', self.target_env]
  93         args += ['-o', out_file.name, in_file.name]
  94         with subprocess.Popen(args,
  95                               stdout = subprocess.PIPE,
  96                               stderr = subprocess.PIPE,
  97                               stdin = subprocess.PIPE) as proc:
  98
  99             out, err = proc.communicate(timeout=30)
 100             in_file.close()
 101
 102             if proc.returncode != 0:
 103                 # Unfortunately, glslang dumps errors to standard out.
 104                 # However, since we don't really want to count on that,
 105                 # we'll grab the output of both
 106                 message = out.decode('utf-8') + '\n' + err.decode('utf-8')
 107                 raise ShaderCompileError(message.strip())
 108
 109             out_file.seek(0)
 110             spirv = out_file.read()
 111             out_file.close()
 112             return (spirv, out)
 113
 114     def _parse_declarations(self):
 115         for line in self.glsl.splitlines():
 116             res = re.match(match_decl_re, line.lstrip().rstrip())
 117             if res == None:
 118                 continue
 119             res = {k:v for k, v in res.groupdict().items() if v != None}
 120             name = [v for k, v in res.items() if k.startswith('name_')][0]
 121             data_type = ([v for k, v in res.items() if k.startswith('dtype_')] + [''])[0]
 122             decl_type = [k for k, v in res.items() if k.startswith('type_')][0][5:]
 123             location = int(res.get('location', 0))
 124             component = int(res.get('component', 0))
 125             binding = int(res.get('binding', 0))
 126             desc_set = int(res.get('set', 0))
 127             self.declarations.append('{"%s", "%s", QoShaderDeclType_%s, %d, %d, %d, %d}' %
 128                                      (name, data_type, decl_type, location, component, binding, desc_set))
 129
 130     def compile(self):
 131         def dwords(f):
 132             while True:
 133                 dword_str = f.read(4)
 134                 if not dword_str:
 135                     return
 136                 assert len(dword_str) == 4
 137                 yield struct.unpack('I', dword_str)[0]
 138
 139         (spirv, assembly) = self.__run_glslang()
 140         self.dwords = list(dwords(io.BytesIO(spirv)))
 141         self.assembly = str(assembly, 'utf-8')
 142
 143         self._parse_declarations()
 144
 145     def _dump_glsl_code(self, f):
 146         # Dump GLSL code for reference.  Use // instead of /* */
 147         # comments so we don't need to escape the GLSL code.
 148         f.write('// GLSL code:\n')
 149         f.write('//')
 150         for line in self.glsl.splitlines():
 151             f.write('\n// {0}'.format(line))
 152         f.write('\n\n')
 153
 154     def _dump_spirv_code(self, f, var_name):
 155         f.write('/* SPIR-V Assembly:\n')
 156         f.write(' *\n')
 157         for line in self.assembly.splitlines():
 158             f.write(' * ' + line + '\n')
 159         f.write(' */\n')
 160
 161         f.write('static const uint32_t {0}[] = {{'.format(var_name))
 162         line_start = 0
 163         while line_start < len(self.dwords):
 164             f.write('\n    ')
 165             for i in range(line_start, min(line_start + 6, len(self.dwords))):
 166                 f.write(' 0x{:08x},'.format(self.dwords[i]))
 167             line_start += 6
 168         f.write('\n};\n')
 169
 170     def dump_c_code(self, f):
 171         f.write('\n\n')
 172         var_prefix = '__qonos_shader{0}'.format(self.end_line)
 173
 174         self._dump_glsl_code(f)
 175         self._dump_spirv_code(f, var_prefix + '_spir_v_src')
 176         f.write('static const QoShaderDecl {0}_decls[] = {{{1}}};\n'.format(var_prefix, ', '.join(self.declarations)))
 177
 178         f.write(dedent("""\
 179             static const QoShaderModuleCreateInfo {0}_info = {{
 180                 .spirvSize = sizeof({0}_spir_v_src),
 181                 .pSpirv = {0}_spir_v_src,
 182                 .declarationCount = sizeof({0}_decls) / sizeof({0}_decls[0]),
 183                 .pDeclarations = {0}_decls,
 184             """.format(var_prefix)))
 185
 186         f.write("    .stage = VK_SHADER_STAGE_{0}_BIT,\n".format(self.stage))
 187
 188         f.write('};\n')
 189
 190         f.write('#define __qonos_shader{0}_info __qonos_shader{1}_info\n'\
 191                 .format(self.start_line, self.end_line))
 192
 193 token_exp = re.compile(r'(qoShaderModuleCreateInfoGLSL|qoCreateShaderModuleGLSL|\(|\)|,)')
 194
 195 class Parser:
 196     def __init__(self, f):
 197         self.infile = f
 198         self.paren_depth = 0
 199         self.shader = None
 200         self.line_number = 1
 201         self.shaders = []
 202
 203         def tokenize(f):
 204             leftover = ''
 205             for line in f:
 206                 pos = 0
 207                 while True:
 208                     m = token_exp.search(line, pos)
 209                     if m:
 210                         if m.start() > pos:
 211                             leftover += line[pos:m.start()]
 212                         pos = m.end()
 213
 214                         if leftover:
 215                             yield leftover
 216                             leftover = ''
 217
 218                         yield m.group(0)
 219
 220                     else:
 221                         leftover += line[pos:]
 222                         break
 223
 224                 self.line_number += 1
 225
 226             if leftover:
 227                 yield leftover
 228
 229         self.token_iter = tokenize(self.infile)
 230
 231     def handle_shader_src(self):
 232         paren_depth = 1
 233         for t in self.token_iter:
 234             if t == '(':
 235                 paren_depth += 1
 236             elif t == ')':
 237                 paren_depth -= 1
 238                 if paren_depth == 0:
 239                     return
 240
 241             self.current_shader.add_text(t)
 242
 243     def handle_macro(self, macro):
 244         t = next(self.token_iter)
 245         assert t == '('
 246
 247         start_line = self.line_number
 248
 249         if macro == 'qoCreateShaderModuleGLSL':
 250             # Throw away the device parameter
 251             t = next(self.token_iter)
 252             t = next(self.token_iter)
 253             assert t == ','
 254
 255         stage = next(self.token_iter).strip()
 256
 257         t = next(self.token_iter)
 258         assert t == ','
 259
 260         self.current_shader = Shader(stage)
 261         self.handle_shader_src()
 262         self.current_shader.finish_text(start_line, self.line_number)
 263
 264         self.shaders.append(self.current_shader)
 265         self.current_shader = None
 266
 267     def run(self):
 268         for t in self.token_iter:
 269             if t in ('qoShaderModuleCreateInfoGLSL', 'qoCreateShaderModuleGLSL'):
 270                 self.handle_macro(t)
 271
 272 def open_file(name, mode):
 273     if name == '-':
 274         if mode == 'w':
 275             return sys.stdout
 276         elif mode == 'r':
 277             return sys.stdin
 278         else:
 279             assert False
 280     else:
 281         return open(name, mode)
 282
 283 def parse_args():
 284     description = dedent("""\
 285         This program scrapes a C file for any instance of the
 286         qoShaderModuleCreateInfoGLSL and qoCreateShaderModuleGLSL macaros,
 287         grabs the GLSL source code, compiles it to SPIR-V.  The resulting
 288         SPIR-V code is written to another C file as an array of 32-bit
 289         words.
 290
 291         If '-' is passed as the input file or output file, stdin or stdout
 292         will be used instead of a file on disc.""")
 293
 294     p = argparse.ArgumentParser(
 295             description=description,
 296             formatter_class=argparse.RawDescriptionHelpFormatter)
 297     p.add_argument('-o', '--outfile', default='-',
 298                         help='Output to the given file (default: stdout).')
 299     p.add_argument('--with-glslang', metavar='PATH',
 300                         default='glslangValidator',
 301                         dest='glslang',
 302                         help='Full path to the glslangValidator shader compiler.')
 303     p.add_argument('infile', metavar='INFILE')
 304
 305     return p.parse_args()
 306
 307
 308 args = parse_args()
 309 infname = args.infile
 310 outfname = args.outfile
 311 glslang = args.glslang
 312
 313 with open_file(infname, 'r') as infile:
 314     parser = Parser(infile)
 315     parser.run()
 316
 317 for shader in parser.shaders:
 318     shader.compile()
 319
 320 with open_file(outfname, 'w') as outfile:
 321     outfile.write(dedent("""\
 322         /* ==========================  DO NOT EDIT!  ==========================
 323          *             This file is autogenerated by glsl_scraper.py.
 324          */
 325
 326         #include <stdint.h>
 327
 328         #define __QO_SHADER_INFO_VAR2(_line) __qonos_shader ## _line ## _info
 329         #define __QO_SHADER_INFO_VAR(_line) __QO_SHADER_INFO_VAR2(_line)
 330
 331         #define qoShaderModuleCreateInfoGLSL(stage, ...)  \\
 332             __QO_SHADER_INFO_VAR(__LINE__)
 333
 334         #define qoCreateShaderModuleGLSL(dev, stage, ...) \\
 335             __qoCreateShaderModule((dev), &__QO_SHADER_INFO_VAR(__LINE__))
 336         """))
 337
 338     for shader in parser.shaders:
 339         shader.dump_c_code(outfile)