src/vulkan/glsl_scraper.py

   1 #! /usr/bin/env python
   2
   3 def print_usage(err):
   4    print """\
   5 glsl_scraper.py [options] file
   6
   7 This program scrapes a C file for any instance of the GLSL_VK_SHADER macro,
   8 grabs the GLSL source code, compiles it to SPIR-V.  The resulting SPIR-V
   9 code is written to another C file as an array of 32-bit words.
  10
  11 If '-' is passed as the input file or output file, stdin or stdout will be
  12 used instead of a file on disc.
  13
  14 Options:
  15    -o outfile           Output to the given file (default: stdout)
  16    --with-glslang=PATH  Full path to the glslangValidator program"""
  17    exit(err)
  18
  19 import os, sys, re, cStringIO, tempfile, subprocess, struct, shutil
  20
  21 class Shader:
  22    def __init__(self, stage):
  23       self.stream = cStringIO.StringIO()
  24       self.stage = stage
  25
  26       if self.stage == 'VERTEX':
  27          self.ext = 'vert'
  28       elif self.stage == 'TESS_CONTROL':
  29          self.ext = 'tesc'
  30       elif self.stage == 'TESS_EVALUATION':
  31          self.ext = 'tese'
  32       elif self.stage == 'GEOMETRY':
  33          self.ext = 'geom'
  34       elif self.stage == 'FRAGMENT':
  35          self.ext = 'frag'
  36       elif self.stage == 'COMPUTE':
  37          self.ext = 'comp'
  38       else:
  39          assert False
  40
  41    def add_text(self, s):
  42       self.stream.write(s)
  43
  44    def finish_text(self, line):
  45       self.line = line
  46
  47    def glsl_source(self):
  48       return self.stream.getvalue()
  49
  50    def compile(self):
  51       # We can assume if we got here that we have a temp directory and that
  52       # we're currently living in it.
  53       glsl_fname = 'shader{0}.{1}'.format(self.line, self.ext)
  54       spirv_fname = self.ext + '.spv'
  55
  56       glsl_file = open(glsl_fname, 'w')
  57       glsl_file.write('#version 420 core\n')
  58       glsl_file.write(self.glsl_source())
  59       glsl_file.close()
  60
  61       out = open('glslang.out', 'wb')
  62       err = subprocess.call([glslang, '-V', glsl_fname], stdout=out)
  63       if err != 0:
  64          out = open('glslang.out', 'r')
  65          sys.stderr.write(out.read())
  66          out.close()
  67          exit(1)
  68
  69       def dwords(f):
  70          while True:
  71             dword_str = f.read(4)
  72             if not dword_str:
  73                return
  74             assert len(dword_str) == 4
  75             yield struct.unpack('I', dword_str)[0]
  76
  77       spirv_file = open(spirv_fname, 'rb')
  78       self.dwords = list(dwords(spirv_file))
  79       spirv_file.close()
  80
  81       os.remove(glsl_fname)
  82       os.remove(spirv_fname)
  83
  84    def dump_c_code(self, f, glsl_only = False):
  85       f.write('\n\n')
  86       var_prefix = '_glsl_helpers_shader{0}'.format(self.line)
  87
  88       # First dump the GLSL source as strings
  89       f.write('static const char {0}_glsl_src[] ='.format(var_prefix))
  90       f.write('\n"#version 330\\n"')
  91       for line in self.glsl_source().splitlines():
  92          if not line.strip():
  93             continue
  94          f.write('\n"{0}\\n"'.format(line))
  95       f.write(';\n\n')
  96
  97       if glsl_only:
  98          return
  99
 100       # Now dump the SPIR-V source
 101       f.write('static const uint32_t {0}_spir_v_src[] = {{'.format(var_prefix))
 102       line_start = 0
 103       while line_start < len(self.dwords):
 104          f.write('\n   ')
 105          for i in range(line_start, min(line_start + 6, len(self.dwords))):
 106             f.write(' 0x{:08x},'.format(self.dwords[i]))
 107          line_start += 6
 108       f.write('\n};\n')
 109
 110 token_exp = re.compile(r'(GLSL_VK_SHADER|\(|\)|,)')
 111
 112 class Parser:
 113    def __init__(self, f):
 114       self.infile = f
 115       self.paren_depth = 0
 116       self.shader = None
 117       self.line_number = 1
 118       self.shaders = []
 119
 120       def tokenize(f):
 121          leftover = ''
 122          for line in f:
 123             pos = 0
 124             while True:
 125                m = token_exp.search(line, pos)
 126                if m:
 127                   if m.start() > pos:
 128                      leftover += line[pos:m.start()]
 129                   pos = m.end()
 130
 131                   if leftover:
 132                      yield leftover
 133                      leftover = ''
 134
 135                   yield m.group(0)
 136
 137                else:
 138                   leftover += line[pos:]
 139                   break
 140
 141             self.line_number += 1
 142
 143          if leftover:
 144             yield leftover
 145
 146       self.token_iter = tokenize(self.infile)
 147
 148    def handle_shader_src(self):
 149       paren_depth = 1
 150       for t in self.token_iter:
 151          if t == '(':
 152             paren_depth += 1
 153          elif t == ')':
 154             paren_depth -= 1
 155             if paren_depth == 0:
 156                return
 157
 158          self.current_shader.add_text(t)
 159
 160    def handle_macro(self):
 161       t = self.token_iter.next()
 162       assert t == '('
 163       t = self.token_iter.next()
 164       t = self.token_iter.next()
 165       assert t == ','
 166
 167       stage = self.token_iter.next().strip()
 168
 169       t = self.token_iter.next()
 170       assert t == ','
 171
 172       self.current_shader = Shader(stage)
 173       self.handle_shader_src()
 174       self.current_shader.finish_text(self.line_number)
 175
 176       self.shaders.append(self.current_shader)
 177       self.current_shader = None
 178
 179    def run(self):
 180       for t in self.token_iter:
 181          if t == 'GLSL_VK_SHADER':
 182             self.handle_macro()
 183
 184 def open_file(name, mode):
 185    if name == '-':
 186       if mode == 'w':
 187          return sys.stdout
 188       elif mode == 'r':
 189          return sys.stdin
 190       else:
 191          assert False
 192    else:
 193       return open(name, mode)
 194
 195 infname = None
 196 outfname = '-'
 197 glslang = 'glslangValidator'
 198 glsl_only = False
 199
 200 arg_idx = 1
 201 while arg_idx < len(sys.argv):
 202    if sys.argv[arg_idx] == '-h':
 203       print_usage(0)
 204    elif sys.argv[arg_idx] == '-o':
 205       arg_idx += 1
 206       outfname = sys.argv[arg_idx]
 207    elif sys.argv[arg_idx].startswith('--with-glslang='):
 208       glslang = sys.argv[arg_idx][len('--with-glslang='):]
 209    elif sys.argv[arg_idx] == '--glsl-only':
 210       glsl_only = True;
 211    else:
 212       infname = sys.argv[arg_idx]
 213       break
 214    arg_idx += 1
 215
 216 if arg_idx < len(sys.argv) - 1 or not infname or not outfname:
 217    print_usage(1)
 218
 219 with open_file(infname, 'r') as infile:
 220    parser = Parser(infile)
 221    parser.run()
 222
 223 if not glsl_only:
 224    # glslangValidator has an absolutely *insane* interface.  We pretty much
 225    # have to run in a temporary directory.  Sad day.
 226    current_dir = os.getcwd()
 227    tmpdir = tempfile.mkdtemp('glsl_scraper')
 228
 229    try:
 230       os.chdir(tmpdir)
 231
 232       for shader in parser.shaders:
 233          shader.compile()
 234
 235       os.chdir(current_dir)
 236    finally:
 237       shutil.rmtree(tmpdir)
 238
 239 with open_file(outfname, 'w') as outfile:
 240    outfile.write("""\
 241 /* ===========================  DO NOT EDIT!  ===========================
 242  *            This file is autogenerated by glsl_scraper.py.
 243  */
 244
 245 #include <stdint.h>
 246
 247 #define _GLSL_SRC_VAR2(_line) _glsl_helpers_shader ## _line ## _glsl_src
 248 #define _GLSL_SRC_VAR(_line) _GLSL_SRC_VAR2(_line)
 249
 250 #define GLSL_VK_SHADER(device, stage, ...) ({                           \\
 251    VkShader __shader;                                                   \\
 252    VkShaderCreateInfo __shader_create_info = {                          \\
 253       .sType = VK_STRUCTURE_TYPE_SHADER_CREATE_INFO,                    \\
 254       .codeSize = sizeof(_GLSL_SRC_VAR(__LINE__)),                      \\
 255       .pCode = _GLSL_SRC_VAR(__LINE__),                                 \\
 256    };                                                                   \\
 257    vkCreateShader((VkDevice) device, &__shader_create_info, &__shader); \\
 258    __shader;                                                            \\
 259 })
 260 """)
 261
 262    for shader in parser.shaders:
 263       shader.dump_c_code(outfile, glsl_only)