From 41db8db0f2310f2620ef63a3c24ab5842fe88118 Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Thu, 14 May 2015 19:07:50 -0700 Subject: [PATCH] vk: Add a GLSL scraper utility This new utility, glsl_scraper.py scrapes C files for instances of the GLSL_VK_SHADER macro, pulls out the shader source, and compiles it to SPIR-V. The compilation is done using glslValidator. The result is then placed into another C file as arrays of dwords that can be easiliy handed to a Vulkan driver. --- src/vulkan/glsl_scraper.py | 236 +++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 src/vulkan/glsl_scraper.py diff --git a/src/vulkan/glsl_scraper.py b/src/vulkan/glsl_scraper.py new file mode 100644 index 00000000000..4b99ae0a3e2 --- /dev/null +++ b/src/vulkan/glsl_scraper.py @@ -0,0 +1,236 @@ +#! /usr/bin/env python + +def print_usage(err): + print """\ +glsl_scraper.py [options] file + +This program scrapes a C file for any instance of the GLSL_VK_SHADER macro, +grabs the GLSL source code, compiles it to SPIR-V. The resulting SPIR-V +code is written to another C file as an array of 32-bit words. + +If '-' is passed as the input file or output file, stdin or stdout will be +used instead of a file on disc. + +Options: + -o outfile Output to the given file (default: stdout) + --with-glslang=PATH Full path to the glslangValidator program""" + exit(err) + +import os, sys, re, cStringIO, tempfile, subprocess, struct, shutil + +class Shader: + def __init__(self, stage, line): + self.stream = cStringIO.StringIO() + self.stage = stage + self.line = line + + if self.stage == 'VERTEX': + self.ext = 'vert' + elif self.stage == 'TESS_CONTROL': + self.ext = 'tesc' + elif self.stage == 'TESS_EVALUATION': + self.ext = 'tese' + elif self.stage == 'GEOMETRY': + self.ext = 'geom' + elif self.stage == 'FRAGMENT': + self.ext = 'frag' + elif self.stage == 'COMPUTE': + self.ext = 'comp' + else: + assert False + + def add_text(self, s): + self.stream.write(s) + + def glsl_source(self): + return self.stream.getvalue() + + def compile(self): + # We can assume if we got here that we have a temp directory and that + # we're currently living in it. + glsl_fname = 'shader{0}.{1}'.format(self.line, self.ext) + spirv_fname = self.ext + '.spv' + + glsl_file = open(glsl_fname, 'w') + glsl_file.write('#version 330\n') + glsl_file.write(self.glsl_source()) + glsl_file.close() + + out = open('glslang.out', 'wb') + err = subprocess.call([glslang, '-V', glsl_fname], stdout=out) + if err != 0: + out = open('glslang.out', 'r') + sys.stderr.write(out.read()) + out.close() + exit(1) + + def dwords(f): + while True: + dword_str = f.read(4) + if not dword_str: + return + assert len(dword_str) == 4 + yield struct.unpack('I', dword_str)[0] + + spirv_file = open(spirv_fname, 'rb') + self.dwords = list(dwords(spirv_file)) + spirv_file.close() + + os.remove(glsl_fname) + os.remove(spirv_fname) + + def dump_c_code(self, f): + f.write('\n\n') + var_prefix = '_glsl_helpers_shader{0}'.format(self.line) + + # First dump the GLSL source as strings + f.write('static const char {0}_glsl_src[] ='.format(var_prefix)) + f.write('\n"#version 330\\n"') + for line in self.glsl_source().splitlines(): + if not line.strip(): + continue + f.write('\n"{0}\\n"'.format(line)) + f.write(';\n\n') + + # Now dump the SPIR-V source + f.write('static const uint32_t {0}_spir_v_src[] = {{'.format(var_prefix)) + line_start = 0 + while line_start < len(self.dwords): + f.write('\n ') + for i in range(line_start, min(line_start + 6, len(self.dwords))): + f.write(' 0x{:08x},'.format(self.dwords[i])) + line_start += 6 + f.write('\n};\n') + +token_exp = re.compile(r'(GLSL_VK_SHADER|\(|\)|,)') + +class Parser: + def __init__(self, f): + self.infile = f + self.paren_depth = 0 + self.shader = None + self.line_number = 1 + self.shaders = [] + + def tokenize(f): + leftover = '' + for line in f: + pos = 0 + while True: + m = token_exp.search(line, pos) + if m: + if m.start() > pos: + leftover += line[pos:m.start()] + pos = m.end() + + if leftover: + yield leftover + leftover = '' + + yield m.group(0) + + else: + leftover += line[pos:] + break + + self.line_number += 1 + + if leftover: + yield leftover + + self.token_iter = tokenize(self.infile) + + def handle_shader_src(self): + paren_depth = 1 + for t in self.token_iter: + if t == '(': + paren_depth += 1 + elif t == ')': + paren_depth -= 1 + if paren_depth == 0: + return + + self.current_shader.add_text(t) + + def handle_macro(self): + line_number = self.line_number + + t = self.token_iter.next() + assert t == '(' + t = self.token_iter.next() + t = self.token_iter.next() + assert t == ',' + + stage = self.token_iter.next().strip() + + t = self.token_iter.next() + assert t == ',' + + self.current_shader = Shader(stage, line_number) + self.handle_shader_src() + self.shaders.append(self.current_shader) + + def run(self): + for t in self.token_iter: + if t == 'GLSL_VK_SHADER': + self.handle_macro() + +def open_file(name, mode): + if name == '-': + if mode == 'w': + return sys.stdout + elif mode == 'r': + return sys.stdin + else: + assert False + else: + return open(name, mode) + +infile = None +outfile = sys.stdout +glslang = 'glslangValidator' + +arg_idx = 1 +while arg_idx < len(sys.argv): + if sys.argv[arg_idx] == '-h': + print_usage(0) + elif sys.argv[arg_idx] == '-o': + arg_idx += 1 + outfile = open_file(sys.argv[arg_idx], 'w') + elif sys.argv[arg_idx].startswith('--with-glslang='): + glslang = sys.argv[arg_idx][len('--with-glslang='):] + else: + infile = open_file(sys.argv[arg_idx], 'r') + break + arg_idx += 1 + +if arg_idx < len(sys.argv) - 1 or not infile or not outfile: + print_usage(1) + +parser = Parser(infile) +parser.run() + +# glslangValidator has an absolutely *insane* interface. We pretty much +# have to run in a temporary directory. Sad day. +current_dir = os.getcwd() +tmpdir = tempfile.mkdtemp('glsl_scraper') + +try: + os.chdir(tmpdir) + + for shader in parser.shaders: + shader.compile() + + os.chdir(current_dir) +finally: + shutil.rmtree(tmpdir) + +outfile.write("""\ +/* =========================== DO NOT EDIT! =========================== + * This file is autogenerated by glsl_scraper.py. + */ + +#include """) + +for shader in parser.shaders: + shader.dump_c_code(outfile) -- 2.30.2