From be0a4da6a571dc82dd491005c2cb01d5a3b09061 Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Tue, 1 Sep 2015 15:16:06 -0700 Subject: [PATCH] vk/meta: Use SPIR-V for shaders We are also now using glslc for compiling the Vulkan driver like we do in curcible. --- configure.ac | 6 + src/vulkan/Makefile.am | 2 +- src/vulkan/glsl_scraper.py | 432 ++++++++++++++++++------------------- 3 files changed, 212 insertions(+), 228 deletions(-) diff --git a/configure.ac b/configure.ac index 0c496136d3d..5828885e92a 100644 --- a/configure.ac +++ b/configure.ac @@ -99,6 +99,7 @@ AM_PROG_CC_C_O AM_PROG_AS AX_CHECK_GNU_MAKE AC_CHECK_PROGS([PYTHON2], [python2 python]) +AC_CHECK_PROGS([PYTHON3], [python3]) AC_PROG_SED AC_PROG_MKDIR_P @@ -1523,6 +1524,10 @@ AC_SUBST([GBM_PC_LIB_PRIV]) AM_CONDITIONAL(HAVE_VULKAN, true) +AC_ARG_VAR([GLSLC], [Path to the glslc executable]) +AC_CHECK_PROGS([GLSLC], [glslc]) +AC_SUBST([GLSLC]) + dnl dnl EGL configuration dnl @@ -2533,6 +2538,7 @@ if test "x$MESA_LLVM" = x1; then echo "" fi echo " PYTHON2: $PYTHON2" +echo " PYTHON3: $PYTHON3" echo "" echo " Run '${MAKE-make}' to build Mesa" diff --git a/src/vulkan/Makefile.am b/src/vulkan/Makefile.am index 89880b77c01..13897090287 100644 --- a/src/vulkan/Makefile.am +++ b/src/vulkan/Makefile.am @@ -96,7 +96,7 @@ anv_entrypoints.c : anv_entrypoints_gen.py $(vulkan_include_HEADERS) $(AM_V_GEN)cat $(vulkan_include_HEADERS) | $(PYTHON2) $< code > $@ %_spirv_autogen.h: %.c glsl_scraper.py - $(AM_V_GEN) $(PYTHON2) $(srcdir)/glsl_scraper.py --glsl-only -o $@ $< + $(AM_V_GEN) $(PYTHON3) $(srcdir)/glsl_scraper.py --with-glslc=$(GLSLC) -o $@ $< CLEANFILES = $(BUILT_SOURCES) diff --git a/src/vulkan/glsl_scraper.py b/src/vulkan/glsl_scraper.py index d1514712762..4963742ea36 100644 --- a/src/vulkan/glsl_scraper.py +++ b/src/vulkan/glsl_scraper.py @@ -1,7 +1,7 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 import argparse -import cStringIO +import io import os import re import shutil @@ -11,231 +11,223 @@ import sys import tempfile from textwrap import dedent +class ShaderCompileError(RuntimeError): + def __init__(self): + super(ShaderCompileError, self).__init__('Compile error') + class Shader: - def __init__(self, stage): - self.stream = cStringIO.StringIO() - self.stage = stage - - if self.stage == 'VERTEX': - self.ext = 'vert' - elif self.stage == 'TESS_CONTROL': - self.ext = 'tesc' - elif self.stage == 'TESS_EVALUATION': - self.ext = 'tese' - elif self.stage == 'GEOMETRY': - self.ext = 'geom' - elif self.stage == 'FRAGMENT': - self.ext = 'frag' - elif self.stage == 'COMPUTE': - self.ext = 'comp' - else: - assert False - - def add_text(self, s): - self.stream.write(s) - - def finish_text(self, line): - self.line = line - - def glsl_source(self): - return self.stream.getvalue() - - def compile(self): - # We can assume if we got here that we have a temp directory and that - # we're currently living in it. - glsl_fname = 'shader{0}.{1}'.format(self.line, self.ext) - spirv_fname = self.ext + '.spv' - - glsl_file = open(glsl_fname, 'w') - glsl_file.write('#version 420 core\n') - glsl_file.write(self.glsl_source()) - glsl_file.close() - - out = open('glslang.out', 'wb') - err = subprocess.call([glslang, '-V', glsl_fname], stdout=out) - if err != 0: - out = open('glslang.out', 'r') - sys.stderr.write(out.read()) - out.close() - exit(1) - - def dwords(f): - while True: - dword_str = f.read(4) - if not dword_str: - return - assert len(dword_str) == 4 - yield struct.unpack('I', dword_str)[0] - - spirv_file = open(spirv_fname, 'rb') - self.dwords = list(dwords(spirv_file)) - spirv_file.close() - - os.remove(glsl_fname) - os.remove(spirv_fname) - - def dump_c_code(self, f, glsl_only = False): - f.write('\n\n') - var_prefix = '_glsl_helpers_shader{0}'.format(self.line) - - # First dump the GLSL source as strings - f.write('static const char {0}_glsl_src[] ='.format(var_prefix)) - f.write('\n_ANV_SPIRV_' + self.stage) - f.write('\n"#version 330\\n"') - for line in self.glsl_source().splitlines(): - if not line.strip(): - continue - f.write('\n"{0}\\n"'.format(line)) - f.write(';\n\n') - - if glsl_only: - return - - # Now dump the SPIR-V source - f.write('static const uint32_t {0}_spir_v_src[] = {{'.format(var_prefix)) - line_start = 0 - while line_start < len(self.dwords): - f.write('\n ') - for i in range(line_start, min(line_start + 6, len(self.dwords))): - f.write(' 0x{:08x},'.format(self.dwords[i])) - line_start += 6 - f.write('\n};\n') + def __init__(self, stage): + self.stream = io.StringIO() + self.stage = stage + self.dwords = None + + def add_text(self, s): + self.stream.write(s) + + def finish_text(self, line): + self.line = line + + def glsl_source(self): + return dedent(self.stream.getvalue()) + + def __run_glslc(self, extra_args=[]): + stage_flag = '-fshader-stage=' + if self.stage == 'VERTEX': + stage_flag += 'vertex' + elif self.stage == 'TESS_CONTROL': + stage_flag += 'tesscontrol' + elif self.stage == 'TESS_EVALUATION': + stage_flag += 'tesseval' + elif self.stage == 'GEOMETRY': + stage_flag += 'geometry' + elif self.stage == 'FRAGMENT': + stage_flag += 'fragment' + elif self.stage == 'COMPUTE': + stage_flag += 'compute' + else: + assert False + + with subprocess.Popen([glslc] + extra_args + + [stage_flag, '-std=430core', '-o', '-', '-'], + stdout = subprocess.PIPE, + stdin = subprocess.PIPE) as proc: + + proc.stdin.write(self.glsl_source().encode('utf-8')) + out, err = proc.communicate(timeout=30) + + if proc.returncode != 0: + raise ShaderCompileError() + + return out + + def compile(self): + def dwords(f): + while True: + dword_str = f.read(4) + if not dword_str: + return + assert len(dword_str) == 4 + yield struct.unpack('I', dword_str)[0] + + spirv = self.__run_glslc() + self.dwords = list(dwords(io.BytesIO(spirv))) + self.assembly = str(self.__run_glslc(['-S']), 'utf-8') + + def dump_c_code(self, f): + f.write('\n\n') + prefix = '_anv_glsl_helpers_shader{0}'.format(self.line) + + f.write('/* GLSL Source code:\n') + for line in self.glsl_source().splitlines(): + f.write(' * ' + line + '\n') + + f.write(' *\n') + + f.write(' * SPIR-V Assembly:\n') + f.write(' *\n') + for line in self.assembly.splitlines(): + f.write(' * ' + line + '\n') + f.write(' */\n') + + f.write('static const uint32_t {0}_spirv_code[] = {{'.format(prefix)) + line_start = 0 + while line_start < len(self.dwords): + f.write('\n ') + for i in range(line_start, min(line_start + 6, len(self.dwords))): + f.write(' 0x{:08x},'.format(self.dwords[i])) + line_start += 6 + f.write('\n};\n') + + f.write(dedent("""\ + static const VkShaderModuleCreateInfo {0}_info = {{ + .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, + .codeSize = sizeof({0}_spirv_code), + .pCode = {0}_spirv_code, + }}; + """.format(prefix))) token_exp = re.compile(r'(GLSL_VK_SHADER_MODULE|\(|\)|,)') class Parser: - def __init__(self, f): - self.infile = f - self.paren_depth = 0 - self.shader = None - self.line_number = 1 - self.shaders = [] - - def tokenize(f): - leftover = '' - for line in f: - pos = 0 - while True: - m = token_exp.search(line, pos) - if m: - if m.start() > pos: - leftover += line[pos:m.start()] - pos = m.end() - - if leftover: - yield leftover - leftover = '' - - yield m.group(0) - - else: - leftover += line[pos:] - break - - self.line_number += 1 - - if leftover: - yield leftover - - self.token_iter = tokenize(self.infile) - - def handle_shader_src(self): - paren_depth = 1 - for t in self.token_iter: - if t == '(': - paren_depth += 1 - elif t == ')': - paren_depth -= 1 - if paren_depth == 0: - return - - self.current_shader.add_text(t) - - def handle_macro(self): - t = self.token_iter.next() - assert t == '(' - t = self.token_iter.next() - t = self.token_iter.next() - assert t == ',' + def __init__(self, f): + self.infile = f + self.paren_depth = 0 + self.shader = None + self.line_number = 1 + self.shaders = [] + + def tokenize(f): + leftover = '' + for line in f: + pos = 0 + while True: + m = token_exp.search(line, pos) + if m: + if m.start() > pos: + leftover += line[pos:m.start()] + pos = m.end() + + if leftover: + yield leftover + leftover = '' + + yield m.group(0) + + else: + leftover += line[pos:] + break + + self.line_number += 1 + + if leftover: + yield leftover + + self.token_iter = tokenize(self.infile) + + def handle_shader_src(self): + paren_depth = 1 + for t in self.token_iter: + if t == '(': + paren_depth += 1 + elif t == ')': + paren_depth -= 1 + if paren_depth == 0: + return + + self.current_shader.add_text(t) + + def handle_macro(self, macro): + t = next(self.token_iter) + assert t == '(' + + # Throw away the device parameter + t = next(self.token_iter) + t = next(self.token_iter) + assert t == ',' + + stage = next(self.token_iter).strip() + + t = next(self.token_iter) + assert t == ',' + + self.current_shader = Shader(stage) + self.handle_shader_src() + self.current_shader.finish_text(self.line_number) + + self.shaders.append(self.current_shader) + self.current_shader = None + + def run(self): + for t in self.token_iter: + if t == 'GLSL_VK_SHADER_MODULE': + self.handle_macro(t) - stage = self.token_iter.next().strip() - - t = self.token_iter.next() - assert t == ',' - - self.current_shader = Shader(stage) - self.handle_shader_src() - self.current_shader.finish_text(self.line_number) +def open_file(name, mode): + if name == '-': + if mode == 'w': + return sys.stdout + elif mode == 'r': + return sys.stdin + else: + assert False + else: + return open(name, mode) - self.shaders.append(self.current_shader) - self.current_shader = None +def parse_args(): + description = dedent("""\ + This program scrapes a C file for any instance of the + qoShaderCreateInfoGLSL and qoCreateShaderGLSL macaros, grabs the + GLSL source code, compiles it to SPIR-V. The resulting SPIR-V code + is written to another C file as an array of 32-bit words. - def run(self): - for t in self.token_iter: - if t == 'GLSL_VK_SHADER_MODULE': - self.handle_macro() + If '-' is passed as the input file or output file, stdin or stdout + will be used instead of a file on disc.""") -def open_file(name, mode): - if name == '-': - if mode == 'w': - return sys.stdout - elif mode == 'r': - return sys.stdin - else: - assert False - else: - return open(name, mode) + p = argparse.ArgumentParser( + description=description, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('-o', '--outfile', default='-', + help='Output to the given file (default: stdout).') + p.add_argument('--with-glslc', metavar='PATH', + default='glslc', + dest='glslc', + help='Full path to the glslc shader compiler.') + p.add_argument('infile', metavar='INFILE') -def parse_args(): - description = dedent("""\ - This program scrapes a C file for any instance of the - GLSL_VK_SHADER_MODULE macro, grabs the GLSL source code, compiles it - to SPIR-V. The resulting SPIR-V code is written to another C file as - an array of 32-bit words. - - If '-' is passed as the input file or output file, stdin or stdout will be - used instead of a file on disc.""") - - p = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument('-o', '--outfile', default='-', - help='Output to the given file (default: stdout).') - p.add_argument('--with-glslang', metavar='PATH', - default='glslangValidator', - dest='glslang', - help='Full path to the glslangValidator program.') - p.add_argument('--glsl-only', action='store_true') - p.add_argument('infile', metavar='INFILE') - - return p.parse_args() + return p.parse_args() args = parse_args() infname = args.infile outfname = args.outfile -glslang = args.glslang -glsl_only = args.glsl_only +glslc = args.glslc with open_file(infname, 'r') as infile: - parser = Parser(infile) - parser.run() + parser = Parser(infile) + parser.run() -if not glsl_only: - # glslangValidator has an absolutely *insane* interface. We pretty much - # have to run in a temporary directory. Sad day. - current_dir = os.getcwd() - tmpdir = tempfile.mkdtemp('glsl_scraper') - - try: - os.chdir(tmpdir) - - for shader in parser.shaders: - shader.compile() - - os.chdir(current_dir) - finally: - shutil.rmtree(tmpdir) +for shader in parser.shaders: + shader.compile() with open_file(outfname, 'w') as outfile: outfile.write(dedent("""\ @@ -245,30 +237,16 @@ with open_file(outfname, 'w') as outfile: #include - #define _ANV_SPIRV_MAGIC "\\x03\\x02\\x23\\x07\\0\\0\\0\\0" - - #define _ANV_SPIRV_VERTEX _ANV_SPIRV_MAGIC "\\0\\0\\0\\0" - #define _ANV_SPIRV_TESS_CONTROL _ANV_SPIRV_MAGIC "\\1\\0\\0\\0" - #define _ANV_SPIRV_TESS_EVALUATION _ANV_SPIRV_MAGIC "\\2\\0\\0\\0" - #define _ANV_SPIRV_GEOMETRY _ANV_SPIRV_MAGIC "\\3\\0\\0\\0" - #define _ANV_SPIRV_FRAGMENT _ANV_SPIRV_MAGIC "\\4\\0\\0\\0" - #define _ANV_SPIRV_COMPUTE _ANV_SPIRV_MAGIC "\\5\\0\\0\\0" - - #define _ANV_GLSL_SRC_VAR2(_line) _glsl_helpers_shader ## _line ## _glsl_src - #define _ANV_GLSL_SRC_VAR(_line) _ANV_GLSL_SRC_VAR2(_line) + #define _ANV_SPIRV_MODULE_INFO2(_line) _anv_glsl_helpers_shader ## _line ## _info + #define _ANV_SPIRV_MODULE_INFO(_line) _ANV_SPIRV_MODULE_INFO2(_line) #define GLSL_VK_SHADER_MODULE(device, stage, ...) ({ \\ VkShaderModule __module; \\ - VkShaderModuleCreateInfo __shader_create_info = { \\ - .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, \\ - .codeSize = sizeof(_ANV_GLSL_SRC_VAR(__LINE__)), \\ - .pCode = _ANV_GLSL_SRC_VAR(__LINE__), \\ - }; \\ vkCreateShaderModule(anv_device_to_handle(device), \\ - &__shader_create_info, &__module); \\ + &_ANV_SPIRV_MODULE_INFO(__LINE__), &__module); \\ __module; \\ }) """)) for shader in parser.shaders: - shader.dump_c_code(outfile, glsl_only) + shader.dump_c_code(outfile) -- 2.30.2