From deb04adf2ae605a017d7ce4e81f57db679567dfa Mon Sep 17 00:00:00 2001 From: Karol Herbst Date: Tue, 6 Aug 2019 20:35:48 +0200 Subject: [PATCH] clover: add support for passing kernels as nir to the driver v2: minor formatting fixes v3: call glsl_type_singleton_init_or_ref and glsl_type_singleton_decref v4: capitalize and punctuate comments fix text_executable -> text_intermediate in TODO make glsl_type_singleton wrapper static v5: rewrite how we run the nir passes v6: fix unhandled case switch warning in st/mesa Signed-off-by: Karol Herbst Reviewed-by: Francisco Jerez (v4) --- src/gallium/include/pipe/p_defines.h | 1 + .../state_trackers/clover/core/compiler.hpp | 9 + .../state_trackers/clover/core/device.cpp | 26 ++- .../state_trackers/clover/core/device.hpp | 1 + src/gallium/state_trackers/clover/meson.build | 11 +- .../state_trackers/clover/nir/invocation.cpp | 169 ++++++++++++++++++ .../state_trackers/clover/nir/invocation.hpp | 36 ++++ src/mesa/state_tracker/st_program.c | 3 + 8 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 src/gallium/state_trackers/clover/nir/invocation.cpp create mode 100644 src/gallium/state_trackers/clover/nir/invocation.hpp diff --git a/src/gallium/include/pipe/p_defines.h b/src/gallium/include/pipe/p_defines.h index de0d88f6548..0d4adcf6f4b 100644 --- a/src/gallium/include/pipe/p_defines.h +++ b/src/gallium/include/pipe/p_defines.h @@ -1004,6 +1004,7 @@ enum pipe_shader_ir PIPE_SHADER_IR_TGSI = 0, PIPE_SHADER_IR_NATIVE, PIPE_SHADER_IR_NIR, + PIPE_SHADER_IR_NIR_SERIALIZED, }; /** diff --git a/src/gallium/state_trackers/clover/core/compiler.hpp b/src/gallium/state_trackers/clover/core/compiler.hpp index 4ed20061627..6ef30df9b7f 100644 --- a/src/gallium/state_trackers/clover/core/compiler.hpp +++ b/src/gallium/state_trackers/clover/core/compiler.hpp @@ -26,6 +26,8 @@ #include "core/device.hpp" #include "core/module.hpp" #include "llvm/invocation.hpp" +#include "nir/invocation.hpp" +#include "spirv/invocation.hpp" namespace clover { namespace compiler { @@ -34,6 +36,10 @@ namespace clover { const device &dev, const std::string &opts, std::string &log) { switch (dev.ir_format()) { +#ifdef HAVE_CLOVER_SPIRV + case PIPE_SHADER_IR_NIR_SERIALIZED: + return llvm::compile_to_spirv(source, headers, dev, opts, log); +#endif case PIPE_SHADER_IR_NATIVE: return llvm::compile_program(source, headers, dev, opts, log); default: @@ -46,6 +52,9 @@ namespace clover { link_program(const std::vector &ms, const device &dev, const std::string &opts, std::string &log) { switch (dev.ir_format()) { + case PIPE_SHADER_IR_NIR_SERIALIZED: + return nir::spirv_to_nir(spirv::link_program(ms, dev, opts, log), + dev, log); case PIPE_SHADER_IR_NATIVE: return llvm::link_program(ms, dev, opts, log); default: diff --git a/src/gallium/state_trackers/clover/core/device.cpp b/src/gallium/state_trackers/clover/core/device.cpp index c7aea827938..298cde278b4 100644 --- a/src/gallium/state_trackers/clover/core/device.cpp +++ b/src/gallium/state_trackers/clover/core/device.cpp @@ -46,12 +46,17 @@ namespace { device::device(clover::platform &platform, pipe_loader_device *ldev) : platform(platform), ldev(ldev) { pipe = pipe_loader_create_screen(ldev); - if (!pipe || !pipe->get_param(pipe, PIPE_CAP_COMPUTE) || - !supports_ir(PIPE_SHADER_IR_NATIVE)) { - if (pipe) - pipe->destroy(pipe); - throw error(CL_INVALID_DEVICE); + if (pipe && pipe->get_param(pipe, PIPE_CAP_COMPUTE)) { + if (supports_ir(PIPE_SHADER_IR_NATIVE)) + return; +#ifdef HAVE_CLOVER_SPIRV + if (supports_ir(PIPE_SHADER_IR_NIR_SERIALIZED)) + return; +#endif } + if (pipe) + pipe->destroy(pipe); + throw error(CL_INVALID_DEVICE); } device::~device() { @@ -246,7 +251,11 @@ device::vendor_name() const { enum pipe_shader_ir device::ir_format() const { - return PIPE_SHADER_IR_NATIVE; + if (supports_ir(PIPE_SHADER_IR_NATIVE)) + return PIPE_SHADER_IR_NATIVE; + + assert(supports_ir(PIPE_SHADER_IR_NIR_SERIALIZED)); + return PIPE_SHADER_IR_NIR_SERIALIZED; } std::string @@ -294,3 +303,8 @@ device::supported_extensions() const { + std::string(has_doubles() ? " cl_khr_fp64" : "") + std::string(has_halves() ? " cl_khr_fp16" : ""); } + +const void * +device::get_compiler_options(enum pipe_shader_ir ir) const { + return pipe->get_compiler_options(pipe, ir, PIPE_SHADER_COMPUTE); +} diff --git a/src/gallium/state_trackers/clover/core/device.hpp b/src/gallium/state_trackers/clover/core/device.hpp index a7084e863f4..828dfaa65e6 100644 --- a/src/gallium/state_trackers/clover/core/device.hpp +++ b/src/gallium/state_trackers/clover/core/device.hpp @@ -90,6 +90,7 @@ namespace clover { friend class hard_event; friend std::set supported_formats(const context &, cl_mem_object_type); + const void *get_compiler_options(enum pipe_shader_ir ir) const; clover::platform &platform; diff --git a/src/gallium/state_trackers/clover/meson.build b/src/gallium/state_trackers/clover/meson.build index 13d554c3fbf..04c6f1ea0eb 100644 --- a/src/gallium/state_trackers/clover/meson.build +++ b/src/gallium/state_trackers/clover/meson.build @@ -66,6 +66,15 @@ libclspirv = static_library( override_options : clover_cpp_std, ) +libclnir = static_library( + 'clnir', + [files('nir/invocation.cpp', 'nir/invocation.hpp'), nir_opcodes_h], + include_directories : [clover_incs, inc_mesa], + cpp_args : [clover_spirv_cpp_args, cpp_vis_args], + link_with : [libnir], + override_options : clover_cpp_std, +) + clover_files = files( 'api/context.cpp', 'api/device.cpp', @@ -127,6 +136,6 @@ libclover = static_library( [clover_files, sha1_h], include_directories : clover_incs, cpp_args : [clover_spirv_cpp_args, clover_cpp_args, cpp_vis_args], - link_with : [libclllvm, libclspirv], + link_with : [libclllvm, libclspirv, libclnir], override_options : clover_cpp_std, ) diff --git a/src/gallium/state_trackers/clover/nir/invocation.cpp b/src/gallium/state_trackers/clover/nir/invocation.cpp new file mode 100644 index 00000000000..0ec1c9a3c01 --- /dev/null +++ b/src/gallium/state_trackers/clover/nir/invocation.cpp @@ -0,0 +1,169 @@ +// +// Copyright 2019 Karol Herbst +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "invocation.hpp" + +#include + +#include "core/device.hpp" +#include "core/error.hpp" +#include "pipe/p_state.h" +#include "util/algorithm.hpp" +#include "util/functional.hpp" + +#include +#include +#include +#include + +using namespace clover; + +#ifdef HAVE_CLOVER_SPIRV + +// Refs and unrefs the glsl_type_singleton. +static class glsl_type_ref { +public: + glsl_type_ref() { + glsl_type_singleton_init_or_ref(); + } + + ~glsl_type_ref() { + glsl_type_singleton_decref(); + } +} glsl_type_ref; + +static const nir_shader_compiler_options * +dev_get_nir_compiler_options(const device &dev) +{ + const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR); + return static_cast(co); +} + +module clover::nir::spirv_to_nir(const module &mod, const device &dev, + std::string &r_log) +{ + const struct spirv_to_nir_options spirv_options = { + .caps = { + .address = true, + .float64 = true, + .int8 = true, + .int16 = true, + .int64 = true, + .kernel = true, + }, + }; + + module m; + // We only insert one section. + assert(mod.secs.size() == 1); + auto §ion = mod.secs[0]; + + module::resource_id section_id = 0; + for (const auto &sym : mod.syms) { + assert(sym.section == 0); + + const auto *binary = + reinterpret_cast(section.data.data()); + const uint32_t *data = reinterpret_cast(binary->blob); + const size_t num_words = binary->num_bytes / 4; + const char *name = sym.name.c_str(); + auto *compiler_options = dev_get_nir_compiler_options(dev); + + nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0, + MESA_SHADER_KERNEL, name, + &spirv_options, compiler_options); + + nir->info.cs.local_size_variable = true; + nir_validate_shader(nir, "clover"); + + // Calculate input offsets. + unsigned offset = 0; + nir_foreach_variable_safe(var, &nir->inputs) { + offset = align(offset, glsl_get_cl_alignment(var->type)); + var->data.driver_location = offset; + offset += glsl_get_cl_size(var->type); + } + + // Inline all functions first. + // according to the comment on nir_inline_functions + NIR_PASS_V(nir, nir_lower_constant_initializers, nir_var_function_temp); + NIR_PASS_V(nir, nir_lower_returns); + NIR_PASS_V(nir, nir_inline_functions); + NIR_PASS_V(nir, nir_opt_deref); + + // Pick off the single entrypoint that we want. + foreach_list_typed_safe(nir_function, func, node, &nir->functions) { + if (!func->is_entrypoint) + exec_node_remove(&func->node); + } + assert(exec_list_length(&nir->functions) == 1); + + nir_validate_shader(nir, "clover after function inlining"); + + NIR_PASS_V(nir, nir_lower_constant_initializers, + static_cast(~nir_var_function_temp)); + + // copy propagate to prepare for lower_explicit_io + NIR_PASS_V(nir, nir_split_var_copies); + NIR_PASS_V(nir, nir_opt_copy_prop_vars); + NIR_PASS_V(nir, nir_lower_var_copies); + NIR_PASS_V(nir, nir_lower_vars_to_ssa); + NIR_PASS_V(nir, nir_opt_dce); + + nir_variable_mode modes = (nir_variable_mode)( + nir_var_shader_in | + nir_var_mem_global | + nir_var_mem_shared); + nir_address_format format = nir->info.cs.ptr_size == 64 ? + nir_address_format_64bit_global : nir_address_format_32bit_global; + NIR_PASS_V(nir, nir_lower_explicit_io, modes, format); + + NIR_PASS_V(nir, nir_lower_system_values); + if (compiler_options->lower_int64_options) + NIR_PASS_V(nir, nir_lower_int64, + compiler_options->lower_int64_options); + + NIR_PASS_V(nir, nir_opt_dce); + + struct blob blob; + blob_init(&blob); + nir_serialize(&blob, nir); + + const pipe_binary_program_header header { uint32_t(blob.size) }; + module::section text { section_id, module::section::text_executable, header.num_bytes, {} }; + text.data.insert(text.data.end(), reinterpret_cast(&header), + reinterpret_cast(&header) + sizeof(header)); + text.data.insert(text.data.end(), blob.data, blob.data + blob.size); + + m.syms.emplace_back(sym.name, section_id, 0, sym.args); + m.secs.push_back(text); + section_id++; + } + return m; +} +#else +module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log) +{ + r_log += "SPIR-V support in clover is not enabled.\n"; + throw error(CL_LINKER_NOT_AVAILABLE); +} +#endif diff --git a/src/gallium/state_trackers/clover/nir/invocation.hpp b/src/gallium/state_trackers/clover/nir/invocation.hpp new file mode 100644 index 00000000000..41407a79765 --- /dev/null +++ b/src/gallium/state_trackers/clover/nir/invocation.hpp @@ -0,0 +1,36 @@ +// +// Copyright 2019 Karol Herbst +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_NIR_INVOCATION_HPP +#define CLOVER_NIR_INVOCATION_HPP + +#include "core/module.hpp" + +namespace clover { + class device; + namespace nir { + // converts a given spirv module to nir + module spirv_to_nir(const module &mod, const device &dev, std::string &r_log); + } +} + +#endif diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c index df6c8e81f55..f3143cef95c 100644 --- a/src/mesa/state_tracker/st_program.c +++ b/src/mesa/state_tracker/st_program.c @@ -418,6 +418,9 @@ st_release_cp_variants(struct st_context *st, struct st_compute_program *stcp) /* ??? */ stcp->tgsi.prog = NULL; break; + case PIPE_SHADER_IR_NIR_SERIALIZED: + unreachable("serialized nirs aren't passed through st/mesa"); + break; } } } -- 2.30.2