924651433330e8acd732593c19fd8525d88e547b
[mesa.git] / src / gallium / frontends / clover / nir / invocation.cpp
1 //
2 // Copyright 2019 Karol Herbst
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a
5 // copy of this software and associated documentation files (the "Software"),
6 // to deal in the Software without restriction, including without limitation
7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 // and/or sell copies of the Software, and to permit persons to whom the
9 // Software is furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 // OTHER DEALINGS IN THE SOFTWARE.
21 //
22
23 #include "invocation.hpp"
24
25 #include <tuple>
26
27 #include "core/device.hpp"
28 #include "core/error.hpp"
29 #include "core/module.hpp"
30 #include "pipe/p_state.h"
31 #include "util/algorithm.hpp"
32 #include "util/functional.hpp"
33
34 #include <compiler/glsl_types.h>
35 #include <compiler/nir/nir_builder.h>
36 #include <compiler/nir/nir_serialize.h>
37 #include <compiler/spirv/nir_spirv.h>
38 #include <util/u_math.h>
39
40 using namespace clover;
41
42 #ifdef HAVE_CLOVER_SPIRV
43
44 // Refs and unrefs the glsl_type_singleton.
45 static class glsl_type_ref {
46 public:
47 glsl_type_ref() {
48 glsl_type_singleton_init_or_ref();
49 }
50
51 ~glsl_type_ref() {
52 glsl_type_singleton_decref();
53 }
54 } glsl_type_ref;
55
56 static const nir_shader_compiler_options *
57 dev_get_nir_compiler_options(const device &dev)
58 {
59 const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR);
60 return static_cast<const nir_shader_compiler_options*>(co);
61 }
62
63 static void debug_function(void *private_data,
64 enum nir_spirv_debug_level level, size_t spirv_offset,
65 const char *message)
66 {
67 assert(private_data);
68 auto r_log = reinterpret_cast<std::string *>(private_data);
69 *r_log += message;
70 }
71
72 struct clover_lower_nir_state {
73 std::vector<module::argument> &args;
74 uint32_t global_dims;
75 nir_variable *offset_vars[3];
76 };
77
78 static bool
79 clover_lower_nir_filter(const nir_instr *instr, const void *)
80 {
81 return instr->type == nir_instr_type_intrinsic;
82 }
83
84 static nir_ssa_def *
85 clover_lower_nir_instr(nir_builder *b, nir_instr *instr, void *_state)
86 {
87 clover_lower_nir_state *state = reinterpret_cast<clover_lower_nir_state*>(_state);
88 nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
89
90 switch (intrinsic->intrinsic) {
91 case nir_intrinsic_load_base_global_invocation_id: {
92 nir_ssa_def *loads[3];
93
94 /* create variables if we didn't do so alrady */
95 if (!state->offset_vars[0]) {
96 /* TODO: fix for 64 bit */
97 /* Even though we only place one scalar argument, clover will bind up to
98 * three 32 bit values
99 */
100 state->args.emplace_back(module::argument::scalar, 4, 4, 4,
101 module::argument::zero_ext,
102 module::argument::grid_offset);
103
104 const glsl_type *type = glsl_uint_type();
105 for (uint32_t i = 0; i < 3; i++) {
106 state->offset_vars[i] =
107 nir_variable_create(b->shader, nir_var_uniform, type,
108 "global_invocation_id_offsets");
109 state->offset_vars[i]->data.location = b->shader->num_uniforms++;
110 }
111 }
112
113 for (int i = 0; i < 3; i++) {
114 nir_variable *var = state->offset_vars[i];
115 loads[i] = var ? nir_load_var(b, var) : nir_imm_int(b, 0);
116 }
117
118 return nir_u2u(b, nir_vec(b, loads, state->global_dims),
119 nir_dest_bit_size(intrinsic->dest));
120 }
121 default:
122 return NULL;
123 }
124 }
125
126 static bool
127 clover_lower_nir(nir_shader *nir, std::vector<module::argument> &args, uint32_t dims)
128 {
129 clover_lower_nir_state state = { args, dims };
130 return nir_shader_lower_instructions(nir,
131 clover_lower_nir_filter, clover_lower_nir_instr, &state);
132 }
133
134 module clover::nir::spirv_to_nir(const module &mod, const device &dev,
135 std::string &r_log)
136 {
137 struct spirv_to_nir_options spirv_options = {};
138 spirv_options.environment = NIR_SPIRV_OPENCL;
139 if (dev.address_bits() == 32u) {
140 spirv_options.shared_addr_format = nir_address_format_32bit_offset;
141 spirv_options.global_addr_format = nir_address_format_32bit_global;
142 spirv_options.temp_addr_format = nir_address_format_32bit_offset;
143 spirv_options.constant_addr_format = nir_address_format_32bit_global;
144 } else {
145 spirv_options.shared_addr_format = nir_address_format_32bit_offset_as_64bit;
146 spirv_options.global_addr_format = nir_address_format_64bit_global;
147 spirv_options.temp_addr_format = nir_address_format_32bit_offset_as_64bit;
148 spirv_options.constant_addr_format = nir_address_format_64bit_global;
149 }
150 spirv_options.caps.address = true;
151 spirv_options.caps.float64 = true;
152 spirv_options.caps.int8 = true;
153 spirv_options.caps.int16 = true;
154 spirv_options.caps.int64 = true;
155 spirv_options.caps.kernel = true;
156 spirv_options.caps.int64_atomics = dev.has_int64_atomics();
157 spirv_options.debug.func = &debug_function;
158 spirv_options.debug.private_data = &r_log;
159
160 module m;
161 // We only insert one section.
162 assert(mod.secs.size() == 1);
163 auto &section = mod.secs[0];
164
165 module::resource_id section_id = 0;
166 for (const auto &sym : mod.syms) {
167 assert(sym.section == 0);
168
169 const auto *binary =
170 reinterpret_cast<const pipe_binary_program_header *>(section.data.data());
171 const uint32_t *data = reinterpret_cast<const uint32_t *>(binary->blob);
172 const size_t num_words = binary->num_bytes / 4;
173 const char *name = sym.name.c_str();
174 auto *compiler_options = dev_get_nir_compiler_options(dev);
175
176 nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0,
177 MESA_SHADER_KERNEL, name,
178 &spirv_options, compiler_options);
179 if (!nir) {
180 r_log += "Translation from SPIR-V to NIR for kernel \"" + sym.name +
181 "\" failed.\n";
182 throw build_error();
183 }
184
185 nir->info.cs.local_size_variable = true;
186 nir_validate_shader(nir, "clover");
187
188 // Inline all functions first.
189 // according to the comment on nir_inline_functions
190 NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
191 NIR_PASS_V(nir, nir_lower_returns);
192 NIR_PASS_V(nir, nir_inline_functions);
193 NIR_PASS_V(nir, nir_copy_prop);
194 NIR_PASS_V(nir, nir_opt_deref);
195
196 // Pick off the single entrypoint that we want.
197 foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
198 if (!func->is_entrypoint)
199 exec_node_remove(&func->node);
200 }
201 assert(exec_list_length(&nir->functions) == 1);
202
203 nir_validate_shader(nir, "clover after function inlining");
204
205 NIR_PASS_V(nir, nir_lower_variable_initializers, ~nir_var_function_temp);
206
207 // copy propagate to prepare for lower_explicit_io
208 NIR_PASS_V(nir, nir_split_var_copies);
209 NIR_PASS_V(nir, nir_opt_copy_prop_vars);
210 NIR_PASS_V(nir, nir_lower_var_copies);
211 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
212 NIR_PASS_V(nir, nir_opt_dce);
213
214 NIR_PASS_V(nir, nir_lower_system_values);
215 nir_lower_compute_system_values_options sysval_options = { 0 };
216 sysval_options.has_base_global_invocation_id = true;
217 NIR_PASS_V(nir, nir_lower_compute_system_values, &sysval_options);
218
219 auto args = sym.args;
220 NIR_PASS_V(nir, clover_lower_nir, args, dev.max_block_size().size());
221
222 // Calculate input offsets.
223 unsigned offset = 0;
224 nir_foreach_uniform_variable(var, nir) {
225 offset = align(offset, glsl_get_cl_alignment(var->type));
226 var->data.driver_location = offset;
227 offset += glsl_get_cl_size(var->type);
228 }
229
230 NIR_PASS_V(nir, nir_lower_vars_to_explicit_types,
231 nir_var_mem_shared | nir_var_function_temp,
232 glsl_get_cl_type_size_align);
233
234 /* use offsets for uniform and shared memory */
235 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform,
236 nir_address_format_32bit_offset);
237
238 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant,
239 spirv_options.constant_addr_format);
240 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_shared,
241 spirv_options.shared_addr_format);
242
243 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_function_temp,
244 spirv_options.temp_addr_format);
245
246 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_global,
247 spirv_options.global_addr_format);
248
249 NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_all, NULL);
250
251 if (compiler_options->lower_int64_options)
252 NIR_PASS_V(nir, nir_lower_int64);
253
254 NIR_PASS_V(nir, nir_opt_dce);
255
256 struct blob blob;
257 blob_init(&blob);
258 nir_serialize(&blob, nir, false);
259
260 const pipe_binary_program_header header { uint32_t(blob.size) };
261 module::section text { section_id, module::section::text_executable, header.num_bytes, {} };
262 text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
263 reinterpret_cast<const char *>(&header) + sizeof(header));
264 text.data.insert(text.data.end(), blob.data, blob.data + blob.size);
265
266 m.syms.emplace_back(sym.name, section_id, 0, args);
267 m.secs.push_back(text);
268 section_id++;
269 }
270 return m;
271 }
272 #else
273 module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log)
274 {
275 r_log += "SPIR-V support in clover is not enabled.\n";
276 throw error(CL_LINKER_NOT_AVAILABLE);
277 }
278 #endif