clover: Use args.size() to compute new var locations
[mesa.git] / src / gallium / frontends / clover / nir / invocation.cpp
1 //
2 // Copyright 2019 Karol Herbst
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a
5 // copy of this software and associated documentation files (the "Software"),
6 // to deal in the Software without restriction, including without limitation
7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 // and/or sell copies of the Software, and to permit persons to whom the
9 // Software is furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 // OTHER DEALINGS IN THE SOFTWARE.
21 //
22
23 #include "invocation.hpp"
24
25 #include <tuple>
26
27 #include "core/device.hpp"
28 #include "core/error.hpp"
29 #include "core/module.hpp"
30 #include "pipe/p_state.h"
31 #include "util/algorithm.hpp"
32 #include "util/functional.hpp"
33
34 #include <compiler/glsl_types.h>
35 #include <compiler/nir/nir_builder.h>
36 #include <compiler/nir/nir_serialize.h>
37 #include <compiler/spirv/nir_spirv.h>
38 #include <util/u_math.h>
39
40 using namespace clover;
41
42 #ifdef HAVE_CLOVER_SPIRV
43
44 // Refs and unrefs the glsl_type_singleton.
45 static class glsl_type_ref {
46 public:
47 glsl_type_ref() {
48 glsl_type_singleton_init_or_ref();
49 }
50
51 ~glsl_type_ref() {
52 glsl_type_singleton_decref();
53 }
54 } glsl_type_ref;
55
56 static const nir_shader_compiler_options *
57 dev_get_nir_compiler_options(const device &dev)
58 {
59 const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR);
60 return static_cast<const nir_shader_compiler_options*>(co);
61 }
62
63 static void debug_function(void *private_data,
64 enum nir_spirv_debug_level level, size_t spirv_offset,
65 const char *message)
66 {
67 assert(private_data);
68 auto r_log = reinterpret_cast<std::string *>(private_data);
69 *r_log += message;
70 }
71
72 struct clover_lower_nir_state {
73 std::vector<module::argument> &args;
74 uint32_t global_dims;
75 nir_variable *offset_vars[3];
76 };
77
78 static bool
79 clover_lower_nir_filter(const nir_instr *instr, const void *)
80 {
81 return instr->type == nir_instr_type_intrinsic;
82 }
83
84 static nir_ssa_def *
85 clover_lower_nir_instr(nir_builder *b, nir_instr *instr, void *_state)
86 {
87 clover_lower_nir_state *state = reinterpret_cast<clover_lower_nir_state*>(_state);
88 nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
89
90 switch (intrinsic->intrinsic) {
91 case nir_intrinsic_load_base_global_invocation_id: {
92 nir_ssa_def *loads[3];
93
94 /* create variables if we didn't do so alrady */
95 if (!state->offset_vars[0]) {
96 /* TODO: fix for 64 bit */
97 /* Even though we only place one scalar argument, clover will bind up to
98 * three 32 bit values
99 */
100 unsigned location = state->args.size();
101 state->args.emplace_back(module::argument::scalar, 4, 4, 4,
102 module::argument::zero_ext,
103 module::argument::grid_offset);
104
105 const glsl_type *type = glsl_uint_type();
106 for (uint32_t i = 0; i < 3; i++) {
107 state->offset_vars[i] =
108 nir_variable_create(b->shader, nir_var_uniform, type,
109 "global_invocation_id_offsets");
110 state->offset_vars[i]->data.location = location + i;
111 }
112 }
113
114 for (int i = 0; i < 3; i++) {
115 nir_variable *var = state->offset_vars[i];
116 loads[i] = var ? nir_load_var(b, var) : nir_imm_int(b, 0);
117 }
118
119 return nir_u2u(b, nir_vec(b, loads, state->global_dims),
120 nir_dest_bit_size(intrinsic->dest));
121 }
122 default:
123 return NULL;
124 }
125 }
126
127 static bool
128 clover_lower_nir(nir_shader *nir, std::vector<module::argument> &args, uint32_t dims)
129 {
130 clover_lower_nir_state state = { args, dims };
131 return nir_shader_lower_instructions(nir,
132 clover_lower_nir_filter, clover_lower_nir_instr, &state);
133 }
134
135 module clover::nir::spirv_to_nir(const module &mod, const device &dev,
136 std::string &r_log)
137 {
138 struct spirv_to_nir_options spirv_options = {};
139 spirv_options.environment = NIR_SPIRV_OPENCL;
140 if (dev.address_bits() == 32u) {
141 spirv_options.shared_addr_format = nir_address_format_32bit_offset;
142 spirv_options.global_addr_format = nir_address_format_32bit_global;
143 spirv_options.temp_addr_format = nir_address_format_32bit_offset;
144 spirv_options.constant_addr_format = nir_address_format_32bit_global;
145 } else {
146 spirv_options.shared_addr_format = nir_address_format_32bit_offset_as_64bit;
147 spirv_options.global_addr_format = nir_address_format_64bit_global;
148 spirv_options.temp_addr_format = nir_address_format_32bit_offset_as_64bit;
149 spirv_options.constant_addr_format = nir_address_format_64bit_global;
150 }
151 spirv_options.caps.address = true;
152 spirv_options.caps.float64 = true;
153 spirv_options.caps.int8 = true;
154 spirv_options.caps.int16 = true;
155 spirv_options.caps.int64 = true;
156 spirv_options.caps.kernel = true;
157 spirv_options.caps.int64_atomics = dev.has_int64_atomics();
158 spirv_options.debug.func = &debug_function;
159 spirv_options.debug.private_data = &r_log;
160
161 module m;
162 // We only insert one section.
163 assert(mod.secs.size() == 1);
164 auto &section = mod.secs[0];
165
166 module::resource_id section_id = 0;
167 for (const auto &sym : mod.syms) {
168 assert(sym.section == 0);
169
170 const auto *binary =
171 reinterpret_cast<const pipe_binary_program_header *>(section.data.data());
172 const uint32_t *data = reinterpret_cast<const uint32_t *>(binary->blob);
173 const size_t num_words = binary->num_bytes / 4;
174 const char *name = sym.name.c_str();
175 auto *compiler_options = dev_get_nir_compiler_options(dev);
176
177 nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0,
178 MESA_SHADER_KERNEL, name,
179 &spirv_options, compiler_options);
180 if (!nir) {
181 r_log += "Translation from SPIR-V to NIR for kernel \"" + sym.name +
182 "\" failed.\n";
183 throw build_error();
184 }
185
186 nir->info.cs.local_size_variable = true;
187 nir_validate_shader(nir, "clover");
188
189 // Inline all functions first.
190 // according to the comment on nir_inline_functions
191 NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
192 NIR_PASS_V(nir, nir_lower_returns);
193 NIR_PASS_V(nir, nir_inline_functions);
194 NIR_PASS_V(nir, nir_copy_prop);
195 NIR_PASS_V(nir, nir_opt_deref);
196
197 // Pick off the single entrypoint that we want.
198 foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
199 if (!func->is_entrypoint)
200 exec_node_remove(&func->node);
201 }
202 assert(exec_list_length(&nir->functions) == 1);
203
204 nir_validate_shader(nir, "clover after function inlining");
205
206 NIR_PASS_V(nir, nir_lower_variable_initializers, ~nir_var_function_temp);
207
208 // copy propagate to prepare for lower_explicit_io
209 NIR_PASS_V(nir, nir_split_var_copies);
210 NIR_PASS_V(nir, nir_opt_copy_prop_vars);
211 NIR_PASS_V(nir, nir_lower_var_copies);
212 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
213 NIR_PASS_V(nir, nir_opt_dce);
214
215 NIR_PASS_V(nir, nir_lower_system_values);
216 nir_lower_compute_system_values_options sysval_options = { 0 };
217 sysval_options.has_base_global_invocation_id = true;
218 NIR_PASS_V(nir, nir_lower_compute_system_values, &sysval_options);
219
220 auto args = sym.args;
221 NIR_PASS_V(nir, clover_lower_nir, args, dev.max_block_size().size());
222
223 // Calculate input offsets.
224 unsigned offset = 0;
225 nir_foreach_uniform_variable(var, nir) {
226 offset = align(offset, glsl_get_cl_alignment(var->type));
227 var->data.driver_location = offset;
228 offset += glsl_get_cl_size(var->type);
229 }
230
231 NIR_PASS_V(nir, nir_lower_mem_constant_vars,
232 glsl_get_cl_type_size_align);
233 NIR_PASS_V(nir, nir_lower_vars_to_explicit_types,
234 nir_var_mem_shared | nir_var_function_temp,
235 glsl_get_cl_type_size_align);
236
237 /* use offsets for kernel inputs (uniform) */
238 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform,
239 nir->info.cs.ptr_size == 64 ?
240 nir_address_format_32bit_offset_as_64bit :
241 nir_address_format_32bit_offset);
242
243 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant,
244 spirv_options.constant_addr_format);
245 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_shared,
246 spirv_options.shared_addr_format);
247
248 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_function_temp,
249 spirv_options.temp_addr_format);
250
251 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_global,
252 spirv_options.global_addr_format);
253
254 NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_all, NULL);
255
256 if (compiler_options->lower_int64_options)
257 NIR_PASS_V(nir, nir_lower_int64);
258
259 NIR_PASS_V(nir, nir_opt_dce);
260
261 struct blob blob;
262 blob_init(&blob);
263 nir_serialize(&blob, nir, false);
264
265 const pipe_binary_program_header header { uint32_t(blob.size) };
266 module::section text { section_id, module::section::text_executable, header.num_bytes, {} };
267 text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
268 reinterpret_cast<const char *>(&header) + sizeof(header));
269 text.data.insert(text.data.end(), blob.data, blob.data + blob.size);
270
271 m.syms.emplace_back(sym.name, section_id, 0, args);
272 m.secs.push_back(text);
273 section_id++;
274 }
275 return m;
276 }
277 #else
278 module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log)
279 {
280 r_log += "SPIR-V support in clover is not enabled.\n";
281 throw error(CL_LINKER_NOT_AVAILABLE);
282 }
283 #endif