2 // Copyright 2019 Karol Herbst
4 // Permission is hereby granted, free of charge, to any person obtaining a
5 // copy of this software and associated documentation files (the "Software"),
6 // to deal in the Software without restriction, including without limitation
7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 // and/or sell copies of the Software, and to permit persons to whom the
9 // Software is furnished to do so, subject to the following conditions:
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 // OTHER DEALINGS IN THE SOFTWARE.
23 #include "invocation.hpp"
27 #include "core/device.hpp"
28 #include "core/error.hpp"
29 #include "core/module.hpp"
30 #include "pipe/p_state.h"
31 #include "util/algorithm.hpp"
32 #include "util/functional.hpp"
34 #include <compiler/glsl_types.h>
35 #include <compiler/nir/nir_builder.h>
36 #include <compiler/nir/nir_serialize.h>
37 #include <compiler/spirv/nir_spirv.h>
38 #include <util/u_math.h>
40 using namespace clover
;
42 #ifdef HAVE_CLOVER_SPIRV
44 // Refs and unrefs the glsl_type_singleton.
45 static class glsl_type_ref
{
48 glsl_type_singleton_init_or_ref();
52 glsl_type_singleton_decref();
56 static const nir_shader_compiler_options
*
57 dev_get_nir_compiler_options(const device
&dev
)
59 const void *co
= dev
.get_compiler_options(PIPE_SHADER_IR_NIR
);
60 return static_cast<const nir_shader_compiler_options
*>(co
);
63 static void debug_function(void *private_data
,
64 enum nir_spirv_debug_level level
, size_t spirv_offset
,
68 auto r_log
= reinterpret_cast<std::string
*>(private_data
);
72 struct clover_lower_nir_state
{
73 std::vector
<module::argument
> &args
;
75 nir_variable
*offset_vars
[3];
79 clover_lower_nir_filter(const nir_instr
*instr
, const void *)
81 return instr
->type
== nir_instr_type_intrinsic
;
85 clover_lower_nir_instr(nir_builder
*b
, nir_instr
*instr
, void *_state
)
87 clover_lower_nir_state
*state
= reinterpret_cast<clover_lower_nir_state
*>(_state
);
88 nir_intrinsic_instr
*intrinsic
= nir_instr_as_intrinsic(instr
);
90 switch (intrinsic
->intrinsic
) {
91 case nir_intrinsic_load_base_global_invocation_id
: {
92 nir_ssa_def
*loads
[3];
94 /* create variables if we didn't do so alrady */
95 if (!state
->offset_vars
[0]) {
96 /* TODO: fix for 64 bit */
97 /* Even though we only place one scalar argument, clover will bind up to
100 state
->args
.emplace_back(module::argument::scalar
, 4, 4, 4,
101 module::argument::zero_ext
,
102 module::argument::grid_offset
);
104 const glsl_type
*type
= glsl_uint_type();
105 for (uint32_t i
= 0; i
< 3; i
++) {
106 state
->offset_vars
[i
] =
107 nir_variable_create(b
->shader
, nir_var_uniform
, type
,
108 "global_invocation_id_offsets");
109 state
->offset_vars
[i
]->data
.location
= b
->shader
->num_uniforms
++;
113 for (int i
= 0; i
< 3; i
++) {
114 nir_variable
*var
= state
->offset_vars
[i
];
115 loads
[i
] = var
? nir_load_var(b
, var
) : nir_imm_int(b
, 0);
118 return nir_u2u(b
, nir_vec(b
, loads
, state
->global_dims
),
119 nir_dest_bit_size(intrinsic
->dest
));
127 clover_lower_nir(nir_shader
*nir
, std::vector
<module::argument
> &args
, uint32_t dims
)
129 clover_lower_nir_state state
= { args
, dims
};
130 return nir_shader_lower_instructions(nir
,
131 clover_lower_nir_filter
, clover_lower_nir_instr
, &state
);
134 module
clover::nir::spirv_to_nir(const module
&mod
, const device
&dev
,
137 struct spirv_to_nir_options spirv_options
= {};
138 spirv_options
.environment
= NIR_SPIRV_OPENCL
;
139 if (dev
.address_bits() == 32u) {
140 spirv_options
.shared_addr_format
= nir_address_format_32bit_offset
;
141 spirv_options
.global_addr_format
= nir_address_format_32bit_global
;
142 spirv_options
.temp_addr_format
= nir_address_format_32bit_offset
;
143 spirv_options
.constant_addr_format
= nir_address_format_32bit_global
;
145 spirv_options
.shared_addr_format
= nir_address_format_32bit_offset_as_64bit
;
146 spirv_options
.global_addr_format
= nir_address_format_64bit_global
;
147 spirv_options
.temp_addr_format
= nir_address_format_32bit_offset_as_64bit
;
148 spirv_options
.constant_addr_format
= nir_address_format_64bit_global
;
150 spirv_options
.caps
.address
= true;
151 spirv_options
.caps
.float64
= true;
152 spirv_options
.caps
.int8
= true;
153 spirv_options
.caps
.int16
= true;
154 spirv_options
.caps
.int64
= true;
155 spirv_options
.caps
.kernel
= true;
156 spirv_options
.caps
.int64_atomics
= dev
.has_int64_atomics();
157 spirv_options
.debug
.func
= &debug_function
;
158 spirv_options
.debug
.private_data
= &r_log
;
161 // We only insert one section.
162 assert(mod
.secs
.size() == 1);
163 auto §ion
= mod
.secs
[0];
165 module::resource_id section_id
= 0;
166 for (const auto &sym
: mod
.syms
) {
167 assert(sym
.section
== 0);
170 reinterpret_cast<const pipe_binary_program_header
*>(section
.data
.data());
171 const uint32_t *data
= reinterpret_cast<const uint32_t *>(binary
->blob
);
172 const size_t num_words
= binary
->num_bytes
/ 4;
173 const char *name
= sym
.name
.c_str();
174 auto *compiler_options
= dev_get_nir_compiler_options(dev
);
176 nir_shader
*nir
= spirv_to_nir(data
, num_words
, nullptr, 0,
177 MESA_SHADER_KERNEL
, name
,
178 &spirv_options
, compiler_options
);
180 r_log
+= "Translation from SPIR-V to NIR for kernel \"" + sym
.name
+
185 nir
->info
.cs
.local_size_variable
= true;
186 nir_validate_shader(nir
, "clover");
188 // Inline all functions first.
189 // according to the comment on nir_inline_functions
190 NIR_PASS_V(nir
, nir_lower_variable_initializers
, nir_var_function_temp
);
191 NIR_PASS_V(nir
, nir_lower_returns
);
192 NIR_PASS_V(nir
, nir_inline_functions
);
193 NIR_PASS_V(nir
, nir_copy_prop
);
194 NIR_PASS_V(nir
, nir_opt_deref
);
196 // Pick off the single entrypoint that we want.
197 foreach_list_typed_safe(nir_function
, func
, node
, &nir
->functions
) {
198 if (!func
->is_entrypoint
)
199 exec_node_remove(&func
->node
);
201 assert(exec_list_length(&nir
->functions
) == 1);
203 nir_validate_shader(nir
, "clover after function inlining");
205 NIR_PASS_V(nir
, nir_lower_variable_initializers
, ~nir_var_function_temp
);
207 // copy propagate to prepare for lower_explicit_io
208 NIR_PASS_V(nir
, nir_split_var_copies
);
209 NIR_PASS_V(nir
, nir_opt_copy_prop_vars
);
210 NIR_PASS_V(nir
, nir_lower_var_copies
);
211 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
212 NIR_PASS_V(nir
, nir_opt_dce
);
214 NIR_PASS_V(nir
, nir_lower_system_values
);
215 nir_lower_compute_system_values_options sysval_options
= { 0 };
216 sysval_options
.has_base_global_invocation_id
= true;
217 NIR_PASS_V(nir
, nir_lower_compute_system_values
, &sysval_options
);
219 auto args
= sym
.args
;
220 NIR_PASS_V(nir
, clover_lower_nir
, args
, dev
.max_block_size().size());
222 // Calculate input offsets.
224 nir_foreach_uniform_variable(var
, nir
) {
225 offset
= align(offset
, glsl_get_cl_alignment(var
->type
));
226 var
->data
.driver_location
= offset
;
227 offset
+= glsl_get_cl_size(var
->type
);
230 NIR_PASS_V(nir
, nir_lower_mem_constant_vars
,
231 glsl_get_cl_type_size_align
);
232 NIR_PASS_V(nir
, nir_lower_vars_to_explicit_types
,
233 nir_var_mem_shared
| nir_var_function_temp
,
234 glsl_get_cl_type_size_align
);
236 /* use offsets for kernel inputs (uniform) */
237 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_uniform
,
238 nir
->info
.cs
.ptr_size
== 64 ?
239 nir_address_format_32bit_offset_as_64bit
:
240 nir_address_format_32bit_offset
);
242 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_mem_constant
,
243 spirv_options
.constant_addr_format
);
244 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_mem_shared
,
245 spirv_options
.shared_addr_format
);
247 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_function_temp
,
248 spirv_options
.temp_addr_format
);
250 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_mem_global
,
251 spirv_options
.global_addr_format
);
253 NIR_PASS_V(nir
, nir_remove_dead_variables
, nir_var_all
, NULL
);
255 if (compiler_options
->lower_int64_options
)
256 NIR_PASS_V(nir
, nir_lower_int64
);
258 NIR_PASS_V(nir
, nir_opt_dce
);
262 nir_serialize(&blob
, nir
, false);
264 const pipe_binary_program_header header
{ uint32_t(blob
.size
) };
265 module::section text
{ section_id
, module::section::text_executable
, header
.num_bytes
, {} };
266 text
.data
.insert(text
.data
.end(), reinterpret_cast<const char *>(&header
),
267 reinterpret_cast<const char *>(&header
) + sizeof(header
));
268 text
.data
.insert(text
.data
.end(), blob
.data
, blob
.data
+ blob
.size
);
270 m
.syms
.emplace_back(sym
.name
, section_id
, 0, args
);
271 m
.secs
.push_back(text
);
277 module
clover::nir::spirv_to_nir(const module
&mod
, const device
&dev
, std::string
&r_log
)
279 r_log
+= "SPIR-V support in clover is not enabled.\n";
280 throw error(CL_LINKER_NOT_AVAILABLE
);