2 // Copyright 2019 Karol Herbst
4 // Permission is hereby granted, free of charge, to any person obtaining a
5 // copy of this software and associated documentation files (the "Software"),
6 // to deal in the Software without restriction, including without limitation
7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 // and/or sell copies of the Software, and to permit persons to whom the
9 // Software is furnished to do so, subject to the following conditions:
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 // OTHER DEALINGS IN THE SOFTWARE.
23 #include "invocation.hpp"
27 #include "core/device.hpp"
28 #include "core/error.hpp"
29 #include "core/module.hpp"
30 #include "pipe/p_state.h"
31 #include "util/algorithm.hpp"
32 #include "util/functional.hpp"
34 #include <compiler/glsl_types.h>
35 #include <compiler/nir/nir_builder.h>
36 #include <compiler/nir/nir_serialize.h>
37 #include <compiler/spirv/nir_spirv.h>
38 #include <util/u_math.h>
40 using namespace clover
;
42 #ifdef HAVE_CLOVER_SPIRV
44 // Refs and unrefs the glsl_type_singleton.
45 static class glsl_type_ref
{
48 glsl_type_singleton_init_or_ref();
52 glsl_type_singleton_decref();
56 static const nir_shader_compiler_options
*
57 dev_get_nir_compiler_options(const device
&dev
)
59 const void *co
= dev
.get_compiler_options(PIPE_SHADER_IR_NIR
);
60 return static_cast<const nir_shader_compiler_options
*>(co
);
63 static void debug_function(void *private_data
,
64 enum nir_spirv_debug_level level
, size_t spirv_offset
,
68 auto r_log
= reinterpret_cast<std::string
*>(private_data
);
72 struct clover_lower_nir_state
{
73 std::vector
<module::argument
> &args
;
75 nir_variable
*offset_vars
[3];
79 clover_lower_nir_filter(const nir_instr
*instr
, const void *)
81 return instr
->type
== nir_instr_type_intrinsic
;
85 clover_lower_nir_instr(nir_builder
*b
, nir_instr
*instr
, void *_state
)
87 clover_lower_nir_state
*state
= reinterpret_cast<clover_lower_nir_state
*>(_state
);
88 nir_intrinsic_instr
*intrinsic
= nir_instr_as_intrinsic(instr
);
90 switch (intrinsic
->intrinsic
) {
91 case nir_intrinsic_load_base_global_invocation_id
: {
92 nir_ssa_def
*loads
[3];
94 /* create variables if we didn't do so alrady */
95 if (!state
->offset_vars
[0]) {
96 /* TODO: fix for 64 bit */
97 /* Even though we only place one scalar argument, clover will bind up to
100 unsigned location
= state
->args
.size();
101 state
->args
.emplace_back(module::argument::scalar
, 4, 4, 4,
102 module::argument::zero_ext
,
103 module::argument::grid_offset
);
105 const glsl_type
*type
= glsl_uint_type();
106 for (uint32_t i
= 0; i
< 3; i
++) {
107 state
->offset_vars
[i
] =
108 nir_variable_create(b
->shader
, nir_var_uniform
, type
,
109 "global_invocation_id_offsets");
110 state
->offset_vars
[i
]->data
.location
= location
+ i
;
114 for (int i
= 0; i
< 3; i
++) {
115 nir_variable
*var
= state
->offset_vars
[i
];
116 loads
[i
] = var
? nir_load_var(b
, var
) : nir_imm_int(b
, 0);
119 return nir_u2u(b
, nir_vec(b
, loads
, state
->global_dims
),
120 nir_dest_bit_size(intrinsic
->dest
));
128 clover_lower_nir(nir_shader
*nir
, std::vector
<module::argument
> &args
, uint32_t dims
)
130 clover_lower_nir_state state
= { args
, dims
};
131 return nir_shader_lower_instructions(nir
,
132 clover_lower_nir_filter
, clover_lower_nir_instr
, &state
);
135 module
clover::nir::spirv_to_nir(const module
&mod
, const device
&dev
,
138 struct spirv_to_nir_options spirv_options
= {};
139 spirv_options
.environment
= NIR_SPIRV_OPENCL
;
140 if (dev
.address_bits() == 32u) {
141 spirv_options
.shared_addr_format
= nir_address_format_32bit_offset
;
142 spirv_options
.global_addr_format
= nir_address_format_32bit_global
;
143 spirv_options
.temp_addr_format
= nir_address_format_32bit_offset
;
144 spirv_options
.constant_addr_format
= nir_address_format_32bit_global
;
146 spirv_options
.shared_addr_format
= nir_address_format_32bit_offset_as_64bit
;
147 spirv_options
.global_addr_format
= nir_address_format_64bit_global
;
148 spirv_options
.temp_addr_format
= nir_address_format_32bit_offset_as_64bit
;
149 spirv_options
.constant_addr_format
= nir_address_format_64bit_global
;
151 spirv_options
.caps
.address
= true;
152 spirv_options
.caps
.float64
= true;
153 spirv_options
.caps
.int8
= true;
154 spirv_options
.caps
.int16
= true;
155 spirv_options
.caps
.int64
= true;
156 spirv_options
.caps
.kernel
= true;
157 spirv_options
.caps
.int64_atomics
= dev
.has_int64_atomics();
158 spirv_options
.debug
.func
= &debug_function
;
159 spirv_options
.debug
.private_data
= &r_log
;
162 // We only insert one section.
163 assert(mod
.secs
.size() == 1);
164 auto §ion
= mod
.secs
[0];
166 module::resource_id section_id
= 0;
167 for (const auto &sym
: mod
.syms
) {
168 assert(sym
.section
== 0);
171 reinterpret_cast<const pipe_binary_program_header
*>(section
.data
.data());
172 const uint32_t *data
= reinterpret_cast<const uint32_t *>(binary
->blob
);
173 const size_t num_words
= binary
->num_bytes
/ 4;
174 const char *name
= sym
.name
.c_str();
175 auto *compiler_options
= dev_get_nir_compiler_options(dev
);
177 nir_shader
*nir
= spirv_to_nir(data
, num_words
, nullptr, 0,
178 MESA_SHADER_KERNEL
, name
,
179 &spirv_options
, compiler_options
);
181 r_log
+= "Translation from SPIR-V to NIR for kernel \"" + sym
.name
+
186 nir
->info
.cs
.local_size_variable
= true;
187 nir_validate_shader(nir
, "clover");
189 // Inline all functions first.
190 // according to the comment on nir_inline_functions
191 NIR_PASS_V(nir
, nir_lower_variable_initializers
, nir_var_function_temp
);
192 NIR_PASS_V(nir
, nir_lower_returns
);
193 NIR_PASS_V(nir
, nir_inline_functions
);
194 NIR_PASS_V(nir
, nir_copy_prop
);
195 NIR_PASS_V(nir
, nir_opt_deref
);
197 // Pick off the single entrypoint that we want.
198 foreach_list_typed_safe(nir_function
, func
, node
, &nir
->functions
) {
199 if (!func
->is_entrypoint
)
200 exec_node_remove(&func
->node
);
202 assert(exec_list_length(&nir
->functions
) == 1);
204 nir_validate_shader(nir
, "clover after function inlining");
206 NIR_PASS_V(nir
, nir_lower_variable_initializers
, ~nir_var_function_temp
);
208 // copy propagate to prepare for lower_explicit_io
209 NIR_PASS_V(nir
, nir_split_var_copies
);
210 NIR_PASS_V(nir
, nir_opt_copy_prop_vars
);
211 NIR_PASS_V(nir
, nir_lower_var_copies
);
212 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
213 NIR_PASS_V(nir
, nir_opt_dce
);
215 NIR_PASS_V(nir
, nir_lower_system_values
);
216 nir_lower_compute_system_values_options sysval_options
= { 0 };
217 sysval_options
.has_base_global_invocation_id
= true;
218 NIR_PASS_V(nir
, nir_lower_compute_system_values
, &sysval_options
);
220 auto args
= sym
.args
;
221 NIR_PASS_V(nir
, clover_lower_nir
, args
, dev
.max_block_size().size());
223 NIR_PASS_V(nir
, nir_lower_mem_constant_vars
,
224 glsl_get_cl_type_size_align
);
225 NIR_PASS_V(nir
, nir_lower_vars_to_explicit_types
,
226 nir_var_uniform
| nir_var_mem_shared
|
227 nir_var_mem_global
| nir_var_function_temp
,
228 glsl_get_cl_type_size_align
);
230 /* use offsets for kernel inputs (uniform) */
231 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_uniform
,
232 nir
->info
.cs
.ptr_size
== 64 ?
233 nir_address_format_32bit_offset_as_64bit
:
234 nir_address_format_32bit_offset
);
236 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_mem_constant
,
237 spirv_options
.constant_addr_format
);
238 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_mem_shared
,
239 spirv_options
.shared_addr_format
);
241 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_function_temp
,
242 spirv_options
.temp_addr_format
);
244 NIR_PASS_V(nir
, nir_lower_explicit_io
, nir_var_mem_global
,
245 spirv_options
.global_addr_format
);
247 NIR_PASS_V(nir
, nir_remove_dead_variables
, nir_var_all
, NULL
);
249 if (compiler_options
->lower_int64_options
)
250 NIR_PASS_V(nir
, nir_lower_int64
);
252 NIR_PASS_V(nir
, nir_opt_dce
);
256 nir_serialize(&blob
, nir
, false);
258 const pipe_binary_program_header header
{ uint32_t(blob
.size
) };
259 module::section text
{ section_id
, module::section::text_executable
, header
.num_bytes
, {} };
260 text
.data
.insert(text
.data
.end(), reinterpret_cast<const char *>(&header
),
261 reinterpret_cast<const char *>(&header
) + sizeof(header
));
262 text
.data
.insert(text
.data
.end(), blob
.data
, blob
.data
+ blob
.size
);
264 m
.syms
.emplace_back(sym
.name
, section_id
, 0, args
);
265 m
.secs
.push_back(text
);
271 module
clover::nir::spirv_to_nir(const module
&mod
, const device
&dev
, std::string
&r_log
)
273 r_log
+= "SPIR-V support in clover is not enabled.\n";
274 throw error(CL_LINKER_NOT_AVAILABLE
);