2 * Copyright © 2019 Google LLC
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
24 #include "tu_private.h"
26 #include "spirv/nir_spirv.h"
27 #include "util/mesa-sha1.h"
29 #include "ir3/ir3_nir.h"
32 tu_spirv_to_nir(struct ir3_compiler
*compiler
,
33 const uint32_t *words
,
35 gl_shader_stage stage
,
36 const char *entry_point_name
,
37 const VkSpecializationInfo
*spec_info
)
39 /* TODO these are made-up */
40 const struct spirv_to_nir_options spirv_options
= {
41 .frag_coord_is_sysval
= true,
42 .lower_ubo_ssbo_access_to_offsets
= true,
45 const nir_shader_compiler_options
*nir_options
=
46 ir3_get_compiler_options(compiler
);
48 /* convert VkSpecializationInfo */
49 struct nir_spirv_specialization
*spec
= NULL
;
50 uint32_t num_spec
= 0;
51 if (spec_info
&& spec_info
->mapEntryCount
) {
52 spec
= malloc(sizeof(*spec
) * spec_info
->mapEntryCount
);
56 for (uint32_t i
= 0; i
< spec_info
->mapEntryCount
; i
++) {
57 const VkSpecializationMapEntry
*entry
= &spec_info
->pMapEntries
[i
];
58 const void *data
= spec_info
->pData
+ entry
->offset
;
59 assert(data
+ entry
->size
<= spec_info
->pData
+ spec_info
->dataSize
);
60 spec
[i
].id
= entry
->constantID
;
62 spec
[i
].data64
= *(const uint64_t *) data
;
64 spec
[i
].data32
= *(const uint32_t *) data
;
65 spec
[i
].defined_on_module
= false;
68 num_spec
= spec_info
->mapEntryCount
;
72 spirv_to_nir(words
, word_count
, spec
, num_spec
, stage
, entry_point_name
,
73 &spirv_options
, nir_options
);
77 assert(nir
->info
.stage
== stage
);
78 nir_validate_shader(nir
, "after spirv_to_nir");
84 tu_sort_variables_by_location(struct exec_list
*variables
)
86 struct exec_list sorted
;
87 exec_list_make_empty(&sorted
);
89 nir_foreach_variable_safe(var
, variables
)
91 exec_node_remove(&var
->node
);
93 /* insert the variable into the sorted list */
94 nir_variable
*next
= NULL
;
95 nir_foreach_variable(tmp
, &sorted
)
97 if (var
->data
.location
< tmp
->data
.location
) {
103 exec_node_insert_node_before(&next
->node
, &var
->node
);
105 exec_list_push_tail(&sorted
, &var
->node
);
108 exec_list_move_nodes_to(&sorted
, variables
);
112 map_add(struct tu_descriptor_map
*map
, int set
, int binding
)
115 for (index
= 0; index
< map
->num
; index
++) {
116 if (set
== map
->set
[index
] && binding
== map
->binding
[index
])
120 assert(index
< ARRAY_SIZE(map
->set
));
122 map
->set
[index
] = set
;
123 map
->binding
[index
] = binding
;
124 map
->num
= MAX2(map
->num
, index
+ 1);
129 lower_tex_src_to_offset(nir_builder
*b
, nir_tex_instr
*instr
, unsigned src_idx
,
130 struct tu_shader
*shader
)
132 nir_ssa_def
*index
= NULL
;
133 unsigned base_index
= 0;
134 unsigned array_elements
= 1;
135 nir_tex_src
*src
= &instr
->src
[src_idx
];
136 bool is_sampler
= src
->src_type
== nir_tex_src_sampler_deref
;
138 /* We compute first the offsets */
139 nir_deref_instr
*deref
= nir_instr_as_deref(src
->src
.ssa
->parent_instr
);
140 while (deref
->deref_type
!= nir_deref_type_var
) {
141 assert(deref
->parent
.is_ssa
);
142 nir_deref_instr
*parent
=
143 nir_instr_as_deref(deref
->parent
.ssa
->parent_instr
);
145 assert(deref
->deref_type
== nir_deref_type_array
);
147 if (nir_src_is_const(deref
->arr
.index
) && index
== NULL
) {
148 /* We're still building a direct index */
149 base_index
+= nir_src_as_uint(deref
->arr
.index
) * array_elements
;
152 /* We used to be direct but not anymore */
153 index
= nir_imm_int(b
, base_index
);
157 index
= nir_iadd(b
, index
,
158 nir_imul(b
, nir_imm_int(b
, array_elements
),
159 nir_ssa_for_src(b
, deref
->arr
.index
, 1)));
162 array_elements
*= glsl_get_length(parent
->type
);
168 index
= nir_umin(b
, index
, nir_imm_int(b
, array_elements
- 1));
170 /* We have the offsets, we apply them, rewriting the source or removing
174 nir_instr_rewrite_src(&instr
->instr
, &src
->src
,
175 nir_src_for_ssa(index
));
177 src
->src_type
= is_sampler
?
178 nir_tex_src_sampler_offset
:
179 nir_tex_src_texture_offset
;
181 instr
->texture_array_size
= array_elements
;
183 nir_tex_instr_remove_src(instr
, src_idx
);
186 if (array_elements
> 1)
187 tu_finishme("texture/sampler array");
190 instr
->sampler_index
= map_add(&shader
->sampler_map
,
191 deref
->var
->data
.descriptor_set
,
192 deref
->var
->data
.binding
);
193 instr
->sampler_index
+= base_index
;
195 instr
->texture_index
= map_add(&shader
->texture_map
,
196 deref
->var
->data
.descriptor_set
,
197 deref
->var
->data
.binding
);
198 instr
->texture_index
+= base_index
;
199 instr
->texture_array_size
= array_elements
;
204 lower_sampler(nir_builder
*b
, nir_tex_instr
*instr
, struct tu_shader
*shader
)
207 nir_tex_instr_src_index(instr
, nir_tex_src_texture_deref
);
209 if (texture_idx
>= 0)
210 lower_tex_src_to_offset(b
, instr
, texture_idx
, shader
);
213 nir_tex_instr_src_index(instr
, nir_tex_src_sampler_deref
);
215 if (sampler_idx
>= 0)
216 lower_tex_src_to_offset(b
, instr
, sampler_idx
, shader
);
218 if (texture_idx
< 0 && sampler_idx
< 0)
225 lower_intrinsic(nir_builder
*b
, nir_intrinsic_instr
*instr
,
226 struct tu_shader
*shader
)
228 if (instr
->intrinsic
== nir_intrinsic_load_push_constant
) {
229 /* note: ir3 wants load_ubo, not load_uniform */
230 assert(nir_intrinsic_base(instr
) == 0);
232 nir_intrinsic_instr
*load
=
233 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_load_ubo
);
234 load
->num_components
= instr
->num_components
;
235 load
->src
[0] = nir_src_for_ssa(nir_imm_int(b
, 0));
236 load
->src
[1] = instr
->src
[0];
237 nir_ssa_dest_init(&load
->instr
, &load
->dest
,
238 load
->num_components
, instr
->dest
.ssa
.bit_size
,
239 instr
->dest
.ssa
.name
);
240 nir_builder_instr_insert(b
, &load
->instr
);
241 nir_ssa_def_rewrite_uses(&instr
->dest
.ssa
, nir_src_for_ssa(&load
->dest
.ssa
));
243 nir_instr_remove(&instr
->instr
);
248 if (instr
->intrinsic
!= nir_intrinsic_vulkan_resource_index
)
251 nir_const_value
*const_val
= nir_src_as_const_value(instr
->src
[0]);
252 if (!const_val
|| const_val
->u32
!= 0)
253 tu_finishme("non-zero vulkan_resource_index array index");
256 unsigned set
= nir_intrinsic_desc_set(instr
);
257 unsigned binding
= nir_intrinsic_binding(instr
);
260 switch (nir_intrinsic_desc_type(instr
)) {
261 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER
:
262 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC
:
263 /* skip index 0 which is used for push constants */
264 index
= map_add(&shader
->ubo_map
, set
, binding
) + 1;
266 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
:
267 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC
:
268 index
= map_add(&shader
->ssbo_map
, set
, binding
);
271 tu_finishme("unsupported desc_type for vulkan_resource_index");
275 nir_ssa_def_rewrite_uses(&instr
->dest
.ssa
,
276 nir_src_for_ssa(nir_imm_int(b
, index
)));
277 nir_instr_remove(&instr
->instr
);
283 lower_impl(nir_function_impl
*impl
, struct tu_shader
*shader
)
286 nir_builder_init(&b
, impl
);
287 bool progress
= false;
289 nir_foreach_block(block
, impl
) {
290 nir_foreach_instr_safe(instr
, block
) {
291 b
.cursor
= nir_before_instr(instr
);
292 switch (instr
->type
) {
293 case nir_instr_type_tex
:
294 progress
|= lower_sampler(&b
, nir_instr_as_tex(instr
), shader
);
296 case nir_instr_type_intrinsic
:
297 progress
|= lower_intrinsic(&b
, nir_instr_as_intrinsic(instr
), shader
);
309 tu_lower_io(nir_shader
*shader
, struct tu_shader
*tu_shader
)
311 bool progress
= false;
313 nir_foreach_function(function
, shader
) {
315 progress
|= lower_impl(function
->impl
, tu_shader
);
322 tu_shader_create(struct tu_device
*dev
,
323 gl_shader_stage stage
,
324 const VkPipelineShaderStageCreateInfo
*stage_info
,
325 const VkAllocationCallbacks
*alloc
)
327 const struct tu_shader_module
*module
=
328 tu_shader_module_from_handle(stage_info
->module
);
329 struct tu_shader
*shader
;
331 const uint32_t max_variant_count
= (stage
== MESA_SHADER_VERTEX
) ? 2 : 1;
334 sizeof(*shader
) + sizeof(struct ir3_shader_variant
) * max_variant_count
,
335 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
339 /* translate SPIR-V to NIR */
340 assert(module
->code_size
% 4 == 0);
341 nir_shader
*nir
= tu_spirv_to_nir(
342 dev
->compiler
, (const uint32_t *) module
->code
, module
->code_size
/ 4,
343 stage
, stage_info
->pName
, stage_info
->pSpecializationInfo
);
345 vk_free2(&dev
->alloc
, alloc
, shader
);
349 if (unlikely(dev
->physical_device
->instance
->debug_flags
& TU_DEBUG_NIR
)) {
350 fprintf(stderr
, "translated nir:\n");
351 nir_print_shader(nir
, stderr
);
354 /* multi step inlining procedure */
355 NIR_PASS_V(nir
, nir_lower_constant_initializers
, nir_var_function_temp
);
356 NIR_PASS_V(nir
, nir_lower_returns
);
357 NIR_PASS_V(nir
, nir_inline_functions
);
358 NIR_PASS_V(nir
, nir_opt_deref
);
359 foreach_list_typed_safe(nir_function
, func
, node
, &nir
->functions
) {
360 if (!func
->is_entrypoint
)
361 exec_node_remove(&func
->node
);
363 assert(exec_list_length(&nir
->functions
) == 1);
364 NIR_PASS_V(nir
, nir_lower_constant_initializers
, ~nir_var_function_temp
);
366 /* Split member structs. We do this before lower_io_to_temporaries so that
367 * it doesn't lower system values to temporaries by accident.
369 NIR_PASS_V(nir
, nir_split_var_copies
);
370 NIR_PASS_V(nir
, nir_split_per_member_structs
);
372 NIR_PASS_V(nir
, nir_remove_dead_variables
,
373 nir_var_shader_in
| nir_var_shader_out
| nir_var_system_value
| nir_var_mem_shared
);
375 NIR_PASS_V(nir
, nir_propagate_invariant
);
377 NIR_PASS_V(nir
, nir_lower_io_to_temporaries
, nir_shader_get_entrypoint(nir
), true, true);
379 NIR_PASS_V(nir
, nir_lower_global_vars_to_local
);
380 NIR_PASS_V(nir
, nir_split_var_copies
);
381 NIR_PASS_V(nir
, nir_lower_var_copies
);
383 NIR_PASS_V(nir
, nir_opt_copy_prop_vars
);
384 NIR_PASS_V(nir
, nir_opt_combine_stores
, nir_var_all
);
386 /* ir3 doesn't support indirect input/output */
387 NIR_PASS_V(nir
, nir_lower_indirect_derefs
, nir_var_shader_in
| nir_var_shader_out
);
390 case MESA_SHADER_VERTEX
:
391 tu_sort_variables_by_location(&nir
->outputs
);
393 case MESA_SHADER_TESS_CTRL
:
394 case MESA_SHADER_TESS_EVAL
:
395 case MESA_SHADER_GEOMETRY
:
396 tu_sort_variables_by_location(&nir
->inputs
);
397 tu_sort_variables_by_location(&nir
->outputs
);
399 case MESA_SHADER_FRAGMENT
:
400 tu_sort_variables_by_location(&nir
->inputs
);
402 case MESA_SHADER_COMPUTE
:
405 unreachable("invalid gl_shader_stage");
409 nir_assign_io_var_locations(&nir
->inputs
, &nir
->num_inputs
, stage
);
410 nir_assign_io_var_locations(&nir
->outputs
, &nir
->num_outputs
, stage
);
412 NIR_PASS_V(nir
, nir_lower_system_values
);
413 NIR_PASS_V(nir
, nir_lower_frexp
);
415 NIR_PASS_V(nir
, tu_lower_io
, shader
);
417 NIR_PASS_V(nir
, nir_lower_io
, nir_var_all
, ir3_glsl_type_size
, 0);
419 if (stage
== MESA_SHADER_FRAGMENT
) {
420 /* NOTE: lower load_barycentric_at_sample first, since it
421 * produces load_barycentric_at_offset:
423 NIR_PASS_V(nir
, ir3_nir_lower_load_barycentric_at_sample
);
424 NIR_PASS_V(nir
, ir3_nir_lower_load_barycentric_at_offset
);
426 NIR_PASS_V(nir
, ir3_nir_move_varying_inputs
);
429 NIR_PASS_V(nir
, nir_lower_io_arrays_to_elements_no_indirects
, false);
431 nir_shader_gather_info(nir
, nir_shader_get_entrypoint(nir
));
433 /* num_uniforms only used by ir3 for size of ubo 0 (push constants) */
434 nir
->num_uniforms
= MAX_PUSH_CONSTANTS_SIZE
/ 16;
436 shader
->ir3_shader
.compiler
= dev
->compiler
;
437 shader
->ir3_shader
.type
= stage
;
438 shader
->ir3_shader
.nir
= nir
;
444 tu_shader_destroy(struct tu_device
*dev
,
445 struct tu_shader
*shader
,
446 const VkAllocationCallbacks
*alloc
)
448 if (shader
->ir3_shader
.nir
)
449 ralloc_free(shader
->ir3_shader
.nir
);
451 for (uint32_t i
= 0; i
< 1 + shader
->has_binning_pass
; i
++) {
452 if (shader
->variants
[i
].ir
)
453 ir3_destroy(shader
->variants
[i
].ir
);
456 if (shader
->ir3_shader
.const_state
.immediates
)
457 free(shader
->ir3_shader
.const_state
.immediates
);
459 free(shader
->binary
);
460 if (shader
->binning_binary
)
461 free(shader
->binning_binary
);
463 vk_free2(&dev
->alloc
, alloc
, shader
);
467 tu_shader_compile_options_init(
468 struct tu_shader_compile_options
*options
,
469 const VkGraphicsPipelineCreateInfo
*pipeline_info
)
471 *options
= (struct tu_shader_compile_options
) {
474 /* TODO: VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT
475 * some optimizations need to happen otherwise shader might not compile
478 .include_binning_pass
= true,
483 tu_compile_shader_variant(struct ir3_shader
*shader
,
484 const struct ir3_shader_key
*key
,
485 struct ir3_shader_variant
*nonbinning
,
486 struct ir3_shader_variant
*variant
)
488 variant
->shader
= shader
;
489 variant
->type
= shader
->type
;
491 variant
->binning_pass
= !!nonbinning
;
492 variant
->nonbinning
= nonbinning
;
494 int ret
= ir3_compile_shader_nir(shader
->compiler
, variant
);
498 /* when assemble fails, we rely on tu_shader_destroy to clean up the
501 return ir3_shader_assemble(variant
, shader
->compiler
->gpu_id
);
505 tu_shader_compile(struct tu_device
*dev
,
506 struct tu_shader
*shader
,
507 const struct tu_shader
*next_stage
,
508 const struct tu_shader_compile_options
*options
,
509 const VkAllocationCallbacks
*alloc
)
511 if (options
->optimize
) {
512 /* ignore the key for the first pass of optimization */
513 ir3_optimize_nir(&shader
->ir3_shader
, shader
->ir3_shader
.nir
, NULL
);
515 if (unlikely(dev
->physical_device
->instance
->debug_flags
&
517 fprintf(stderr
, "optimized nir:\n");
518 nir_print_shader(shader
->ir3_shader
.nir
, stderr
);
522 shader
->binary
= tu_compile_shader_variant(
523 &shader
->ir3_shader
, &options
->key
, NULL
, &shader
->variants
[0]);
525 return VK_ERROR_OUT_OF_HOST_MEMORY
;
527 /* compile another variant for the binning pass */
528 if (options
->include_binning_pass
&&
529 shader
->ir3_shader
.type
== MESA_SHADER_VERTEX
) {
530 shader
->binning_binary
= tu_compile_shader_variant(
531 &shader
->ir3_shader
, &options
->key
, &shader
->variants
[0],
532 &shader
->variants
[1]);
533 if (!shader
->binning_binary
)
534 return VK_ERROR_OUT_OF_HOST_MEMORY
;
536 shader
->has_binning_pass
= true;
539 if (unlikely(dev
->physical_device
->instance
->debug_flags
& TU_DEBUG_IR3
)) {
540 fprintf(stderr
, "disassembled ir3:\n");
541 fprintf(stderr
, "shader: %s\n",
542 gl_shader_stage_name(shader
->ir3_shader
.type
));
543 ir3_shader_disasm(&shader
->variants
[0], shader
->binary
, stderr
);
545 if (shader
->has_binning_pass
) {
546 fprintf(stderr
, "disassembled ir3:\n");
547 fprintf(stderr
, "shader: %s (binning)\n",
548 gl_shader_stage_name(shader
->ir3_shader
.type
));
549 ir3_shader_disasm(&shader
->variants
[1], shader
->binning_binary
,
558 tu_CreateShaderModule(VkDevice _device
,
559 const VkShaderModuleCreateInfo
*pCreateInfo
,
560 const VkAllocationCallbacks
*pAllocator
,
561 VkShaderModule
*pShaderModule
)
563 TU_FROM_HANDLE(tu_device
, device
, _device
);
564 struct tu_shader_module
*module
;
566 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO
);
567 assert(pCreateInfo
->flags
== 0);
568 assert(pCreateInfo
->codeSize
% 4 == 0);
570 module
= vk_alloc2(&device
->alloc
, pAllocator
,
571 sizeof(*module
) + pCreateInfo
->codeSize
, 8,
572 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
574 return vk_error(device
->instance
, VK_ERROR_OUT_OF_HOST_MEMORY
);
576 module
->code_size
= pCreateInfo
->codeSize
;
577 memcpy(module
->code
, pCreateInfo
->pCode
, pCreateInfo
->codeSize
);
579 _mesa_sha1_compute(module
->code
, module
->code_size
, module
->sha1
);
581 *pShaderModule
= tu_shader_module_to_handle(module
);
587 tu_DestroyShaderModule(VkDevice _device
,
588 VkShaderModule _module
,
589 const VkAllocationCallbacks
*pAllocator
)
591 TU_FROM_HANDLE(tu_device
, device
, _device
);
592 TU_FROM_HANDLE(tu_shader_module
, module
, _module
);
597 vk_free2(&device
->alloc
, pAllocator
, module
);