2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Jason Ekstrand (jason@jlekstrand.net)
28 #include "vtn_private.h"
29 #include "spirv_info.h"
30 #include "nir_deref.h"
31 #include <vulkan/vulkan_core.h>
33 static struct vtn_pointer
*
34 vtn_align_pointer(struct vtn_builder
*b
, struct vtn_pointer
*ptr
,
40 if (!util_is_power_of_two_nonzero(alignment
)) {
41 vtn_warn("Provided alignment is not a power of two");
42 alignment
= 1 << (ffs(alignment
) - 1);
45 /* If this pointer doesn't have a deref, bail. This either means we're
46 * using the old offset+alignment pointers which don't support carrying
47 * alignment information or we're a pointer that is below the block
48 * boundary in our access chain in which case alignment is meaningless.
50 if (ptr
->deref
== NULL
)
53 /* Ignore alignment information on logical pointers. This way, we don't
54 * trip up drivers with unnecessary casts.
56 nir_address_format addr_format
= vtn_mode_to_address_format(b
, ptr
->mode
);
57 if (addr_format
== nir_address_format_logical
)
60 struct vtn_pointer
*copy
= ralloc(b
, struct vtn_pointer
);
62 copy
->deref
= nir_alignment_deref_cast(&b
->nb
, ptr
->deref
, alignment
, 0);
68 ptr_decoration_cb(struct vtn_builder
*b
, struct vtn_value
*val
, int member
,
69 const struct vtn_decoration
*dec
, void *void_ptr
)
71 struct vtn_pointer
*ptr
= void_ptr
;
73 switch (dec
->decoration
) {
74 case SpvDecorationNonUniformEXT
:
75 ptr
->access
|= ACCESS_NON_UNIFORM
;
84 enum gl_access_qualifier access
;
89 access_align_cb(struct vtn_builder
*b
, struct vtn_value
*val
, int member
,
90 const struct vtn_decoration
*dec
, void *void_ptr
)
92 struct access_align
*aa
= void_ptr
;
94 switch (dec
->decoration
) {
95 case SpvDecorationAlignment
:
96 aa
->alignment
= dec
->operands
[0];
99 case SpvDecorationNonUniformEXT
:
100 aa
->access
|= ACCESS_NON_UNIFORM
;
108 static struct vtn_pointer
*
109 vtn_decorate_pointer(struct vtn_builder
*b
, struct vtn_value
*val
,
110 struct vtn_pointer
*ptr
)
112 struct access_align aa
= { 0, };
113 vtn_foreach_decoration(b
, val
, access_align_cb
, &aa
);
115 ptr
= vtn_align_pointer(b
, ptr
, aa
.alignment
);
117 /* If we're adding access flags, make a copy of the pointer. We could
118 * probably just OR them in without doing so but this prevents us from
119 * leaking them any further than actually specified in the SPIR-V.
121 if (aa
.access
& ~ptr
->access
) {
122 struct vtn_pointer
*copy
= ralloc(b
, struct vtn_pointer
);
124 copy
->access
|= aa
.access
;
132 vtn_push_pointer(struct vtn_builder
*b
, uint32_t value_id
,
133 struct vtn_pointer
*ptr
)
135 struct vtn_value
*val
= vtn_push_value(b
, value_id
, vtn_value_type_pointer
);
136 val
->pointer
= vtn_decorate_pointer(b
, val
, ptr
);
141 vtn_copy_value(struct vtn_builder
*b
, uint32_t src_value_id
,
142 uint32_t dst_value_id
)
144 struct vtn_value
*src
= vtn_untyped_value(b
, src_value_id
);
145 struct vtn_value
*dst
= vtn_untyped_value(b
, dst_value_id
);
146 struct vtn_value src_copy
= *src
;
148 vtn_fail_if(dst
->value_type
!= vtn_value_type_invalid
,
149 "SPIR-V id %u has already been written by another instruction",
152 vtn_fail_if(dst
->type
->id
!= src
->type
->id
,
153 "Result Type must equal Operand type");
155 src_copy
.name
= dst
->name
;
156 src_copy
.decoration
= dst
->decoration
;
157 src_copy
.type
= dst
->type
;
160 if (dst
->value_type
== vtn_value_type_pointer
)
161 dst
->pointer
= vtn_decorate_pointer(b
, dst
, dst
->pointer
);
164 static struct vtn_access_chain
*
165 vtn_access_chain_create(struct vtn_builder
*b
, unsigned length
)
167 struct vtn_access_chain
*chain
;
169 /* Subtract 1 from the length since there's already one built in */
170 size_t size
= sizeof(*chain
) +
171 (MAX2(length
, 1) - 1) * sizeof(chain
->link
[0]);
172 chain
= rzalloc_size(b
, size
);
173 chain
->length
= length
;
179 vtn_mode_uses_ssa_offset(struct vtn_builder
*b
,
180 enum vtn_variable_mode mode
)
182 return ((mode
== vtn_variable_mode_ubo
||
183 mode
== vtn_variable_mode_ssbo
) &&
184 b
->options
->lower_ubo_ssbo_access_to_offsets
) ||
185 mode
== vtn_variable_mode_push_constant
;
189 vtn_mode_is_cross_invocation(struct vtn_builder
*b
,
190 enum vtn_variable_mode mode
)
192 return mode
== vtn_variable_mode_ssbo
||
193 mode
== vtn_variable_mode_ubo
||
194 mode
== vtn_variable_mode_phys_ssbo
||
195 mode
== vtn_variable_mode_push_constant
||
196 mode
== vtn_variable_mode_workgroup
||
197 mode
== vtn_variable_mode_cross_workgroup
;
201 vtn_pointer_is_external_block(struct vtn_builder
*b
,
202 struct vtn_pointer
*ptr
)
204 return ptr
->mode
== vtn_variable_mode_ssbo
||
205 ptr
->mode
== vtn_variable_mode_ubo
||
206 ptr
->mode
== vtn_variable_mode_phys_ssbo
||
207 ptr
->mode
== vtn_variable_mode_push_constant
;
211 vtn_access_link_as_ssa(struct vtn_builder
*b
, struct vtn_access_link link
,
212 unsigned stride
, unsigned bit_size
)
214 vtn_assert(stride
> 0);
215 if (link
.mode
== vtn_access_mode_literal
) {
216 return nir_imm_intN_t(&b
->nb
, link
.id
* stride
, bit_size
);
218 nir_ssa_def
*ssa
= vtn_ssa_value(b
, link
.id
)->def
;
219 if (ssa
->bit_size
!= bit_size
)
220 ssa
= nir_i2i(&b
->nb
, ssa
, bit_size
);
221 return nir_imul_imm(&b
->nb
, ssa
, stride
);
225 static VkDescriptorType
226 vk_desc_type_for_mode(struct vtn_builder
*b
, enum vtn_variable_mode mode
)
229 case vtn_variable_mode_ubo
:
230 return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER
;
231 case vtn_variable_mode_ssbo
:
232 return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
;
234 vtn_fail("Invalid mode for vulkan_resource_index");
239 vtn_variable_resource_index(struct vtn_builder
*b
, struct vtn_variable
*var
,
240 nir_ssa_def
*desc_array_index
)
242 vtn_assert(b
->options
->environment
== NIR_SPIRV_VULKAN
);
244 if (!desc_array_index
) {
245 vtn_assert(glsl_type_is_struct_or_ifc(var
->type
->type
));
246 desc_array_index
= nir_imm_int(&b
->nb
, 0);
249 nir_intrinsic_instr
*instr
=
250 nir_intrinsic_instr_create(b
->nb
.shader
,
251 nir_intrinsic_vulkan_resource_index
);
252 instr
->src
[0] = nir_src_for_ssa(desc_array_index
);
253 nir_intrinsic_set_desc_set(instr
, var
->descriptor_set
);
254 nir_intrinsic_set_binding(instr
, var
->binding
);
255 nir_intrinsic_set_desc_type(instr
, vk_desc_type_for_mode(b
, var
->mode
));
257 vtn_fail_if(var
->mode
!= vtn_variable_mode_ubo
&&
258 var
->mode
!= vtn_variable_mode_ssbo
,
259 "Invalid mode for vulkan_resource_index");
261 nir_address_format addr_format
= vtn_mode_to_address_format(b
, var
->mode
);
262 const struct glsl_type
*index_type
=
263 b
->options
->lower_ubo_ssbo_access_to_offsets
?
264 glsl_uint_type() : nir_address_format_to_glsl_type(addr_format
);
266 instr
->num_components
= glsl_get_vector_elements(index_type
);
267 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
, instr
->num_components
,
268 glsl_get_bit_size(index_type
), NULL
);
269 nir_builder_instr_insert(&b
->nb
, &instr
->instr
);
271 return &instr
->dest
.ssa
;
275 vtn_resource_reindex(struct vtn_builder
*b
, enum vtn_variable_mode mode
,
276 nir_ssa_def
*base_index
, nir_ssa_def
*offset_index
)
278 vtn_assert(b
->options
->environment
== NIR_SPIRV_VULKAN
);
280 nir_intrinsic_instr
*instr
=
281 nir_intrinsic_instr_create(b
->nb
.shader
,
282 nir_intrinsic_vulkan_resource_reindex
);
283 instr
->src
[0] = nir_src_for_ssa(base_index
);
284 instr
->src
[1] = nir_src_for_ssa(offset_index
);
285 nir_intrinsic_set_desc_type(instr
, vk_desc_type_for_mode(b
, mode
));
287 vtn_fail_if(mode
!= vtn_variable_mode_ubo
&& mode
!= vtn_variable_mode_ssbo
,
288 "Invalid mode for vulkan_resource_reindex");
290 nir_address_format addr_format
= vtn_mode_to_address_format(b
, mode
);
291 const struct glsl_type
*index_type
=
292 b
->options
->lower_ubo_ssbo_access_to_offsets
?
293 glsl_uint_type() : nir_address_format_to_glsl_type(addr_format
);
295 instr
->num_components
= glsl_get_vector_elements(index_type
);
296 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
, instr
->num_components
,
297 glsl_get_bit_size(index_type
), NULL
);
298 nir_builder_instr_insert(&b
->nb
, &instr
->instr
);
300 return &instr
->dest
.ssa
;
304 vtn_descriptor_load(struct vtn_builder
*b
, enum vtn_variable_mode mode
,
305 nir_ssa_def
*desc_index
)
307 vtn_assert(b
->options
->environment
== NIR_SPIRV_VULKAN
);
309 nir_intrinsic_instr
*desc_load
=
310 nir_intrinsic_instr_create(b
->nb
.shader
,
311 nir_intrinsic_load_vulkan_descriptor
);
312 desc_load
->src
[0] = nir_src_for_ssa(desc_index
);
313 nir_intrinsic_set_desc_type(desc_load
, vk_desc_type_for_mode(b
, mode
));
315 vtn_fail_if(mode
!= vtn_variable_mode_ubo
&& mode
!= vtn_variable_mode_ssbo
,
316 "Invalid mode for load_vulkan_descriptor");
318 nir_address_format addr_format
= vtn_mode_to_address_format(b
, mode
);
319 const struct glsl_type
*ptr_type
=
320 nir_address_format_to_glsl_type(addr_format
);
322 desc_load
->num_components
= glsl_get_vector_elements(ptr_type
);
323 nir_ssa_dest_init(&desc_load
->instr
, &desc_load
->dest
,
324 desc_load
->num_components
,
325 glsl_get_bit_size(ptr_type
), NULL
);
326 nir_builder_instr_insert(&b
->nb
, &desc_load
->instr
);
328 return &desc_load
->dest
.ssa
;
331 /* Dereference the given base pointer by the access chain */
332 static struct vtn_pointer
*
333 vtn_nir_deref_pointer_dereference(struct vtn_builder
*b
,
334 struct vtn_pointer
*base
,
335 struct vtn_access_chain
*deref_chain
)
337 struct vtn_type
*type
= base
->type
;
338 enum gl_access_qualifier access
= base
->access
| deref_chain
->access
;
341 nir_deref_instr
*tail
;
344 } else if (b
->options
->environment
== NIR_SPIRV_VULKAN
&&
345 vtn_pointer_is_external_block(b
, base
)) {
346 nir_ssa_def
*block_index
= base
->block_index
;
348 /* We dereferencing an external block pointer. Correctness of this
349 * operation relies on one particular line in the SPIR-V spec, section
350 * entitled "Validation Rules for Shader Capabilities":
352 * "Block and BufferBlock decorations cannot decorate a structure
353 * type that is nested at any level inside another structure type
354 * decorated with Block or BufferBlock."
356 * This means that we can detect the point where we cross over from
357 * descriptor indexing to buffer indexing by looking for the block
358 * decorated struct type. Anything before the block decorated struct
359 * type is a descriptor indexing operation and anything after the block
360 * decorated struct is a buffer offset operation.
363 /* Figure out the descriptor array index if any
365 * Some of the Vulkan CTS tests with hand-rolled SPIR-V have been known
366 * to forget the Block or BufferBlock decoration from time to time.
367 * It's more robust if we check for both !block_index and for the type
368 * to contain a block. This way there's a decent chance that arrays of
369 * UBOs/SSBOs will work correctly even if variable pointers are
372 nir_ssa_def
*desc_arr_idx
= NULL
;
373 if (!block_index
|| vtn_type_contains_block(b
, type
)) {
374 /* If our type contains a block, then we're still outside the block
375 * and we need to process enough levels of dereferences to get inside
378 if (deref_chain
->ptr_as_array
) {
379 unsigned aoa_size
= glsl_get_aoa_size(type
->type
);
380 desc_arr_idx
= vtn_access_link_as_ssa(b
, deref_chain
->link
[idx
],
381 MAX2(aoa_size
, 1), 32);
385 for (; idx
< deref_chain
->length
; idx
++) {
386 if (type
->base_type
!= vtn_base_type_array
) {
387 vtn_assert(type
->base_type
== vtn_base_type_struct
);
391 unsigned aoa_size
= glsl_get_aoa_size(type
->array_element
->type
);
392 nir_ssa_def
*arr_offset
=
393 vtn_access_link_as_ssa(b
, deref_chain
->link
[idx
],
394 MAX2(aoa_size
, 1), 32);
396 desc_arr_idx
= nir_iadd(&b
->nb
, desc_arr_idx
, arr_offset
);
398 desc_arr_idx
= arr_offset
;
400 type
= type
->array_element
;
401 access
|= type
->access
;
406 vtn_assert(base
->var
&& base
->type
);
407 block_index
= vtn_variable_resource_index(b
, base
->var
, desc_arr_idx
);
408 } else if (desc_arr_idx
) {
409 block_index
= vtn_resource_reindex(b
, base
->mode
,
410 block_index
, desc_arr_idx
);
413 if (idx
== deref_chain
->length
) {
414 /* The entire deref was consumed in finding the block index. Return
415 * a pointer which just has a block index and a later access chain
416 * will dereference deeper.
418 struct vtn_pointer
*ptr
= rzalloc(b
, struct vtn_pointer
);
419 ptr
->mode
= base
->mode
;
421 ptr
->block_index
= block_index
;
422 ptr
->access
= access
;
426 /* If we got here, there's more access chain to handle and we have the
427 * final block index. Insert a descriptor load and cast to a deref to
428 * start the deref chain.
430 nir_ssa_def
*desc
= vtn_descriptor_load(b
, base
->mode
, block_index
);
432 assert(base
->mode
== vtn_variable_mode_ssbo
||
433 base
->mode
== vtn_variable_mode_ubo
);
434 nir_variable_mode nir_mode
=
435 base
->mode
== vtn_variable_mode_ssbo
? nir_var_mem_ssbo
: nir_var_mem_ubo
;
437 tail
= nir_build_deref_cast(&b
->nb
, desc
, nir_mode
,
438 vtn_type_get_nir_type(b
, type
, base
->mode
),
439 base
->ptr_type
->stride
);
441 assert(base
->var
&& base
->var
->var
);
442 tail
= nir_build_deref_var(&b
->nb
, base
->var
->var
);
443 if (base
->ptr_type
&& base
->ptr_type
->type
) {
444 tail
->dest
.ssa
.num_components
=
445 glsl_get_vector_elements(base
->ptr_type
->type
);
446 tail
->dest
.ssa
.bit_size
= glsl_get_bit_size(base
->ptr_type
->type
);
450 if (idx
== 0 && deref_chain
->ptr_as_array
) {
451 /* We start with a deref cast to get the stride. Hopefully, we'll be
452 * able to delete that cast eventually.
454 tail
= nir_build_deref_cast(&b
->nb
, &tail
->dest
.ssa
, tail
->mode
,
455 tail
->type
, base
->ptr_type
->stride
);
457 nir_ssa_def
*index
= vtn_access_link_as_ssa(b
, deref_chain
->link
[0], 1,
458 tail
->dest
.ssa
.bit_size
);
459 tail
= nir_build_deref_ptr_as_array(&b
->nb
, tail
, index
);
463 for (; idx
< deref_chain
->length
; idx
++) {
464 if (glsl_type_is_struct_or_ifc(type
->type
)) {
465 vtn_assert(deref_chain
->link
[idx
].mode
== vtn_access_mode_literal
);
466 unsigned field
= deref_chain
->link
[idx
].id
;
467 tail
= nir_build_deref_struct(&b
->nb
, tail
, field
);
468 type
= type
->members
[field
];
470 nir_ssa_def
*arr_index
=
471 vtn_access_link_as_ssa(b
, deref_chain
->link
[idx
], 1,
472 tail
->dest
.ssa
.bit_size
);
473 tail
= nir_build_deref_array(&b
->nb
, tail
, arr_index
);
474 type
= type
->array_element
;
477 access
|= type
->access
;
480 struct vtn_pointer
*ptr
= rzalloc(b
, struct vtn_pointer
);
481 ptr
->mode
= base
->mode
;
483 ptr
->var
= base
->var
;
485 ptr
->access
= access
;
490 static struct vtn_pointer
*
491 vtn_ssa_offset_pointer_dereference(struct vtn_builder
*b
,
492 struct vtn_pointer
*base
,
493 struct vtn_access_chain
*deref_chain
)
495 nir_ssa_def
*block_index
= base
->block_index
;
496 nir_ssa_def
*offset
= base
->offset
;
497 struct vtn_type
*type
= base
->type
;
498 enum gl_access_qualifier access
= base
->access
;
501 if (base
->mode
== vtn_variable_mode_ubo
||
502 base
->mode
== vtn_variable_mode_ssbo
) {
504 vtn_assert(base
->var
&& base
->type
);
505 nir_ssa_def
*desc_arr_idx
;
506 if (glsl_type_is_array(type
->type
)) {
507 if (deref_chain
->length
>= 1) {
509 vtn_access_link_as_ssa(b
, deref_chain
->link
[0], 1, 32);
511 /* This consumes a level of type */
512 type
= type
->array_element
;
513 access
|= type
->access
;
515 /* This is annoying. We've been asked for a pointer to the
516 * array of UBOs/SSBOs and not a specifc buffer. Return a
517 * pointer with a descriptor index of 0 and we'll have to do
518 * a reindex later to adjust it to the right thing.
520 desc_arr_idx
= nir_imm_int(&b
->nb
, 0);
522 } else if (deref_chain
->ptr_as_array
) {
523 /* You can't have a zero-length OpPtrAccessChain */
524 vtn_assert(deref_chain
->length
>= 1);
525 desc_arr_idx
= vtn_access_link_as_ssa(b
, deref_chain
->link
[0], 1, 32);
527 /* We have a regular non-array SSBO. */
530 block_index
= vtn_variable_resource_index(b
, base
->var
, desc_arr_idx
);
531 } else if (deref_chain
->ptr_as_array
&&
532 type
->base_type
== vtn_base_type_struct
&& type
->block
) {
533 /* We are doing an OpPtrAccessChain on a pointer to a struct that is
534 * decorated block. This is an interesting corner in the SPIR-V
535 * spec. One interpretation would be that they client is clearly
536 * trying to treat that block as if it's an implicit array of blocks
537 * repeated in the buffer. However, the SPIR-V spec for the
538 * OpPtrAccessChain says:
540 * "Base is treated as the address of the first element of an
541 * array, and the Element element’s address is computed to be the
542 * base for the Indexes, as per OpAccessChain."
544 * Taken literally, that would mean that your struct type is supposed
545 * to be treated as an array of such a struct and, since it's
546 * decorated block, that means an array of blocks which corresponds
547 * to an array descriptor. Therefore, we need to do a reindex
548 * operation to add the index from the first link in the access chain
549 * to the index we recieved.
551 * The downside to this interpretation (there always is one) is that
552 * this might be somewhat surprising behavior to apps if they expect
553 * the implicit array behavior described above.
555 vtn_assert(deref_chain
->length
>= 1);
556 nir_ssa_def
*offset_index
=
557 vtn_access_link_as_ssa(b
, deref_chain
->link
[0], 1, 32);
560 block_index
= vtn_resource_reindex(b
, base
->mode
,
561 block_index
, offset_index
);
566 if (base
->mode
== vtn_variable_mode_push_constant
) {
567 /* Push constants neither need nor have a block index */
568 vtn_assert(!block_index
);
570 /* Start off with at the start of the push constant block. */
571 offset
= nir_imm_int(&b
->nb
, 0);
573 /* The code above should have ensured a block_index when needed. */
574 vtn_assert(block_index
);
576 /* Start off with at the start of the buffer. */
577 offset
= nir_imm_int(&b
->nb
, 0);
581 if (deref_chain
->ptr_as_array
&& idx
== 0) {
582 /* We need ptr_type for the stride */
583 vtn_assert(base
->ptr_type
);
585 /* We need at least one element in the chain */
586 vtn_assert(deref_chain
->length
>= 1);
588 nir_ssa_def
*elem_offset
=
589 vtn_access_link_as_ssa(b
, deref_chain
->link
[idx
],
590 base
->ptr_type
->stride
, offset
->bit_size
);
591 offset
= nir_iadd(&b
->nb
, offset
, elem_offset
);
595 for (; idx
< deref_chain
->length
; idx
++) {
596 switch (glsl_get_base_type(type
->type
)) {
599 case GLSL_TYPE_UINT16
:
600 case GLSL_TYPE_INT16
:
601 case GLSL_TYPE_UINT8
:
603 case GLSL_TYPE_UINT64
:
604 case GLSL_TYPE_INT64
:
605 case GLSL_TYPE_FLOAT
:
606 case GLSL_TYPE_FLOAT16
:
607 case GLSL_TYPE_DOUBLE
:
609 case GLSL_TYPE_ARRAY
: {
610 nir_ssa_def
*elem_offset
=
611 vtn_access_link_as_ssa(b
, deref_chain
->link
[idx
],
612 type
->stride
, offset
->bit_size
);
613 offset
= nir_iadd(&b
->nb
, offset
, elem_offset
);
614 type
= type
->array_element
;
615 access
|= type
->access
;
619 case GLSL_TYPE_INTERFACE
:
620 case GLSL_TYPE_STRUCT
: {
621 vtn_assert(deref_chain
->link
[idx
].mode
== vtn_access_mode_literal
);
622 unsigned member
= deref_chain
->link
[idx
].id
;
623 offset
= nir_iadd_imm(&b
->nb
, offset
, type
->offsets
[member
]);
624 type
= type
->members
[member
];
625 access
|= type
->access
;
630 vtn_fail("Invalid type for deref");
634 struct vtn_pointer
*ptr
= rzalloc(b
, struct vtn_pointer
);
635 ptr
->mode
= base
->mode
;
637 ptr
->block_index
= block_index
;
638 ptr
->offset
= offset
;
639 ptr
->access
= access
;
644 /* Dereference the given base pointer by the access chain */
645 static struct vtn_pointer
*
646 vtn_pointer_dereference(struct vtn_builder
*b
,
647 struct vtn_pointer
*base
,
648 struct vtn_access_chain
*deref_chain
)
650 if (vtn_pointer_uses_ssa_offset(b
, base
)) {
651 return vtn_ssa_offset_pointer_dereference(b
, base
, deref_chain
);
653 return vtn_nir_deref_pointer_dereference(b
, base
, deref_chain
);
658 vtn_pointer_to_deref(struct vtn_builder
*b
, struct vtn_pointer
*ptr
)
660 vtn_assert(!vtn_pointer_uses_ssa_offset(b
, ptr
));
662 struct vtn_access_chain chain
= {
665 ptr
= vtn_nir_deref_pointer_dereference(b
, ptr
, &chain
);
672 _vtn_local_load_store(struct vtn_builder
*b
, bool load
, nir_deref_instr
*deref
,
673 struct vtn_ssa_value
*inout
,
674 enum gl_access_qualifier access
)
676 if (glsl_type_is_vector_or_scalar(deref
->type
)) {
678 inout
->def
= nir_load_deref_with_access(&b
->nb
, deref
, access
);
680 nir_store_deref_with_access(&b
->nb
, deref
, inout
->def
, ~0, access
);
682 } else if (glsl_type_is_array(deref
->type
) ||
683 glsl_type_is_matrix(deref
->type
)) {
684 unsigned elems
= glsl_get_length(deref
->type
);
685 for (unsigned i
= 0; i
< elems
; i
++) {
686 nir_deref_instr
*child
=
687 nir_build_deref_array_imm(&b
->nb
, deref
, i
);
688 _vtn_local_load_store(b
, load
, child
, inout
->elems
[i
], access
);
691 vtn_assert(glsl_type_is_struct_or_ifc(deref
->type
));
692 unsigned elems
= glsl_get_length(deref
->type
);
693 for (unsigned i
= 0; i
< elems
; i
++) {
694 nir_deref_instr
*child
= nir_build_deref_struct(&b
->nb
, deref
, i
);
695 _vtn_local_load_store(b
, load
, child
, inout
->elems
[i
], access
);
701 vtn_nir_deref(struct vtn_builder
*b
, uint32_t id
)
703 struct vtn_pointer
*ptr
= vtn_value(b
, id
, vtn_value_type_pointer
)->pointer
;
704 return vtn_pointer_to_deref(b
, ptr
);
708 * Gets the NIR-level deref tail, which may have as a child an array deref
709 * selecting which component due to OpAccessChain supporting per-component
710 * indexing in SPIR-V.
712 static nir_deref_instr
*
713 get_deref_tail(nir_deref_instr
*deref
)
715 if (deref
->deref_type
!= nir_deref_type_array
)
718 nir_deref_instr
*parent
=
719 nir_instr_as_deref(deref
->parent
.ssa
->parent_instr
);
721 if (glsl_type_is_vector(parent
->type
))
727 struct vtn_ssa_value
*
728 vtn_local_load(struct vtn_builder
*b
, nir_deref_instr
*src
,
729 enum gl_access_qualifier access
)
731 nir_deref_instr
*src_tail
= get_deref_tail(src
);
732 struct vtn_ssa_value
*val
= vtn_create_ssa_value(b
, src_tail
->type
);
733 _vtn_local_load_store(b
, true, src_tail
, val
, access
);
735 if (src_tail
!= src
) {
736 val
->type
= src
->type
;
737 val
->def
= nir_vector_extract(&b
->nb
, val
->def
, src
->arr
.index
.ssa
);
744 vtn_local_store(struct vtn_builder
*b
, struct vtn_ssa_value
*src
,
745 nir_deref_instr
*dest
, enum gl_access_qualifier access
)
747 nir_deref_instr
*dest_tail
= get_deref_tail(dest
);
749 if (dest_tail
!= dest
) {
750 struct vtn_ssa_value
*val
= vtn_create_ssa_value(b
, dest_tail
->type
);
751 _vtn_local_load_store(b
, true, dest_tail
, val
, access
);
753 val
->def
= nir_vector_insert(&b
->nb
, val
->def
, src
->def
,
754 dest
->arr
.index
.ssa
);
755 _vtn_local_load_store(b
, false, dest_tail
, val
, access
);
757 _vtn_local_load_store(b
, false, dest_tail
, src
, access
);
762 vtn_pointer_to_offset(struct vtn_builder
*b
, struct vtn_pointer
*ptr
,
763 nir_ssa_def
**index_out
)
765 assert(vtn_pointer_uses_ssa_offset(b
, ptr
));
767 struct vtn_access_chain chain
= {
770 ptr
= vtn_ssa_offset_pointer_dereference(b
, ptr
, &chain
);
772 *index_out
= ptr
->block_index
;
776 /* Tries to compute the size of an interface block based on the strides and
777 * offsets that are provided to us in the SPIR-V source.
780 vtn_type_block_size(struct vtn_builder
*b
, struct vtn_type
*type
)
782 enum glsl_base_type base_type
= glsl_get_base_type(type
->type
);
786 case GLSL_TYPE_UINT16
:
787 case GLSL_TYPE_INT16
:
788 case GLSL_TYPE_UINT8
:
790 case GLSL_TYPE_UINT64
:
791 case GLSL_TYPE_INT64
:
792 case GLSL_TYPE_FLOAT
:
793 case GLSL_TYPE_FLOAT16
:
795 case GLSL_TYPE_DOUBLE
: {
796 unsigned cols
= type
->row_major
? glsl_get_vector_elements(type
->type
) :
797 glsl_get_matrix_columns(type
->type
);
799 vtn_assert(type
->stride
> 0);
800 return type
->stride
* cols
;
802 unsigned type_size
= glsl_get_bit_size(type
->type
) / 8;
803 return glsl_get_vector_elements(type
->type
) * type_size
;
807 case GLSL_TYPE_STRUCT
:
808 case GLSL_TYPE_INTERFACE
: {
810 unsigned num_fields
= glsl_get_length(type
->type
);
811 for (unsigned f
= 0; f
< num_fields
; f
++) {
812 unsigned field_end
= type
->offsets
[f
] +
813 vtn_type_block_size(b
, type
->members
[f
]);
814 size
= MAX2(size
, field_end
);
819 case GLSL_TYPE_ARRAY
:
820 vtn_assert(type
->stride
> 0);
821 vtn_assert(glsl_get_length(type
->type
) > 0);
822 return type
->stride
* glsl_get_length(type
->type
);
825 vtn_fail("Invalid block type");
831 _vtn_load_store_tail(struct vtn_builder
*b
, nir_intrinsic_op op
, bool load
,
832 nir_ssa_def
*index
, nir_ssa_def
*offset
,
833 unsigned access_offset
, unsigned access_size
,
834 struct vtn_ssa_value
**inout
, const struct glsl_type
*type
,
835 enum gl_access_qualifier access
)
837 nir_intrinsic_instr
*instr
= nir_intrinsic_instr_create(b
->nb
.shader
, op
);
838 instr
->num_components
= glsl_get_vector_elements(type
);
840 /* Booleans usually shouldn't show up in external memory in SPIR-V.
841 * However, they do for certain older GLSLang versions and can for shared
842 * memory when we lower access chains internally.
844 const unsigned data_bit_size
= glsl_type_is_boolean(type
) ? 32 :
845 glsl_get_bit_size(type
);
849 nir_intrinsic_set_write_mask(instr
, (1 << instr
->num_components
) - 1);
850 instr
->src
[src
++] = nir_src_for_ssa((*inout
)->def
);
853 if (op
== nir_intrinsic_load_push_constant
) {
854 nir_intrinsic_set_base(instr
, access_offset
);
855 nir_intrinsic_set_range(instr
, access_size
);
858 if (op
== nir_intrinsic_load_ubo
||
859 op
== nir_intrinsic_load_ssbo
||
860 op
== nir_intrinsic_store_ssbo
) {
861 nir_intrinsic_set_access(instr
, access
);
864 /* With extensions like relaxed_block_layout, we really can't guarantee
865 * much more than scalar alignment.
867 if (op
!= nir_intrinsic_load_push_constant
)
868 nir_intrinsic_set_align(instr
, data_bit_size
/ 8, 0);
871 instr
->src
[src
++] = nir_src_for_ssa(index
);
873 if (op
== nir_intrinsic_load_push_constant
) {
874 /* We need to subtract the offset from where the intrinsic will load the
877 nir_src_for_ssa(nir_isub(&b
->nb
, offset
,
878 nir_imm_int(&b
->nb
, access_offset
)));
880 instr
->src
[src
++] = nir_src_for_ssa(offset
);
884 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
,
885 instr
->num_components
, data_bit_size
, NULL
);
886 (*inout
)->def
= &instr
->dest
.ssa
;
889 nir_builder_instr_insert(&b
->nb
, &instr
->instr
);
891 if (load
&& glsl_get_base_type(type
) == GLSL_TYPE_BOOL
)
892 (*inout
)->def
= nir_ine(&b
->nb
, (*inout
)->def
, nir_imm_int(&b
->nb
, 0));
896 _vtn_block_load_store(struct vtn_builder
*b
, nir_intrinsic_op op
, bool load
,
897 nir_ssa_def
*index
, nir_ssa_def
*offset
,
898 unsigned access_offset
, unsigned access_size
,
899 struct vtn_type
*type
, enum gl_access_qualifier access
,
900 struct vtn_ssa_value
**inout
)
902 enum glsl_base_type base_type
= glsl_get_base_type(type
->type
);
906 case GLSL_TYPE_UINT16
:
907 case GLSL_TYPE_INT16
:
908 case GLSL_TYPE_UINT8
:
910 case GLSL_TYPE_UINT64
:
911 case GLSL_TYPE_INT64
:
912 case GLSL_TYPE_FLOAT
:
913 case GLSL_TYPE_FLOAT16
:
914 case GLSL_TYPE_DOUBLE
:
916 /* This is where things get interesting. At this point, we've hit
917 * a vector, a scalar, or a matrix.
919 if (glsl_type_is_matrix(type
->type
)) {
920 /* Loading the whole matrix */
921 struct vtn_ssa_value
*transpose
;
922 unsigned num_ops
, vec_width
, col_stride
;
923 if (type
->row_major
) {
924 num_ops
= glsl_get_vector_elements(type
->type
);
925 vec_width
= glsl_get_matrix_columns(type
->type
);
926 col_stride
= type
->array_element
->stride
;
928 const struct glsl_type
*transpose_type
=
929 glsl_matrix_type(base_type
, vec_width
, num_ops
);
930 *inout
= vtn_create_ssa_value(b
, transpose_type
);
932 transpose
= vtn_ssa_transpose(b
, *inout
);
936 num_ops
= glsl_get_matrix_columns(type
->type
);
937 vec_width
= glsl_get_vector_elements(type
->type
);
938 col_stride
= type
->stride
;
941 for (unsigned i
= 0; i
< num_ops
; i
++) {
942 nir_ssa_def
*elem_offset
=
943 nir_iadd_imm(&b
->nb
, offset
, i
* col_stride
);
944 _vtn_load_store_tail(b
, op
, load
, index
, elem_offset
,
945 access_offset
, access_size
,
947 glsl_vector_type(base_type
, vec_width
),
948 type
->access
| access
);
951 if (load
&& type
->row_major
)
952 *inout
= vtn_ssa_transpose(b
, *inout
);
954 unsigned elems
= glsl_get_vector_elements(type
->type
);
955 unsigned type_size
= glsl_get_bit_size(type
->type
) / 8;
956 if (elems
== 1 || type
->stride
== type_size
) {
957 /* This is a tightly-packed normal scalar or vector load */
958 vtn_assert(glsl_type_is_vector_or_scalar(type
->type
));
959 _vtn_load_store_tail(b
, op
, load
, index
, offset
,
960 access_offset
, access_size
,
962 type
->access
| access
);
964 /* This is a strided load. We have to load N things separately.
965 * This is the single column of a row-major matrix case.
967 vtn_assert(type
->stride
> type_size
);
968 vtn_assert(type
->stride
% type_size
== 0);
970 nir_ssa_def
*per_comp
[4];
971 for (unsigned i
= 0; i
< elems
; i
++) {
972 nir_ssa_def
*elem_offset
=
973 nir_iadd_imm(&b
->nb
, offset
, i
* type
->stride
);
974 struct vtn_ssa_value
*comp
, temp_val
;
976 temp_val
.def
= nir_channel(&b
->nb
, (*inout
)->def
, i
);
977 temp_val
.type
= glsl_scalar_type(base_type
);
980 _vtn_load_store_tail(b
, op
, load
, index
, elem_offset
,
981 access_offset
, access_size
,
982 &comp
, glsl_scalar_type(base_type
),
983 type
->access
| access
);
984 per_comp
[i
] = comp
->def
;
989 *inout
= vtn_create_ssa_value(b
, type
->type
);
990 (*inout
)->def
= nir_vec(&b
->nb
, per_comp
, elems
);
996 case GLSL_TYPE_ARRAY
: {
997 unsigned elems
= glsl_get_length(type
->type
);
998 for (unsigned i
= 0; i
< elems
; i
++) {
999 nir_ssa_def
*elem_off
=
1000 nir_iadd_imm(&b
->nb
, offset
, i
* type
->stride
);
1001 _vtn_block_load_store(b
, op
, load
, index
, elem_off
,
1002 access_offset
, access_size
,
1003 type
->array_element
,
1004 type
->array_element
->access
| access
,
1005 &(*inout
)->elems
[i
]);
1010 case GLSL_TYPE_INTERFACE
:
1011 case GLSL_TYPE_STRUCT
: {
1012 unsigned elems
= glsl_get_length(type
->type
);
1013 for (unsigned i
= 0; i
< elems
; i
++) {
1014 nir_ssa_def
*elem_off
=
1015 nir_iadd_imm(&b
->nb
, offset
, type
->offsets
[i
]);
1016 _vtn_block_load_store(b
, op
, load
, index
, elem_off
,
1017 access_offset
, access_size
,
1019 type
->members
[i
]->access
| access
,
1020 &(*inout
)->elems
[i
]);
1026 vtn_fail("Invalid block member type");
1030 static struct vtn_ssa_value
*
1031 vtn_block_load(struct vtn_builder
*b
, struct vtn_pointer
*src
,
1032 enum gl_access_qualifier access
)
1034 nir_intrinsic_op op
;
1035 unsigned access_offset
= 0, access_size
= 0;
1036 switch (src
->mode
) {
1037 case vtn_variable_mode_ubo
:
1038 op
= nir_intrinsic_load_ubo
;
1040 case vtn_variable_mode_ssbo
:
1041 op
= nir_intrinsic_load_ssbo
;
1043 case vtn_variable_mode_push_constant
:
1044 op
= nir_intrinsic_load_push_constant
;
1045 access_size
= b
->shader
->num_uniforms
;
1047 case vtn_variable_mode_workgroup
:
1048 op
= nir_intrinsic_load_shared
;
1051 vtn_fail("Invalid block variable mode");
1054 nir_ssa_def
*offset
, *index
= NULL
;
1055 offset
= vtn_pointer_to_offset(b
, src
, &index
);
1057 struct vtn_ssa_value
*value
= vtn_create_ssa_value(b
, src
->type
->type
);
1058 _vtn_block_load_store(b
, op
, true, index
, offset
,
1059 access_offset
, access_size
,
1060 src
->type
, src
->access
| access
, &value
);
1065 vtn_block_store(struct vtn_builder
*b
, struct vtn_ssa_value
*src
,
1066 struct vtn_pointer
*dst
, enum gl_access_qualifier access
)
1068 nir_intrinsic_op op
;
1069 switch (dst
->mode
) {
1070 case vtn_variable_mode_ssbo
:
1071 op
= nir_intrinsic_store_ssbo
;
1073 case vtn_variable_mode_workgroup
:
1074 op
= nir_intrinsic_store_shared
;
1077 vtn_fail("Invalid block variable mode");
1080 nir_ssa_def
*offset
, *index
= NULL
;
1081 offset
= vtn_pointer_to_offset(b
, dst
, &index
);
1083 _vtn_block_load_store(b
, op
, false, index
, offset
,
1084 0, 0, dst
->type
, dst
->access
| access
, &src
);
1088 _vtn_variable_load_store(struct vtn_builder
*b
, bool load
,
1089 struct vtn_pointer
*ptr
,
1090 enum gl_access_qualifier access
,
1091 struct vtn_ssa_value
**inout
)
1093 if (ptr
->mode
== vtn_variable_mode_uniform
) {
1094 if (ptr
->type
->base_type
== vtn_base_type_image
||
1095 ptr
->type
->base_type
== vtn_base_type_sampler
) {
1096 /* See also our handling of OpTypeSampler and OpTypeImage */
1098 (*inout
)->def
= vtn_pointer_to_ssa(b
, ptr
);
1100 } else if (ptr
->type
->base_type
== vtn_base_type_sampled_image
) {
1101 /* See also our handling of OpTypeSampledImage */
1103 struct vtn_sampled_image si
= {
1104 .image
= vtn_pointer_to_deref(b
, ptr
),
1105 .sampler
= vtn_pointer_to_deref(b
, ptr
),
1107 (*inout
)->def
= vtn_sampled_image_to_nir_ssa(b
, si
);
1112 enum glsl_base_type base_type
= glsl_get_base_type(ptr
->type
->type
);
1113 switch (base_type
) {
1114 case GLSL_TYPE_UINT
:
1116 case GLSL_TYPE_UINT16
:
1117 case GLSL_TYPE_INT16
:
1118 case GLSL_TYPE_UINT8
:
1119 case GLSL_TYPE_INT8
:
1120 case GLSL_TYPE_UINT64
:
1121 case GLSL_TYPE_INT64
:
1122 case GLSL_TYPE_FLOAT
:
1123 case GLSL_TYPE_FLOAT16
:
1124 case GLSL_TYPE_BOOL
:
1125 case GLSL_TYPE_DOUBLE
:
1126 if (glsl_type_is_vector_or_scalar(ptr
->type
->type
)) {
1127 /* We hit a vector or scalar; go ahead and emit the load[s] */
1128 nir_deref_instr
*deref
= vtn_pointer_to_deref(b
, ptr
);
1129 if (vtn_mode_is_cross_invocation(b
, ptr
->mode
)) {
1130 /* If it's cross-invocation, we call nir_load/store_deref
1131 * directly. The vtn_local_load/store helpers are too clever and
1132 * do magic to avoid array derefs of vectors. That magic is both
1133 * less efficient than the direct load/store and, in the case of
1134 * stores, is broken because it creates a race condition if two
1135 * threads are writing to different components of the same vector
1136 * due to the load+insert+store it uses to emulate the array
1140 (*inout
)->def
= nir_load_deref_with_access(&b
->nb
, deref
,
1141 ptr
->type
->access
| access
);
1143 nir_store_deref_with_access(&b
->nb
, deref
, (*inout
)->def
, ~0,
1144 ptr
->type
->access
| access
);
1148 *inout
= vtn_local_load(b
, deref
, ptr
->type
->access
| access
);
1150 vtn_local_store(b
, *inout
, deref
, ptr
->type
->access
| access
);
1157 case GLSL_TYPE_INTERFACE
:
1158 case GLSL_TYPE_ARRAY
:
1159 case GLSL_TYPE_STRUCT
: {
1160 unsigned elems
= glsl_get_length(ptr
->type
->type
);
1161 struct vtn_access_chain chain
= {
1164 { .mode
= vtn_access_mode_literal
, },
1167 for (unsigned i
= 0; i
< elems
; i
++) {
1168 chain
.link
[0].id
= i
;
1169 struct vtn_pointer
*elem
= vtn_pointer_dereference(b
, ptr
, &chain
);
1170 _vtn_variable_load_store(b
, load
, elem
, ptr
->type
->access
| access
,
1171 &(*inout
)->elems
[i
]);
1177 vtn_fail("Invalid access chain type");
1181 struct vtn_ssa_value
*
1182 vtn_variable_load(struct vtn_builder
*b
, struct vtn_pointer
*src
,
1183 enum gl_access_qualifier access
)
1185 if (vtn_pointer_uses_ssa_offset(b
, src
)) {
1186 return vtn_block_load(b
, src
, access
);
1188 struct vtn_ssa_value
*val
= vtn_create_ssa_value(b
, src
->type
->type
);
1189 _vtn_variable_load_store(b
, true, src
, src
->access
| access
, &val
);
1195 vtn_variable_store(struct vtn_builder
*b
, struct vtn_ssa_value
*src
,
1196 struct vtn_pointer
*dest
, enum gl_access_qualifier access
)
1198 if (vtn_pointer_uses_ssa_offset(b
, dest
)) {
1199 vtn_assert(dest
->mode
== vtn_variable_mode_ssbo
||
1200 dest
->mode
== vtn_variable_mode_workgroup
);
1201 vtn_block_store(b
, src
, dest
, access
);
1203 _vtn_variable_load_store(b
, false, dest
, dest
->access
| access
, &src
);
1208 _vtn_variable_copy(struct vtn_builder
*b
, struct vtn_pointer
*dest
,
1209 struct vtn_pointer
*src
, enum gl_access_qualifier dest_access
,
1210 enum gl_access_qualifier src_access
)
1212 vtn_assert(glsl_get_bare_type(src
->type
->type
) ==
1213 glsl_get_bare_type(dest
->type
->type
));
1214 enum glsl_base_type base_type
= glsl_get_base_type(src
->type
->type
);
1215 switch (base_type
) {
1216 case GLSL_TYPE_UINT
:
1218 case GLSL_TYPE_UINT16
:
1219 case GLSL_TYPE_INT16
:
1220 case GLSL_TYPE_UINT8
:
1221 case GLSL_TYPE_INT8
:
1222 case GLSL_TYPE_UINT64
:
1223 case GLSL_TYPE_INT64
:
1224 case GLSL_TYPE_FLOAT
:
1225 case GLSL_TYPE_FLOAT16
:
1226 case GLSL_TYPE_DOUBLE
:
1227 case GLSL_TYPE_BOOL
:
1228 /* At this point, we have a scalar, vector, or matrix so we know that
1229 * there cannot be any structure splitting still in the way. By
1230 * stopping at the matrix level rather than the vector level, we
1231 * ensure that matrices get loaded in the optimal way even if they
1232 * are storred row-major in a UBO.
1234 vtn_variable_store(b
, vtn_variable_load(b
, src
, src_access
), dest
, dest_access
);
1237 case GLSL_TYPE_INTERFACE
:
1238 case GLSL_TYPE_ARRAY
:
1239 case GLSL_TYPE_STRUCT
: {
1240 struct vtn_access_chain chain
= {
1243 { .mode
= vtn_access_mode_literal
, },
1246 unsigned elems
= glsl_get_length(src
->type
->type
);
1247 for (unsigned i
= 0; i
< elems
; i
++) {
1248 chain
.link
[0].id
= i
;
1249 struct vtn_pointer
*src_elem
=
1250 vtn_pointer_dereference(b
, src
, &chain
);
1251 struct vtn_pointer
*dest_elem
=
1252 vtn_pointer_dereference(b
, dest
, &chain
);
1254 _vtn_variable_copy(b
, dest_elem
, src_elem
, dest_access
, src_access
);
1260 vtn_fail("Invalid access chain type");
1265 vtn_variable_copy(struct vtn_builder
*b
, struct vtn_pointer
*dest
,
1266 struct vtn_pointer
*src
, enum gl_access_qualifier dest_access
,
1267 enum gl_access_qualifier src_access
)
1269 /* TODO: At some point, we should add a special-case for when we can
1270 * just emit a copy_var intrinsic.
1272 _vtn_variable_copy(b
, dest
, src
, dest_access
, src_access
);
1276 set_mode_system_value(struct vtn_builder
*b
, nir_variable_mode
*mode
)
1278 vtn_assert(*mode
== nir_var_system_value
|| *mode
== nir_var_shader_in
);
1279 *mode
= nir_var_system_value
;
1283 vtn_get_builtin_location(struct vtn_builder
*b
,
1284 SpvBuiltIn builtin
, int *location
,
1285 nir_variable_mode
*mode
)
1288 case SpvBuiltInPosition
:
1289 *location
= VARYING_SLOT_POS
;
1291 case SpvBuiltInPointSize
:
1292 *location
= VARYING_SLOT_PSIZ
;
1294 case SpvBuiltInClipDistance
:
1295 *location
= VARYING_SLOT_CLIP_DIST0
; /* XXX CLIP_DIST1? */
1297 case SpvBuiltInCullDistance
:
1298 *location
= VARYING_SLOT_CULL_DIST0
;
1300 case SpvBuiltInVertexId
:
1301 case SpvBuiltInVertexIndex
:
1302 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't
1303 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the
1304 * same as gl_VertexID, which is non-zero-based, and removes
1305 * VertexIndex. Since they're both defined to be non-zero-based, we use
1306 * SYSTEM_VALUE_VERTEX_ID for both.
1308 *location
= SYSTEM_VALUE_VERTEX_ID
;
1309 set_mode_system_value(b
, mode
);
1311 case SpvBuiltInInstanceIndex
:
1312 *location
= SYSTEM_VALUE_INSTANCE_INDEX
;
1313 set_mode_system_value(b
, mode
);
1315 case SpvBuiltInInstanceId
:
1316 *location
= SYSTEM_VALUE_INSTANCE_ID
;
1317 set_mode_system_value(b
, mode
);
1319 case SpvBuiltInPrimitiveId
:
1320 if (b
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1321 vtn_assert(*mode
== nir_var_shader_in
);
1322 *location
= VARYING_SLOT_PRIMITIVE_ID
;
1323 } else if (*mode
== nir_var_shader_out
) {
1324 *location
= VARYING_SLOT_PRIMITIVE_ID
;
1326 *location
= SYSTEM_VALUE_PRIMITIVE_ID
;
1327 set_mode_system_value(b
, mode
);
1330 case SpvBuiltInInvocationId
:
1331 *location
= SYSTEM_VALUE_INVOCATION_ID
;
1332 set_mode_system_value(b
, mode
);
1334 case SpvBuiltInLayer
:
1335 *location
= VARYING_SLOT_LAYER
;
1336 if (b
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
)
1337 *mode
= nir_var_shader_in
;
1338 else if (b
->shader
->info
.stage
== MESA_SHADER_GEOMETRY
)
1339 *mode
= nir_var_shader_out
;
1340 else if (b
->options
&& b
->options
->caps
.shader_viewport_index_layer
&&
1341 (b
->shader
->info
.stage
== MESA_SHADER_VERTEX
||
1342 b
->shader
->info
.stage
== MESA_SHADER_TESS_EVAL
))
1343 *mode
= nir_var_shader_out
;
1345 vtn_fail("invalid stage for SpvBuiltInLayer");
1347 case SpvBuiltInViewportIndex
:
1348 *location
= VARYING_SLOT_VIEWPORT
;
1349 if (b
->shader
->info
.stage
== MESA_SHADER_GEOMETRY
)
1350 *mode
= nir_var_shader_out
;
1351 else if (b
->options
&& b
->options
->caps
.shader_viewport_index_layer
&&
1352 (b
->shader
->info
.stage
== MESA_SHADER_VERTEX
||
1353 b
->shader
->info
.stage
== MESA_SHADER_TESS_EVAL
))
1354 *mode
= nir_var_shader_out
;
1355 else if (b
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
)
1356 *mode
= nir_var_shader_in
;
1358 vtn_fail("invalid stage for SpvBuiltInViewportIndex");
1360 case SpvBuiltInTessLevelOuter
:
1361 *location
= VARYING_SLOT_TESS_LEVEL_OUTER
;
1363 case SpvBuiltInTessLevelInner
:
1364 *location
= VARYING_SLOT_TESS_LEVEL_INNER
;
1366 case SpvBuiltInTessCoord
:
1367 *location
= SYSTEM_VALUE_TESS_COORD
;
1368 set_mode_system_value(b
, mode
);
1370 case SpvBuiltInPatchVertices
:
1371 *location
= SYSTEM_VALUE_VERTICES_IN
;
1372 set_mode_system_value(b
, mode
);
1374 case SpvBuiltInFragCoord
:
1375 vtn_assert(*mode
== nir_var_shader_in
);
1376 if (b
->options
&& b
->options
->frag_coord_is_sysval
) {
1377 *mode
= nir_var_system_value
;
1378 *location
= SYSTEM_VALUE_FRAG_COORD
;
1380 *location
= VARYING_SLOT_POS
;
1383 case SpvBuiltInPointCoord
:
1384 *location
= VARYING_SLOT_PNTC
;
1385 vtn_assert(*mode
== nir_var_shader_in
);
1387 case SpvBuiltInFrontFacing
:
1388 *location
= SYSTEM_VALUE_FRONT_FACE
;
1389 set_mode_system_value(b
, mode
);
1391 case SpvBuiltInSampleId
:
1392 *location
= SYSTEM_VALUE_SAMPLE_ID
;
1393 set_mode_system_value(b
, mode
);
1395 case SpvBuiltInSamplePosition
:
1396 *location
= SYSTEM_VALUE_SAMPLE_POS
;
1397 set_mode_system_value(b
, mode
);
1399 case SpvBuiltInSampleMask
:
1400 if (*mode
== nir_var_shader_out
) {
1401 *location
= FRAG_RESULT_SAMPLE_MASK
;
1403 *location
= SYSTEM_VALUE_SAMPLE_MASK_IN
;
1404 set_mode_system_value(b
, mode
);
1407 case SpvBuiltInFragDepth
:
1408 *location
= FRAG_RESULT_DEPTH
;
1409 vtn_assert(*mode
== nir_var_shader_out
);
1411 case SpvBuiltInHelperInvocation
:
1412 *location
= SYSTEM_VALUE_HELPER_INVOCATION
;
1413 set_mode_system_value(b
, mode
);
1415 case SpvBuiltInNumWorkgroups
:
1416 *location
= SYSTEM_VALUE_NUM_WORK_GROUPS
;
1417 set_mode_system_value(b
, mode
);
1419 case SpvBuiltInWorkgroupSize
:
1420 *location
= SYSTEM_VALUE_LOCAL_GROUP_SIZE
;
1421 set_mode_system_value(b
, mode
);
1423 case SpvBuiltInWorkgroupId
:
1424 *location
= SYSTEM_VALUE_WORK_GROUP_ID
;
1425 set_mode_system_value(b
, mode
);
1427 case SpvBuiltInLocalInvocationId
:
1428 *location
= SYSTEM_VALUE_LOCAL_INVOCATION_ID
;
1429 set_mode_system_value(b
, mode
);
1431 case SpvBuiltInLocalInvocationIndex
:
1432 *location
= SYSTEM_VALUE_LOCAL_INVOCATION_INDEX
;
1433 set_mode_system_value(b
, mode
);
1435 case SpvBuiltInGlobalInvocationId
:
1436 *location
= SYSTEM_VALUE_GLOBAL_INVOCATION_ID
;
1437 set_mode_system_value(b
, mode
);
1439 case SpvBuiltInGlobalLinearId
:
1440 *location
= SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX
;
1441 set_mode_system_value(b
, mode
);
1443 case SpvBuiltInGlobalOffset
:
1444 *location
= SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID
;
1445 set_mode_system_value(b
, mode
);
1447 case SpvBuiltInBaseVertex
:
1448 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same
1449 * semantic as Vulkan BaseVertex (SYSTEM_VALUE_FIRST_VERTEX).
1451 if (b
->options
->environment
== NIR_SPIRV_OPENGL
)
1452 *location
= SYSTEM_VALUE_BASE_VERTEX
;
1454 *location
= SYSTEM_VALUE_FIRST_VERTEX
;
1455 set_mode_system_value(b
, mode
);
1457 case SpvBuiltInBaseInstance
:
1458 *location
= SYSTEM_VALUE_BASE_INSTANCE
;
1459 set_mode_system_value(b
, mode
);
1461 case SpvBuiltInDrawIndex
:
1462 *location
= SYSTEM_VALUE_DRAW_ID
;
1463 set_mode_system_value(b
, mode
);
1465 case SpvBuiltInSubgroupSize
:
1466 *location
= SYSTEM_VALUE_SUBGROUP_SIZE
;
1467 set_mode_system_value(b
, mode
);
1469 case SpvBuiltInSubgroupId
:
1470 *location
= SYSTEM_VALUE_SUBGROUP_ID
;
1471 set_mode_system_value(b
, mode
);
1473 case SpvBuiltInSubgroupLocalInvocationId
:
1474 *location
= SYSTEM_VALUE_SUBGROUP_INVOCATION
;
1475 set_mode_system_value(b
, mode
);
1477 case SpvBuiltInNumSubgroups
:
1478 *location
= SYSTEM_VALUE_NUM_SUBGROUPS
;
1479 set_mode_system_value(b
, mode
);
1481 case SpvBuiltInDeviceIndex
:
1482 *location
= SYSTEM_VALUE_DEVICE_INDEX
;
1483 set_mode_system_value(b
, mode
);
1485 case SpvBuiltInViewIndex
:
1486 if (b
->options
&& b
->options
->view_index_is_input
) {
1487 *location
= VARYING_SLOT_VIEW_INDEX
;
1488 vtn_assert(*mode
== nir_var_shader_in
);
1490 *location
= SYSTEM_VALUE_VIEW_INDEX
;
1491 set_mode_system_value(b
, mode
);
1494 case SpvBuiltInSubgroupEqMask
:
1495 *location
= SYSTEM_VALUE_SUBGROUP_EQ_MASK
,
1496 set_mode_system_value(b
, mode
);
1498 case SpvBuiltInSubgroupGeMask
:
1499 *location
= SYSTEM_VALUE_SUBGROUP_GE_MASK
,
1500 set_mode_system_value(b
, mode
);
1502 case SpvBuiltInSubgroupGtMask
:
1503 *location
= SYSTEM_VALUE_SUBGROUP_GT_MASK
,
1504 set_mode_system_value(b
, mode
);
1506 case SpvBuiltInSubgroupLeMask
:
1507 *location
= SYSTEM_VALUE_SUBGROUP_LE_MASK
,
1508 set_mode_system_value(b
, mode
);
1510 case SpvBuiltInSubgroupLtMask
:
1511 *location
= SYSTEM_VALUE_SUBGROUP_LT_MASK
,
1512 set_mode_system_value(b
, mode
);
1514 case SpvBuiltInFragStencilRefEXT
:
1515 *location
= FRAG_RESULT_STENCIL
;
1516 vtn_assert(*mode
== nir_var_shader_out
);
1518 case SpvBuiltInWorkDim
:
1519 *location
= SYSTEM_VALUE_WORK_DIM
;
1520 set_mode_system_value(b
, mode
);
1522 case SpvBuiltInGlobalSize
:
1523 *location
= SYSTEM_VALUE_GLOBAL_GROUP_SIZE
;
1524 set_mode_system_value(b
, mode
);
1526 case SpvBuiltInBaryCoordNoPerspAMD
:
1527 *location
= SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL
;
1528 set_mode_system_value(b
, mode
);
1530 case SpvBuiltInBaryCoordNoPerspCentroidAMD
:
1531 *location
= SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID
;
1532 set_mode_system_value(b
, mode
);
1534 case SpvBuiltInBaryCoordNoPerspSampleAMD
:
1535 *location
= SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE
;
1536 set_mode_system_value(b
, mode
);
1538 case SpvBuiltInBaryCoordSmoothAMD
:
1539 *location
= SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL
;
1540 set_mode_system_value(b
, mode
);
1542 case SpvBuiltInBaryCoordSmoothCentroidAMD
:
1543 *location
= SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID
;
1544 set_mode_system_value(b
, mode
);
1546 case SpvBuiltInBaryCoordSmoothSampleAMD
:
1547 *location
= SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE
;
1548 set_mode_system_value(b
, mode
);
1550 case SpvBuiltInBaryCoordPullModelAMD
:
1551 *location
= SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL
;
1552 set_mode_system_value(b
, mode
);
1555 vtn_fail("Unsupported builtin: %s (%u)",
1556 spirv_builtin_to_string(builtin
), builtin
);
1561 apply_var_decoration(struct vtn_builder
*b
,
1562 struct nir_variable_data
*var_data
,
1563 const struct vtn_decoration
*dec
)
1565 switch (dec
->decoration
) {
1566 case SpvDecorationRelaxedPrecision
:
1567 break; /* FIXME: Do nothing with this for now. */
1568 case SpvDecorationNoPerspective
:
1569 var_data
->interpolation
= INTERP_MODE_NOPERSPECTIVE
;
1571 case SpvDecorationFlat
:
1572 var_data
->interpolation
= INTERP_MODE_FLAT
;
1574 case SpvDecorationExplicitInterpAMD
:
1575 var_data
->interpolation
= INTERP_MODE_EXPLICIT
;
1577 case SpvDecorationCentroid
:
1578 var_data
->centroid
= true;
1580 case SpvDecorationSample
:
1581 var_data
->sample
= true;
1583 case SpvDecorationInvariant
:
1584 var_data
->invariant
= true;
1586 case SpvDecorationConstant
:
1587 var_data
->read_only
= true;
1589 case SpvDecorationNonReadable
:
1590 var_data
->access
|= ACCESS_NON_READABLE
;
1592 case SpvDecorationNonWritable
:
1593 var_data
->read_only
= true;
1594 var_data
->access
|= ACCESS_NON_WRITEABLE
;
1596 case SpvDecorationRestrict
:
1597 var_data
->access
|= ACCESS_RESTRICT
;
1599 case SpvDecorationAliased
:
1600 var_data
->access
&= ~ACCESS_RESTRICT
;
1602 case SpvDecorationVolatile
:
1603 var_data
->access
|= ACCESS_VOLATILE
;
1605 case SpvDecorationCoherent
:
1606 var_data
->access
|= ACCESS_COHERENT
;
1608 case SpvDecorationComponent
:
1609 var_data
->location_frac
= dec
->operands
[0];
1611 case SpvDecorationIndex
:
1612 var_data
->index
= dec
->operands
[0];
1614 case SpvDecorationBuiltIn
: {
1615 SpvBuiltIn builtin
= dec
->operands
[0];
1617 nir_variable_mode mode
= var_data
->mode
;
1618 vtn_get_builtin_location(b
, builtin
, &var_data
->location
, &mode
);
1619 var_data
->mode
= mode
;
1622 case SpvBuiltInTessLevelOuter
:
1623 case SpvBuiltInTessLevelInner
:
1624 case SpvBuiltInClipDistance
:
1625 case SpvBuiltInCullDistance
:
1626 var_data
->compact
= true;
1633 case SpvDecorationSpecId
:
1634 case SpvDecorationRowMajor
:
1635 case SpvDecorationColMajor
:
1636 case SpvDecorationMatrixStride
:
1637 case SpvDecorationUniform
:
1638 case SpvDecorationUniformId
:
1639 case SpvDecorationLinkageAttributes
:
1640 break; /* Do nothing with these here */
1642 case SpvDecorationPatch
:
1643 var_data
->patch
= true;
1646 case SpvDecorationLocation
:
1647 vtn_fail("Handled above");
1649 case SpvDecorationBlock
:
1650 case SpvDecorationBufferBlock
:
1651 case SpvDecorationArrayStride
:
1652 case SpvDecorationGLSLShared
:
1653 case SpvDecorationGLSLPacked
:
1654 break; /* These can apply to a type but we don't care about them */
1656 case SpvDecorationBinding
:
1657 case SpvDecorationDescriptorSet
:
1658 case SpvDecorationNoContraction
:
1659 case SpvDecorationInputAttachmentIndex
:
1660 vtn_warn("Decoration not allowed for variable or structure member: %s",
1661 spirv_decoration_to_string(dec
->decoration
));
1664 case SpvDecorationXfbBuffer
:
1665 var_data
->explicit_xfb_buffer
= true;
1666 var_data
->xfb
.buffer
= dec
->operands
[0];
1667 var_data
->always_active_io
= true;
1669 case SpvDecorationXfbStride
:
1670 var_data
->explicit_xfb_stride
= true;
1671 var_data
->xfb
.stride
= dec
->operands
[0];
1673 case SpvDecorationOffset
:
1674 var_data
->explicit_offset
= true;
1675 var_data
->offset
= dec
->operands
[0];
1678 case SpvDecorationStream
:
1679 var_data
->stream
= dec
->operands
[0];
1682 case SpvDecorationCPacked
:
1683 case SpvDecorationSaturatedConversion
:
1684 case SpvDecorationFuncParamAttr
:
1685 case SpvDecorationFPRoundingMode
:
1686 case SpvDecorationFPFastMathMode
:
1687 case SpvDecorationAlignment
:
1688 if (b
->shader
->info
.stage
!= MESA_SHADER_KERNEL
) {
1689 vtn_warn("Decoration only allowed for CL-style kernels: %s",
1690 spirv_decoration_to_string(dec
->decoration
));
1694 case SpvDecorationUserSemantic
:
1695 case SpvDecorationUserTypeGOOGLE
:
1696 /* User semantic decorations can safely be ignored by the driver. */
1699 case SpvDecorationRestrictPointerEXT
:
1700 case SpvDecorationAliasedPointerEXT
:
1701 /* TODO: We should actually plumb alias information through NIR. */
1705 vtn_fail_with_decoration("Unhandled decoration", dec
->decoration
);
1710 var_is_patch_cb(struct vtn_builder
*b
, struct vtn_value
*val
, int member
,
1711 const struct vtn_decoration
*dec
, void *out_is_patch
)
1713 if (dec
->decoration
== SpvDecorationPatch
) {
1714 *((bool *) out_is_patch
) = true;
1719 var_decoration_cb(struct vtn_builder
*b
, struct vtn_value
*val
, int member
,
1720 const struct vtn_decoration
*dec
, void *void_var
)
1722 struct vtn_variable
*vtn_var
= void_var
;
1724 /* Handle decorations that apply to a vtn_variable as a whole */
1725 switch (dec
->decoration
) {
1726 case SpvDecorationBinding
:
1727 vtn_var
->binding
= dec
->operands
[0];
1728 vtn_var
->explicit_binding
= true;
1730 case SpvDecorationDescriptorSet
:
1731 vtn_var
->descriptor_set
= dec
->operands
[0];
1733 case SpvDecorationInputAttachmentIndex
:
1734 vtn_var
->input_attachment_index
= dec
->operands
[0];
1736 case SpvDecorationPatch
:
1737 vtn_var
->patch
= true;
1739 case SpvDecorationOffset
:
1740 vtn_var
->offset
= dec
->operands
[0];
1742 case SpvDecorationNonWritable
:
1743 vtn_var
->access
|= ACCESS_NON_WRITEABLE
;
1745 case SpvDecorationNonReadable
:
1746 vtn_var
->access
|= ACCESS_NON_READABLE
;
1748 case SpvDecorationVolatile
:
1749 vtn_var
->access
|= ACCESS_VOLATILE
;
1751 case SpvDecorationCoherent
:
1752 vtn_var
->access
|= ACCESS_COHERENT
;
1754 case SpvDecorationCounterBuffer
:
1755 /* Counter buffer decorations can safely be ignored by the driver. */
1761 if (val
->value_type
== vtn_value_type_pointer
) {
1762 assert(val
->pointer
->var
== void_var
);
1763 assert(member
== -1);
1765 assert(val
->value_type
== vtn_value_type_type
);
1768 /* Location is odd. If applied to a split structure, we have to walk the
1769 * whole thing and accumulate the location. It's easier to handle as a
1772 if (dec
->decoration
== SpvDecorationLocation
) {
1773 unsigned location
= dec
->operands
[0];
1774 if (b
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
&&
1775 vtn_var
->mode
== vtn_variable_mode_output
) {
1776 location
+= FRAG_RESULT_DATA0
;
1777 } else if (b
->shader
->info
.stage
== MESA_SHADER_VERTEX
&&
1778 vtn_var
->mode
== vtn_variable_mode_input
) {
1779 location
+= VERT_ATTRIB_GENERIC0
;
1780 } else if (vtn_var
->mode
== vtn_variable_mode_input
||
1781 vtn_var
->mode
== vtn_variable_mode_output
) {
1782 location
+= vtn_var
->patch
? VARYING_SLOT_PATCH0
: VARYING_SLOT_VAR0
;
1783 } else if (vtn_var
->mode
!= vtn_variable_mode_uniform
) {
1784 vtn_warn("Location must be on input, output, uniform, sampler or "
1789 if (vtn_var
->var
->num_members
== 0) {
1790 /* This handles the member and lone variable cases */
1791 vtn_var
->var
->data
.location
= location
;
1793 /* This handles the structure member case */
1794 assert(vtn_var
->var
->members
);
1797 vtn_var
->base_location
= location
;
1799 vtn_var
->var
->members
[member
].location
= location
;
1805 if (vtn_var
->var
->num_members
== 0) {
1806 /* We call this function on types as well as variables and not all
1807 * struct types get split so we can end up having stray member
1808 * decorations; just ignore them.
1811 apply_var_decoration(b
, &vtn_var
->var
->data
, dec
);
1812 } else if (member
>= 0) {
1813 /* Member decorations must come from a type */
1814 assert(val
->value_type
== vtn_value_type_type
);
1815 apply_var_decoration(b
, &vtn_var
->var
->members
[member
], dec
);
1818 glsl_get_length(glsl_without_array(vtn_var
->type
->type
));
1819 for (unsigned i
= 0; i
< length
; i
++)
1820 apply_var_decoration(b
, &vtn_var
->var
->members
[i
], dec
);
1823 /* A few variables, those with external storage, have no actual
1824 * nir_variables associated with them. Fortunately, all decorations
1825 * we care about for those variables are on the type only.
1827 vtn_assert(vtn_var
->mode
== vtn_variable_mode_ubo
||
1828 vtn_var
->mode
== vtn_variable_mode_ssbo
||
1829 vtn_var
->mode
== vtn_variable_mode_push_constant
);
1834 enum vtn_variable_mode
1835 vtn_storage_class_to_mode(struct vtn_builder
*b
,
1836 SpvStorageClass
class,
1837 struct vtn_type
*interface_type
,
1838 nir_variable_mode
*nir_mode_out
)
1840 enum vtn_variable_mode mode
;
1841 nir_variable_mode nir_mode
;
1843 case SpvStorageClassUniform
:
1844 /* Assume it's an UBO if we lack the interface_type. */
1845 if (!interface_type
|| interface_type
->block
) {
1846 mode
= vtn_variable_mode_ubo
;
1847 nir_mode
= nir_var_mem_ubo
;
1848 } else if (interface_type
->buffer_block
) {
1849 mode
= vtn_variable_mode_ssbo
;
1850 nir_mode
= nir_var_mem_ssbo
;
1852 /* Default-block uniforms, coming from gl_spirv */
1853 mode
= vtn_variable_mode_uniform
;
1854 nir_mode
= nir_var_uniform
;
1857 case SpvStorageClassStorageBuffer
:
1858 mode
= vtn_variable_mode_ssbo
;
1859 nir_mode
= nir_var_mem_ssbo
;
1861 case SpvStorageClassPhysicalStorageBuffer
:
1862 mode
= vtn_variable_mode_phys_ssbo
;
1863 nir_mode
= nir_var_mem_global
;
1865 case SpvStorageClassUniformConstant
:
1866 if (b
->shader
->info
.stage
== MESA_SHADER_KERNEL
) {
1867 mode
= vtn_variable_mode_constant
;
1868 nir_mode
= nir_var_mem_constant
;
1870 mode
= vtn_variable_mode_uniform
;
1871 nir_mode
= nir_var_uniform
;
1874 case SpvStorageClassPushConstant
:
1875 mode
= vtn_variable_mode_push_constant
;
1876 nir_mode
= nir_var_uniform
;
1878 case SpvStorageClassInput
:
1879 mode
= vtn_variable_mode_input
;
1880 nir_mode
= nir_var_shader_in
;
1882 case SpvStorageClassOutput
:
1883 mode
= vtn_variable_mode_output
;
1884 nir_mode
= nir_var_shader_out
;
1886 case SpvStorageClassPrivate
:
1887 mode
= vtn_variable_mode_private
;
1888 nir_mode
= nir_var_shader_temp
;
1890 case SpvStorageClassFunction
:
1891 mode
= vtn_variable_mode_function
;
1892 nir_mode
= nir_var_function_temp
;
1894 case SpvStorageClassWorkgroup
:
1895 mode
= vtn_variable_mode_workgroup
;
1896 nir_mode
= nir_var_mem_shared
;
1898 case SpvStorageClassAtomicCounter
:
1899 mode
= vtn_variable_mode_atomic_counter
;
1900 nir_mode
= nir_var_uniform
;
1902 case SpvStorageClassCrossWorkgroup
:
1903 mode
= vtn_variable_mode_cross_workgroup
;
1904 nir_mode
= nir_var_mem_global
;
1906 case SpvStorageClassImage
:
1907 mode
= vtn_variable_mode_image
;
1908 nir_mode
= nir_var_mem_ubo
;
1910 case SpvStorageClassGeneric
:
1912 vtn_fail("Unhandled variable storage class: %s (%u)",
1913 spirv_storageclass_to_string(class), class);
1917 *nir_mode_out
= nir_mode
;
1923 vtn_mode_to_address_format(struct vtn_builder
*b
, enum vtn_variable_mode mode
)
1926 case vtn_variable_mode_ubo
:
1927 return b
->options
->ubo_addr_format
;
1929 case vtn_variable_mode_ssbo
:
1930 return b
->options
->ssbo_addr_format
;
1932 case vtn_variable_mode_phys_ssbo
:
1933 return b
->options
->phys_ssbo_addr_format
;
1935 case vtn_variable_mode_push_constant
:
1936 return b
->options
->push_const_addr_format
;
1938 case vtn_variable_mode_workgroup
:
1939 return b
->options
->shared_addr_format
;
1941 case vtn_variable_mode_cross_workgroup
:
1942 return b
->options
->global_addr_format
;
1944 case vtn_variable_mode_constant
:
1945 return b
->options
->constant_addr_format
;
1947 case vtn_variable_mode_function
:
1948 if (b
->physical_ptrs
)
1949 return b
->options
->temp_addr_format
;
1952 case vtn_variable_mode_private
:
1953 case vtn_variable_mode_uniform
:
1954 case vtn_variable_mode_atomic_counter
:
1955 case vtn_variable_mode_input
:
1956 case vtn_variable_mode_output
:
1957 case vtn_variable_mode_image
:
1958 return nir_address_format_logical
;
1961 unreachable("Invalid variable mode");
1965 vtn_pointer_to_ssa(struct vtn_builder
*b
, struct vtn_pointer
*ptr
)
1967 if (vtn_pointer_uses_ssa_offset(b
, ptr
)) {
1968 /* This pointer needs to have a pointer type with actual storage */
1969 vtn_assert(ptr
->ptr_type
);
1970 vtn_assert(ptr
->ptr_type
->type
);
1973 /* If we don't have an offset then we must be a pointer to the variable
1976 vtn_assert(!ptr
->offset
&& !ptr
->block_index
);
1978 struct vtn_access_chain chain
= {
1981 ptr
= vtn_ssa_offset_pointer_dereference(b
, ptr
, &chain
);
1984 vtn_assert(ptr
->offset
);
1985 if (ptr
->block_index
) {
1986 vtn_assert(ptr
->mode
== vtn_variable_mode_ubo
||
1987 ptr
->mode
== vtn_variable_mode_ssbo
);
1988 return nir_vec2(&b
->nb
, ptr
->block_index
, ptr
->offset
);
1990 vtn_assert(ptr
->mode
== vtn_variable_mode_workgroup
);
1994 if (vtn_pointer_is_external_block(b
, ptr
) &&
1995 vtn_type_contains_block(b
, ptr
->type
) &&
1996 ptr
->mode
!= vtn_variable_mode_phys_ssbo
) {
1997 /* In this case, we're looking for a block index and not an actual
2000 * For PhysicalStorageBuffer pointers, we don't have a block index
2001 * at all because we get the pointer directly from the client. This
2002 * assumes that there will never be a SSBO binding variable using the
2003 * PhysicalStorageBuffer storage class. This assumption appears
2004 * to be correct according to the Vulkan spec because the table,
2005 * "Shader Resource and Storage Class Correspondence," the only the
2006 * Uniform storage class with BufferBlock or the StorageBuffer
2007 * storage class with Block can be used.
2009 if (!ptr
->block_index
) {
2010 /* If we don't have a block_index then we must be a pointer to the
2013 vtn_assert(!ptr
->deref
);
2015 struct vtn_access_chain chain
= {
2018 ptr
= vtn_nir_deref_pointer_dereference(b
, ptr
, &chain
);
2021 return ptr
->block_index
;
2023 return &vtn_pointer_to_deref(b
, ptr
)->dest
.ssa
;
2028 struct vtn_pointer
*
2029 vtn_pointer_from_ssa(struct vtn_builder
*b
, nir_ssa_def
*ssa
,
2030 struct vtn_type
*ptr_type
)
2032 vtn_assert(ptr_type
->base_type
== vtn_base_type_pointer
);
2034 struct vtn_pointer
*ptr
= rzalloc(b
, struct vtn_pointer
);
2035 struct vtn_type
*without_array
=
2036 vtn_type_without_array(ptr_type
->deref
);
2038 nir_variable_mode nir_mode
;
2039 ptr
->mode
= vtn_storage_class_to_mode(b
, ptr_type
->storage_class
,
2040 without_array
, &nir_mode
);
2041 ptr
->type
= ptr_type
->deref
;
2042 ptr
->ptr_type
= ptr_type
;
2044 if (vtn_pointer_uses_ssa_offset(b
, ptr
)) {
2045 /* This pointer type needs to have actual storage */
2046 vtn_assert(ptr_type
->type
);
2047 if (ptr
->mode
== vtn_variable_mode_ubo
||
2048 ptr
->mode
== vtn_variable_mode_ssbo
) {
2049 vtn_assert(ssa
->num_components
== 2);
2050 ptr
->block_index
= nir_channel(&b
->nb
, ssa
, 0);
2051 ptr
->offset
= nir_channel(&b
->nb
, ssa
, 1);
2053 vtn_assert(ssa
->num_components
== 1);
2054 ptr
->block_index
= NULL
;
2058 const struct glsl_type
*deref_type
=
2059 vtn_type_get_nir_type(b
, ptr_type
->deref
, ptr
->mode
);
2060 if (!vtn_pointer_is_external_block(b
, ptr
)) {
2061 ptr
->deref
= nir_build_deref_cast(&b
->nb
, ssa
, nir_mode
,
2062 deref_type
, ptr_type
->stride
);
2063 } else if (vtn_type_contains_block(b
, ptr
->type
) &&
2064 ptr
->mode
!= vtn_variable_mode_phys_ssbo
) {
2065 /* This is a pointer to somewhere in an array of blocks, not a
2066 * pointer to somewhere inside the block. Set the block index
2067 * instead of making a cast.
2069 ptr
->block_index
= ssa
;
2071 /* This is a pointer to something internal or a pointer inside a
2072 * block. It's just a regular cast.
2074 * For PhysicalStorageBuffer pointers, we don't have a block index
2075 * at all because we get the pointer directly from the client. This
2076 * assumes that there will never be a SSBO binding variable using the
2077 * PhysicalStorageBuffer storage class. This assumption appears
2078 * to be correct according to the Vulkan spec because the table,
2079 * "Shader Resource and Storage Class Correspondence," the only the
2080 * Uniform storage class with BufferBlock or the StorageBuffer
2081 * storage class with Block can be used.
2083 ptr
->deref
= nir_build_deref_cast(&b
->nb
, ssa
, nir_mode
,
2084 deref_type
, ptr_type
->stride
);
2085 ptr
->deref
->dest
.ssa
.num_components
=
2086 glsl_get_vector_elements(ptr_type
->type
);
2087 ptr
->deref
->dest
.ssa
.bit_size
= glsl_get_bit_size(ptr_type
->type
);
2095 is_per_vertex_inout(const struct vtn_variable
*var
, gl_shader_stage stage
)
2097 if (var
->patch
|| !glsl_type_is_array(var
->type
->type
))
2100 if (var
->mode
== vtn_variable_mode_input
) {
2101 return stage
== MESA_SHADER_TESS_CTRL
||
2102 stage
== MESA_SHADER_TESS_EVAL
||
2103 stage
== MESA_SHADER_GEOMETRY
;
2106 if (var
->mode
== vtn_variable_mode_output
)
2107 return stage
== MESA_SHADER_TESS_CTRL
;
2113 assign_missing_member_locations(struct vtn_variable
*var
)
2116 glsl_get_length(glsl_without_array(var
->type
->type
));
2117 int location
= var
->base_location
;
2119 for (unsigned i
= 0; i
< length
; i
++) {
2120 /* From the Vulkan spec:
2122 * “If the structure type is a Block but without a Location, then each
2123 * of its members must have a Location decoration.”
2126 if (var
->type
->block
) {
2127 assert(var
->base_location
!= -1 ||
2128 var
->var
->members
[i
].location
!= -1);
2131 /* From the Vulkan spec:
2133 * “Any member with its own Location decoration is assigned that
2134 * location. Each remaining member is assigned the location after the
2135 * immediately preceding member in declaration order.”
2137 if (var
->var
->members
[i
].location
!= -1)
2138 location
= var
->var
->members
[i
].location
;
2140 var
->var
->members
[i
].location
= location
;
2142 /* Below we use type instead of interface_type, because interface_type
2143 * is only available when it is a Block. This code also supports
2144 * input/outputs that are just structs
2146 const struct glsl_type
*member_type
=
2147 glsl_get_struct_field(glsl_without_array(var
->type
->type
), i
);
2150 glsl_count_attribute_slots(member_type
,
2151 false /* is_gl_vertex_input */);
2157 vtn_create_variable(struct vtn_builder
*b
, struct vtn_value
*val
,
2158 struct vtn_type
*ptr_type
, SpvStorageClass storage_class
,
2159 nir_constant
*const_initializer
, nir_variable
*var_initializer
)
2161 vtn_assert(ptr_type
->base_type
== vtn_base_type_pointer
);
2162 struct vtn_type
*type
= ptr_type
->deref
;
2164 struct vtn_type
*without_array
= vtn_type_without_array(ptr_type
->deref
);
2166 enum vtn_variable_mode mode
;
2167 nir_variable_mode nir_mode
;
2168 mode
= vtn_storage_class_to_mode(b
, storage_class
, without_array
, &nir_mode
);
2171 case vtn_variable_mode_ubo
:
2172 /* There's no other way to get vtn_variable_mode_ubo */
2173 vtn_assert(without_array
->block
);
2174 b
->shader
->info
.num_ubos
++;
2176 case vtn_variable_mode_ssbo
:
2177 if (storage_class
== SpvStorageClassStorageBuffer
&&
2178 !without_array
->block
) {
2179 if (b
->variable_pointers
) {
2180 vtn_fail("Variables in the StorageBuffer storage class must "
2181 "have a struct type with the Block decoration");
2183 /* If variable pointers are not present, it's still malformed
2184 * SPIR-V but we can parse it and do the right thing anyway.
2185 * Since some of the 8-bit storage tests have bugs in this are,
2186 * just make it a warning for now.
2188 vtn_warn("Variables in the StorageBuffer storage class must "
2189 "have a struct type with the Block decoration");
2192 b
->shader
->info
.num_ssbos
++;
2194 case vtn_variable_mode_uniform
:
2195 if (without_array
->base_type
== vtn_base_type_image
) {
2196 if (glsl_type_is_image(without_array
->glsl_image
))
2197 b
->shader
->info
.num_images
++;
2198 else if (glsl_type_is_sampler(without_array
->glsl_image
))
2199 b
->shader
->info
.num_textures
++;
2202 case vtn_variable_mode_push_constant
:
2203 b
->shader
->num_uniforms
= vtn_type_block_size(b
, type
);
2206 case vtn_variable_mode_image
:
2207 vtn_fail("Cannot create a variable with the Image storage class");
2210 case vtn_variable_mode_phys_ssbo
:
2211 vtn_fail("Cannot create a variable with the "
2212 "PhysicalStorageBuffer storage class");
2216 /* No tallying is needed */
2220 struct vtn_variable
*var
= rzalloc(b
, struct vtn_variable
);
2223 var
->base_location
= -1;
2225 val
->pointer
= rzalloc(b
, struct vtn_pointer
);
2226 val
->pointer
->mode
= var
->mode
;
2227 val
->pointer
->type
= var
->type
;
2228 val
->pointer
->ptr_type
= ptr_type
;
2229 val
->pointer
->var
= var
;
2230 val
->pointer
->access
= var
->type
->access
;
2232 switch (var
->mode
) {
2233 case vtn_variable_mode_function
:
2234 case vtn_variable_mode_private
:
2235 case vtn_variable_mode_uniform
:
2236 case vtn_variable_mode_atomic_counter
:
2237 case vtn_variable_mode_constant
:
2238 /* For these, we create the variable normally */
2239 var
->var
= rzalloc(b
->shader
, nir_variable
);
2240 var
->var
->name
= ralloc_strdup(var
->var
, val
->name
);
2241 var
->var
->type
= vtn_type_get_nir_type(b
, var
->type
, var
->mode
);
2242 var
->var
->data
.mode
= nir_mode
;
2243 var
->var
->data
.location
= -1;
2244 var
->var
->interface_type
= NULL
;
2247 case vtn_variable_mode_ubo
:
2248 case vtn_variable_mode_ssbo
:
2249 var
->var
= rzalloc(b
->shader
, nir_variable
);
2250 var
->var
->name
= ralloc_strdup(var
->var
, val
->name
);
2252 var
->var
->type
= vtn_type_get_nir_type(b
, var
->type
, var
->mode
);
2253 var
->var
->interface_type
= var
->var
->type
;
2255 var
->var
->data
.mode
= nir_mode
;
2256 var
->var
->data
.location
= -1;
2260 case vtn_variable_mode_workgroup
:
2261 /* Create the variable normally */
2262 var
->var
= rzalloc(b
->shader
, nir_variable
);
2263 var
->var
->name
= ralloc_strdup(var
->var
, val
->name
);
2264 var
->var
->type
= vtn_type_get_nir_type(b
, var
->type
, var
->mode
);
2265 var
->var
->data
.mode
= nir_var_mem_shared
;
2268 case vtn_variable_mode_input
:
2269 case vtn_variable_mode_output
: {
2270 /* In order to know whether or not we're a per-vertex inout, we need
2271 * the patch qualifier. This means walking the variable decorations
2272 * early before we actually create any variables. Not a big deal.
2274 * GLSLang really likes to place decorations in the most interior
2275 * thing it possibly can. In particular, if you have a struct, it
2276 * will place the patch decorations on the struct members. This
2277 * should be handled by the variable splitting below just fine.
2279 * If you have an array-of-struct, things get even more weird as it
2280 * will place the patch decorations on the struct even though it's
2281 * inside an array and some of the members being patch and others not
2282 * makes no sense whatsoever. Since the only sensible thing is for
2283 * it to be all or nothing, we'll call it patch if any of the members
2284 * are declared patch.
2287 vtn_foreach_decoration(b
, val
, var_is_patch_cb
, &var
->patch
);
2288 if (glsl_type_is_array(var
->type
->type
) &&
2289 glsl_type_is_struct_or_ifc(without_array
->type
)) {
2290 vtn_foreach_decoration(b
, vtn_value(b
, without_array
->id
,
2291 vtn_value_type_type
),
2292 var_is_patch_cb
, &var
->patch
);
2295 /* For inputs and outputs, we immediately split structures. This
2296 * is for a couple of reasons. For one, builtins may all come in
2297 * a struct and we really want those split out into separate
2298 * variables. For another, interpolation qualifiers can be
2299 * applied to members of the top-level struct ane we need to be
2300 * able to preserve that information.
2303 struct vtn_type
*per_vertex_type
= var
->type
;
2304 if (is_per_vertex_inout(var
, b
->shader
->info
.stage
)) {
2305 /* In Geometry shaders (and some tessellation), inputs come
2306 * in per-vertex arrays. However, some builtins come in
2307 * non-per-vertex, hence the need for the is_array check. In
2308 * any case, there are no non-builtin arrays allowed so this
2309 * check should be sufficient.
2311 per_vertex_type
= var
->type
->array_element
;
2314 var
->var
= rzalloc(b
->shader
, nir_variable
);
2315 var
->var
->name
= ralloc_strdup(var
->var
, val
->name
);
2316 var
->var
->type
= vtn_type_get_nir_type(b
, var
->type
, var
->mode
);
2317 var
->var
->data
.mode
= nir_mode
;
2318 var
->var
->data
.patch
= var
->patch
;
2320 /* Figure out the interface block type. */
2321 struct vtn_type
*iface_type
= per_vertex_type
;
2322 if (var
->mode
== vtn_variable_mode_output
&&
2323 (b
->shader
->info
.stage
== MESA_SHADER_VERTEX
||
2324 b
->shader
->info
.stage
== MESA_SHADER_TESS_EVAL
||
2325 b
->shader
->info
.stage
== MESA_SHADER_GEOMETRY
)) {
2326 /* For vertex data outputs, we can end up with arrays of blocks for
2327 * transform feedback where each array element corresponds to a
2328 * different XFB output buffer.
2330 while (iface_type
->base_type
== vtn_base_type_array
)
2331 iface_type
= iface_type
->array_element
;
2333 if (iface_type
->base_type
== vtn_base_type_struct
&& iface_type
->block
)
2334 var
->var
->interface_type
= vtn_type_get_nir_type(b
, iface_type
,
2337 if (per_vertex_type
->base_type
== vtn_base_type_struct
&&
2338 per_vertex_type
->block
) {
2339 /* It's a struct. Set it up as per-member. */
2340 var
->var
->num_members
= glsl_get_length(per_vertex_type
->type
);
2341 var
->var
->members
= rzalloc_array(var
->var
, struct nir_variable_data
,
2342 var
->var
->num_members
);
2344 for (unsigned i
= 0; i
< var
->var
->num_members
; i
++) {
2345 var
->var
->members
[i
].mode
= nir_mode
;
2346 var
->var
->members
[i
].patch
= var
->patch
;
2347 var
->var
->members
[i
].location
= -1;
2351 /* For inputs and outputs, we need to grab locations and builtin
2352 * information from the per-vertex type.
2354 vtn_foreach_decoration(b
, vtn_value(b
, per_vertex_type
->id
,
2355 vtn_value_type_type
),
2356 var_decoration_cb
, var
);
2360 case vtn_variable_mode_push_constant
:
2361 case vtn_variable_mode_cross_workgroup
:
2362 /* These don't need actual variables. */
2365 case vtn_variable_mode_image
:
2366 case vtn_variable_mode_phys_ssbo
:
2367 unreachable("Should have been caught before");
2370 /* We can only have one type of initializer */
2371 assert(!(const_initializer
&& var_initializer
));
2372 if (const_initializer
) {
2373 var
->var
->constant_initializer
=
2374 nir_constant_clone(const_initializer
, var
->var
);
2376 if (var_initializer
)
2377 var
->var
->pointer_initializer
= var_initializer
;
2379 if (var
->mode
== vtn_variable_mode_uniform
||
2380 var
->mode
== vtn_variable_mode_ssbo
) {
2381 /* SSBOs and images are assumed to not alias in the Simple, GLSL and Vulkan memory models */
2382 var
->var
->data
.access
|= b
->mem_model
!= SpvMemoryModelOpenCL
? ACCESS_RESTRICT
: 0;
2385 vtn_foreach_decoration(b
, val
, var_decoration_cb
, var
);
2386 vtn_foreach_decoration(b
, val
, ptr_decoration_cb
, val
->pointer
);
2388 /* Propagate access flags from the OpVariable decorations. */
2389 val
->pointer
->access
|= var
->access
;
2391 if ((var
->mode
== vtn_variable_mode_input
||
2392 var
->mode
== vtn_variable_mode_output
) &&
2393 var
->var
->members
) {
2394 assign_missing_member_locations(var
);
2397 if (var
->mode
== vtn_variable_mode_uniform
||
2398 var
->mode
== vtn_variable_mode_ubo
||
2399 var
->mode
== vtn_variable_mode_ssbo
||
2400 var
->mode
== vtn_variable_mode_atomic_counter
) {
2401 /* XXX: We still need the binding information in the nir_variable
2402 * for these. We should fix that.
2404 var
->var
->data
.binding
= var
->binding
;
2405 var
->var
->data
.explicit_binding
= var
->explicit_binding
;
2406 var
->var
->data
.descriptor_set
= var
->descriptor_set
;
2407 var
->var
->data
.index
= var
->input_attachment_index
;
2408 var
->var
->data
.offset
= var
->offset
;
2410 if (glsl_type_is_image(glsl_without_array(var
->var
->type
)))
2411 var
->var
->data
.image
.format
= without_array
->image_format
;
2414 if (var
->mode
== vtn_variable_mode_function
) {
2415 vtn_assert(var
->var
!= NULL
&& var
->var
->members
== NULL
);
2416 nir_function_impl_add_variable(b
->nb
.impl
, var
->var
);
2417 } else if (var
->var
) {
2418 nir_shader_add_variable(b
->shader
, var
->var
);
2420 vtn_assert(vtn_pointer_is_external_block(b
, val
->pointer
));
2425 vtn_assert_types_equal(struct vtn_builder
*b
, SpvOp opcode
,
2426 struct vtn_type
*dst_type
,
2427 struct vtn_type
*src_type
)
2429 if (dst_type
->id
== src_type
->id
)
2432 if (vtn_types_compatible(b
, dst_type
, src_type
)) {
2433 /* Early versions of GLSLang would re-emit types unnecessarily and you
2434 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have
2435 * mismatched source and destination types.
2437 * https://github.com/KhronosGroup/glslang/issues/304
2438 * https://github.com/KhronosGroup/glslang/issues/307
2439 * https://bugs.freedesktop.org/show_bug.cgi?id=104338
2440 * https://bugs.freedesktop.org/show_bug.cgi?id=104424
2442 vtn_warn("Source and destination types of %s do not have the same "
2443 "ID (but are compatible): %u vs %u",
2444 spirv_op_to_string(opcode
), dst_type
->id
, src_type
->id
);
2448 vtn_fail("Source and destination types of %s do not match: %s vs. %s",
2449 spirv_op_to_string(opcode
),
2450 glsl_get_type_name(dst_type
->type
),
2451 glsl_get_type_name(src_type
->type
));
2454 static nir_ssa_def
*
2455 nir_shrink_zero_pad_vec(nir_builder
*b
, nir_ssa_def
*val
,
2456 unsigned num_components
)
2458 if (val
->num_components
== num_components
)
2461 nir_ssa_def
*comps
[NIR_MAX_VEC_COMPONENTS
];
2462 for (unsigned i
= 0; i
< num_components
; i
++) {
2463 if (i
< val
->num_components
)
2464 comps
[i
] = nir_channel(b
, val
, i
);
2466 comps
[i
] = nir_imm_intN_t(b
, 0, val
->bit_size
);
2468 return nir_vec(b
, comps
, num_components
);
2471 static nir_ssa_def
*
2472 nir_sloppy_bitcast(nir_builder
*b
, nir_ssa_def
*val
,
2473 const struct glsl_type
*type
)
2475 const unsigned num_components
= glsl_get_vector_elements(type
);
2476 const unsigned bit_size
= glsl_get_bit_size(type
);
2478 /* First, zero-pad to ensure that the value is big enough that when we
2479 * bit-cast it, we don't loose anything.
2481 if (val
->bit_size
< bit_size
) {
2482 const unsigned src_num_components_needed
=
2483 vtn_align_u32(val
->num_components
, bit_size
/ val
->bit_size
);
2484 val
= nir_shrink_zero_pad_vec(b
, val
, src_num_components_needed
);
2487 val
= nir_bitcast_vector(b
, val
, bit_size
);
2489 return nir_shrink_zero_pad_vec(b
, val
, num_components
);
2493 vtn_get_mem_operands(struct vtn_builder
*b
, const uint32_t *w
, unsigned count
,
2494 unsigned *idx
, SpvMemoryAccessMask
*access
, unsigned *alignment
,
2495 SpvScope
*dest_scope
, SpvScope
*src_scope
)
2502 *access
= w
[(*idx
)++];
2503 if (*access
& SpvMemoryAccessAlignedMask
) {
2504 vtn_assert(*idx
< count
);
2505 *alignment
= w
[(*idx
)++];
2508 if (*access
& SpvMemoryAccessMakePointerAvailableMask
) {
2509 vtn_assert(*idx
< count
);
2510 vtn_assert(dest_scope
);
2511 *dest_scope
= vtn_constant_uint(b
, w
[(*idx
)++]);
2514 if (*access
& SpvMemoryAccessMakePointerVisibleMask
) {
2515 vtn_assert(*idx
< count
);
2516 vtn_assert(src_scope
);
2517 *src_scope
= vtn_constant_uint(b
, w
[(*idx
)++]);
2523 static enum gl_access_qualifier
2524 spv_access_to_gl_access(SpvMemoryAccessMask access
)
2526 if (access
& SpvMemoryAccessVolatileMask
)
2527 return ACCESS_VOLATILE
;
2533 SpvMemorySemanticsMask
2534 vtn_mode_to_memory_semantics(enum vtn_variable_mode mode
)
2537 case vtn_variable_mode_ssbo
:
2538 case vtn_variable_mode_phys_ssbo
:
2539 return SpvMemorySemanticsUniformMemoryMask
;
2540 case vtn_variable_mode_workgroup
:
2541 return SpvMemorySemanticsWorkgroupMemoryMask
;
2542 case vtn_variable_mode_cross_workgroup
:
2543 return SpvMemorySemanticsCrossWorkgroupMemoryMask
;
2544 case vtn_variable_mode_atomic_counter
:
2545 return SpvMemorySemanticsAtomicCounterMemoryMask
;
2546 case vtn_variable_mode_image
:
2547 return SpvMemorySemanticsImageMemoryMask
;
2548 case vtn_variable_mode_output
:
2549 return SpvMemorySemanticsOutputMemoryMask
;
2551 return SpvMemorySemanticsMaskNone
;
2556 vtn_emit_make_visible_barrier(struct vtn_builder
*b
, SpvMemoryAccessMask access
,
2557 SpvScope scope
, enum vtn_variable_mode mode
)
2559 if (!(access
& SpvMemoryAccessMakePointerVisibleMask
))
2562 vtn_emit_memory_barrier(b
, scope
, SpvMemorySemanticsMakeVisibleMask
|
2563 SpvMemorySemanticsAcquireMask
|
2564 vtn_mode_to_memory_semantics(mode
));
2568 vtn_emit_make_available_barrier(struct vtn_builder
*b
, SpvMemoryAccessMask access
,
2569 SpvScope scope
, enum vtn_variable_mode mode
)
2571 if (!(access
& SpvMemoryAccessMakePointerAvailableMask
))
2574 vtn_emit_memory_barrier(b
, scope
, SpvMemorySemanticsMakeAvailableMask
|
2575 SpvMemorySemanticsReleaseMask
|
2576 vtn_mode_to_memory_semantics(mode
));
2580 ptr_nonuniform_workaround_cb(struct vtn_builder
*b
, struct vtn_value
*val
,
2581 int member
, const struct vtn_decoration
*dec
, void *void_ptr
)
2583 enum gl_access_qualifier
*access
= void_ptr
;
2585 switch (dec
->decoration
) {
2586 case SpvDecorationNonUniformEXT
:
2587 *access
|= ACCESS_NON_UNIFORM
;
2596 vtn_handle_variables(struct vtn_builder
*b
, SpvOp opcode
,
2597 const uint32_t *w
, unsigned count
)
2601 struct vtn_value
*val
= vtn_push_value(b
, w
[2], vtn_value_type_undef
);
2602 val
->type
= vtn_get_type(b
, w
[1]);
2606 case SpvOpVariable
: {
2607 struct vtn_type
*ptr_type
= vtn_get_type(b
, w
[1]);
2609 struct vtn_value
*val
= vtn_push_value(b
, w
[2], vtn_value_type_pointer
);
2611 SpvStorageClass storage_class
= w
[3];
2612 nir_constant
*const_initializer
= NULL
;
2613 nir_variable
*var_initializer
= NULL
;
2615 struct vtn_value
*init
= vtn_untyped_value(b
, w
[4]);
2616 switch (init
->value_type
) {
2617 case vtn_value_type_constant
:
2618 const_initializer
= init
->constant
;
2620 case vtn_value_type_pointer
:
2621 var_initializer
= init
->pointer
->var
->var
;
2624 vtn_fail("SPIR-V variable initializer %u must be constant or pointer",
2629 vtn_create_variable(b
, val
, ptr_type
, storage_class
, const_initializer
, var_initializer
);
2634 case SpvOpConstantSampler
: {
2635 /* Synthesize a pointer-to-sampler type, create a variable of that type,
2636 * and give the variable a constant initializer with the sampler params */
2637 struct vtn_type
*sampler_type
= vtn_value(b
, w
[1], vtn_value_type_type
)->type
;
2638 struct vtn_value
*val
= vtn_push_value(b
, w
[2], vtn_value_type_pointer
);
2640 struct vtn_type
*ptr_type
= rzalloc(b
, struct vtn_type
);
2641 ptr_type
= rzalloc(b
, struct vtn_type
);
2642 ptr_type
->base_type
= vtn_base_type_pointer
;
2643 ptr_type
->deref
= sampler_type
;
2644 ptr_type
->storage_class
= SpvStorageClassUniform
;
2646 ptr_type
->type
= nir_address_format_to_glsl_type(
2647 vtn_mode_to_address_format(b
, vtn_variable_mode_function
));
2649 vtn_create_variable(b
, val
, ptr_type
, ptr_type
->storage_class
, NULL
, NULL
);
2651 nir_variable
*nir_var
= val
->pointer
->var
->var
;
2652 nir_var
->data
.sampler
.is_inline_sampler
= true;
2653 nir_var
->data
.sampler
.addressing_mode
= w
[3];
2654 nir_var
->data
.sampler
.normalized_coordinates
= w
[4];
2655 nir_var
->data
.sampler
.filter_mode
= w
[5];
2660 case SpvOpAccessChain
:
2661 case SpvOpPtrAccessChain
:
2662 case SpvOpInBoundsAccessChain
:
2663 case SpvOpInBoundsPtrAccessChain
: {
2664 struct vtn_access_chain
*chain
= vtn_access_chain_create(b
, count
- 4);
2665 enum gl_access_qualifier access
= 0;
2666 chain
->ptr_as_array
= (opcode
== SpvOpPtrAccessChain
|| opcode
== SpvOpInBoundsPtrAccessChain
);
2669 for (int i
= 4; i
< count
; i
++) {
2670 struct vtn_value
*link_val
= vtn_untyped_value(b
, w
[i
]);
2671 if (link_val
->value_type
== vtn_value_type_constant
) {
2672 chain
->link
[idx
].mode
= vtn_access_mode_literal
;
2673 chain
->link
[idx
].id
= vtn_constant_int(b
, w
[i
]);
2675 chain
->link
[idx
].mode
= vtn_access_mode_id
;
2676 chain
->link
[idx
].id
= w
[i
];
2679 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */
2680 vtn_foreach_decoration(b
, link_val
, ptr_nonuniform_workaround_cb
, &access
);
2685 struct vtn_type
*ptr_type
= vtn_get_type(b
, w
[1]);
2686 struct vtn_pointer
*base
=
2687 vtn_value(b
, w
[3], vtn_value_type_pointer
)->pointer
;
2689 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */
2690 access
|= base
->access
& ACCESS_NON_UNIFORM
;
2692 struct vtn_pointer
*ptr
= vtn_pointer_dereference(b
, base
, chain
);
2693 ptr
->ptr_type
= ptr_type
;
2694 ptr
->access
|= access
;
2695 vtn_push_pointer(b
, w
[2], ptr
);
2699 case SpvOpCopyMemory
: {
2700 struct vtn_value
*dest_val
= vtn_value(b
, w
[1], vtn_value_type_pointer
);
2701 struct vtn_value
*src_val
= vtn_value(b
, w
[2], vtn_value_type_pointer
);
2702 struct vtn_pointer
*dest
= dest_val
->pointer
;
2703 struct vtn_pointer
*src
= src_val
->pointer
;
2705 vtn_assert_types_equal(b
, opcode
, dest_val
->type
->deref
,
2706 src_val
->type
->deref
);
2708 unsigned idx
= 3, dest_alignment
, src_alignment
;
2709 SpvMemoryAccessMask dest_access
, src_access
;
2710 SpvScope dest_scope
, src_scope
;
2711 vtn_get_mem_operands(b
, w
, count
, &idx
, &dest_access
, &dest_alignment
,
2712 &dest_scope
, &src_scope
);
2713 if (!vtn_get_mem_operands(b
, w
, count
, &idx
, &src_access
, &src_alignment
,
2714 NULL
, &src_scope
)) {
2715 src_alignment
= dest_alignment
;
2716 src_access
= dest_access
;
2718 src
= vtn_align_pointer(b
, src
, src_alignment
);
2719 dest
= vtn_align_pointer(b
, dest
, dest_alignment
);
2721 vtn_emit_make_visible_barrier(b
, src_access
, src_scope
, src
->mode
);
2723 vtn_variable_copy(b
, dest
, src
,
2724 spv_access_to_gl_access(dest_access
),
2725 spv_access_to_gl_access(src_access
));
2727 vtn_emit_make_available_barrier(b
, dest_access
, dest_scope
, dest
->mode
);
2732 struct vtn_type
*res_type
= vtn_get_type(b
, w
[1]);
2733 struct vtn_value
*src_val
= vtn_value(b
, w
[3], vtn_value_type_pointer
);
2734 struct vtn_pointer
*src
= src_val
->pointer
;
2736 vtn_assert_types_equal(b
, opcode
, res_type
, src_val
->type
->deref
);
2738 unsigned idx
= 4, alignment
;
2739 SpvMemoryAccessMask access
;
2741 vtn_get_mem_operands(b
, w
, count
, &idx
, &access
, &alignment
, NULL
, &scope
);
2742 src
= vtn_align_pointer(b
, src
, alignment
);
2744 vtn_emit_make_visible_barrier(b
, access
, scope
, src
->mode
);
2746 vtn_push_ssa_value(b
, w
[2], vtn_variable_load(b
, src
, spv_access_to_gl_access(access
)));
2751 struct vtn_value
*dest_val
= vtn_value(b
, w
[1], vtn_value_type_pointer
);
2752 struct vtn_pointer
*dest
= dest_val
->pointer
;
2753 struct vtn_value
*src_val
= vtn_untyped_value(b
, w
[2]);
2755 /* OpStore requires us to actually have a storage type */
2756 vtn_fail_if(dest
->type
->type
== NULL
,
2757 "Invalid destination type for OpStore");
2759 if (glsl_get_base_type(dest
->type
->type
) == GLSL_TYPE_BOOL
&&
2760 glsl_get_base_type(src_val
->type
->type
) == GLSL_TYPE_UINT
) {
2761 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but
2762 * would then store them to a local variable as bool. Work around
2763 * the issue by doing an implicit conversion.
2765 * https://github.com/KhronosGroup/glslang/issues/170
2766 * https://bugs.freedesktop.org/show_bug.cgi?id=104424
2768 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type "
2769 "OpTypeBool. Doing an implicit conversion to work around "
2771 struct vtn_ssa_value
*bool_ssa
=
2772 vtn_create_ssa_value(b
, dest
->type
->type
);
2773 bool_ssa
->def
= nir_i2b(&b
->nb
, vtn_ssa_value(b
, w
[2])->def
);
2774 vtn_variable_store(b
, bool_ssa
, dest
, 0);
2778 vtn_assert_types_equal(b
, opcode
, dest_val
->type
->deref
, src_val
->type
);
2780 unsigned idx
= 3, alignment
;
2781 SpvMemoryAccessMask access
;
2783 vtn_get_mem_operands(b
, w
, count
, &idx
, &access
, &alignment
, &scope
, NULL
);
2784 dest
= vtn_align_pointer(b
, dest
, alignment
);
2786 struct vtn_ssa_value
*src
= vtn_ssa_value(b
, w
[2]);
2787 vtn_variable_store(b
, src
, dest
, spv_access_to_gl_access(access
));
2789 vtn_emit_make_available_barrier(b
, access
, scope
, dest
->mode
);
2793 case SpvOpArrayLength
: {
2794 struct vtn_pointer
*ptr
=
2795 vtn_value(b
, w
[3], vtn_value_type_pointer
)->pointer
;
2796 const uint32_t field
= w
[4];
2798 vtn_fail_if(ptr
->type
->base_type
!= vtn_base_type_struct
,
2799 "OpArrayLength must take a pointer to a structure type");
2800 vtn_fail_if(field
!= ptr
->type
->length
- 1 ||
2801 ptr
->type
->members
[field
]->base_type
!= vtn_base_type_array
,
2802 "OpArrayLength must reference the last memeber of the "
2803 "structure and that must be an array");
2805 const uint32_t offset
= ptr
->type
->offsets
[field
];
2806 const uint32_t stride
= ptr
->type
->members
[field
]->stride
;
2808 if (!ptr
->block_index
) {
2809 struct vtn_access_chain chain
= {
2812 ptr
= vtn_pointer_dereference(b
, ptr
, &chain
);
2813 vtn_assert(ptr
->block_index
);
2816 nir_intrinsic_instr
*instr
=
2817 nir_intrinsic_instr_create(b
->nb
.shader
,
2818 nir_intrinsic_get_buffer_size
);
2819 instr
->src
[0] = nir_src_for_ssa(ptr
->block_index
);
2820 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
, 1, 32, NULL
);
2821 nir_builder_instr_insert(&b
->nb
, &instr
->instr
);
2822 nir_ssa_def
*buf_size
= &instr
->dest
.ssa
;
2824 /* array_length = max(buffer_size - offset, 0) / stride */
2825 nir_ssa_def
*array_length
=
2830 nir_imm_int(&b
->nb
, offset
)),
2831 nir_imm_int(&b
->nb
, 0u)),
2832 nir_imm_int(&b
->nb
, stride
));
2834 vtn_push_nir_ssa(b
, w
[2], array_length
);
2838 case SpvOpConvertPtrToU
: {
2839 struct vtn_type
*u_type
= vtn_get_type(b
, w
[1]);
2840 struct vtn_type
*ptr_type
= vtn_get_value_type(b
, w
[3]);
2842 vtn_fail_if(ptr_type
->base_type
!= vtn_base_type_pointer
||
2843 ptr_type
->type
== NULL
,
2844 "OpConvertPtrToU can only be used on physical pointers");
2846 vtn_fail_if(u_type
->base_type
!= vtn_base_type_vector
&&
2847 u_type
->base_type
!= vtn_base_type_scalar
,
2848 "OpConvertPtrToU can only be used to cast to a vector or "
2851 /* The pointer will be converted to an SSA value automatically */
2852 nir_ssa_def
*ptr
= vtn_get_nir_ssa(b
, w
[3]);
2853 nir_ssa_def
*u
= nir_sloppy_bitcast(&b
->nb
, ptr
, u_type
->type
);
2854 vtn_push_nir_ssa(b
, w
[2], u
);
2858 case SpvOpConvertUToPtr
: {
2859 struct vtn_type
*ptr_type
= vtn_get_type(b
, w
[1]);
2860 struct vtn_type
*u_type
= vtn_get_value_type(b
, w
[3]);
2862 vtn_fail_if(ptr_type
->base_type
!= vtn_base_type_pointer
||
2863 ptr_type
->type
== NULL
,
2864 "OpConvertUToPtr can only be used on physical pointers");
2866 vtn_fail_if(u_type
->base_type
!= vtn_base_type_vector
&&
2867 u_type
->base_type
!= vtn_base_type_scalar
,
2868 "OpConvertUToPtr can only be used to cast from a vector or "
2871 nir_ssa_def
*u
= vtn_get_nir_ssa(b
, w
[3]);
2872 nir_ssa_def
*ptr
= nir_sloppy_bitcast(&b
->nb
, u
, ptr_type
->type
);
2873 vtn_push_pointer(b
, w
[2], vtn_pointer_from_ssa(b
, ptr
, ptr_type
));
2877 case SpvOpCopyMemorySized
:
2879 vtn_fail_with_opcode("Unhandled opcode", opcode
);