b39483516708ce3424963c93146b9bb01949e603
[mesa.git] / src / compiler / spirv / vtn_variables.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Jason Ekstrand (jason@jlekstrand.net)
25 *
26 */
27
28 #include "vtn_private.h"
29 #include "spirv_info.h"
30 #include "nir_deref.h"
31 #include <vulkan/vulkan_core.h>
32
33 static void
34 ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
35 const struct vtn_decoration *dec, void *void_ptr)
36 {
37 struct vtn_pointer *ptr = void_ptr;
38
39 switch (dec->decoration) {
40 case SpvDecorationNonUniformEXT:
41 ptr->access |= ACCESS_NON_UNIFORM;
42 break;
43
44 default:
45 break;
46 }
47 }
48
49 static struct vtn_pointer*
50 vtn_decorate_pointer(struct vtn_builder *b, struct vtn_value *val,
51 struct vtn_pointer *ptr)
52 {
53 struct vtn_pointer dummy = { .access = 0 };
54 vtn_foreach_decoration(b, val, ptr_decoration_cb, &dummy);
55
56 /* If we're adding access flags, make a copy of the pointer. We could
57 * probably just OR them in without doing so but this prevents us from
58 * leaking them any further than actually specified in the SPIR-V.
59 */
60 if (dummy.access & ~ptr->access) {
61 struct vtn_pointer *copy = ralloc(b, struct vtn_pointer);
62 *copy = *ptr;
63 copy->access |= dummy.access;
64 return copy;
65 }
66
67 return ptr;
68 }
69
70 struct vtn_value *
71 vtn_push_pointer(struct vtn_builder *b, uint32_t value_id,
72 struct vtn_pointer *ptr)
73 {
74 struct vtn_value *val = vtn_push_value(b, value_id, vtn_value_type_pointer);
75 val->pointer = vtn_decorate_pointer(b, val, ptr);
76 return val;
77 }
78
79 void
80 vtn_copy_value(struct vtn_builder *b, uint32_t src_value_id,
81 uint32_t dst_value_id)
82 {
83 struct vtn_value *src = vtn_untyped_value(b, src_value_id);
84 struct vtn_value *dst = vtn_untyped_value(b, dst_value_id);
85 struct vtn_value src_copy = *src;
86
87 vtn_fail_if(dst->value_type != vtn_value_type_invalid,
88 "SPIR-V id %u has already been written by another instruction",
89 dst_value_id);
90
91 vtn_fail_if(dst->type->id != src->type->id,
92 "Result Type must equal Operand type");
93
94 src_copy.name = dst->name;
95 src_copy.decoration = dst->decoration;
96 src_copy.type = dst->type;
97 *dst = src_copy;
98
99 if (dst->value_type == vtn_value_type_pointer)
100 dst->pointer = vtn_decorate_pointer(b, dst, dst->pointer);
101 }
102
103 static struct vtn_access_chain *
104 vtn_access_chain_create(struct vtn_builder *b, unsigned length)
105 {
106 struct vtn_access_chain *chain;
107
108 /* Subtract 1 from the length since there's already one built in */
109 size_t size = sizeof(*chain) +
110 (MAX2(length, 1) - 1) * sizeof(chain->link[0]);
111 chain = rzalloc_size(b, size);
112 chain->length = length;
113
114 return chain;
115 }
116
117 bool
118 vtn_mode_uses_ssa_offset(struct vtn_builder *b,
119 enum vtn_variable_mode mode)
120 {
121 return ((mode == vtn_variable_mode_ubo ||
122 mode == vtn_variable_mode_ssbo) &&
123 b->options->lower_ubo_ssbo_access_to_offsets) ||
124 mode == vtn_variable_mode_push_constant;
125 }
126
127 static bool
128 vtn_mode_is_cross_invocation(struct vtn_builder *b,
129 enum vtn_variable_mode mode)
130 {
131 return mode == vtn_variable_mode_ssbo ||
132 mode == vtn_variable_mode_ubo ||
133 mode == vtn_variable_mode_phys_ssbo ||
134 mode == vtn_variable_mode_push_constant ||
135 mode == vtn_variable_mode_workgroup ||
136 mode == vtn_variable_mode_cross_workgroup;
137 }
138
139 static bool
140 vtn_pointer_is_external_block(struct vtn_builder *b,
141 struct vtn_pointer *ptr)
142 {
143 return ptr->mode == vtn_variable_mode_ssbo ||
144 ptr->mode == vtn_variable_mode_ubo ||
145 ptr->mode == vtn_variable_mode_phys_ssbo ||
146 ptr->mode == vtn_variable_mode_push_constant;
147 }
148
149 static nir_ssa_def *
150 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
151 unsigned stride, unsigned bit_size)
152 {
153 vtn_assert(stride > 0);
154 if (link.mode == vtn_access_mode_literal) {
155 return nir_imm_intN_t(&b->nb, link.id * stride, bit_size);
156 } else {
157 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def;
158 if (ssa->bit_size != bit_size)
159 ssa = nir_i2i(&b->nb, ssa, bit_size);
160 return nir_imul_imm(&b->nb, ssa, stride);
161 }
162 }
163
164 static VkDescriptorType
165 vk_desc_type_for_mode(struct vtn_builder *b, enum vtn_variable_mode mode)
166 {
167 switch (mode) {
168 case vtn_variable_mode_ubo:
169 return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
170 case vtn_variable_mode_ssbo:
171 return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
172 default:
173 vtn_fail("Invalid mode for vulkan_resource_index");
174 }
175 }
176
177 static nir_ssa_def *
178 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var,
179 nir_ssa_def *desc_array_index)
180 {
181 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN);
182
183 if (!desc_array_index) {
184 vtn_assert(glsl_type_is_struct_or_ifc(var->type->type));
185 desc_array_index = nir_imm_int(&b->nb, 0);
186 }
187
188 nir_intrinsic_instr *instr =
189 nir_intrinsic_instr_create(b->nb.shader,
190 nir_intrinsic_vulkan_resource_index);
191 instr->src[0] = nir_src_for_ssa(desc_array_index);
192 nir_intrinsic_set_desc_set(instr, var->descriptor_set);
193 nir_intrinsic_set_binding(instr, var->binding);
194 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, var->mode));
195
196 vtn_fail_if(var->mode != vtn_variable_mode_ubo &&
197 var->mode != vtn_variable_mode_ssbo,
198 "Invalid mode for vulkan_resource_index");
199
200 nir_address_format addr_format = vtn_mode_to_address_format(b, var->mode);
201 const struct glsl_type *index_type =
202 b->options->lower_ubo_ssbo_access_to_offsets ?
203 glsl_uint_type() : nir_address_format_to_glsl_type(addr_format);
204
205 instr->num_components = glsl_get_vector_elements(index_type);
206 nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components,
207 glsl_get_bit_size(index_type), NULL);
208 nir_builder_instr_insert(&b->nb, &instr->instr);
209
210 return &instr->dest.ssa;
211 }
212
213 static nir_ssa_def *
214 vtn_resource_reindex(struct vtn_builder *b, enum vtn_variable_mode mode,
215 nir_ssa_def *base_index, nir_ssa_def *offset_index)
216 {
217 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN);
218
219 nir_intrinsic_instr *instr =
220 nir_intrinsic_instr_create(b->nb.shader,
221 nir_intrinsic_vulkan_resource_reindex);
222 instr->src[0] = nir_src_for_ssa(base_index);
223 instr->src[1] = nir_src_for_ssa(offset_index);
224 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, mode));
225
226 vtn_fail_if(mode != vtn_variable_mode_ubo && mode != vtn_variable_mode_ssbo,
227 "Invalid mode for vulkan_resource_reindex");
228
229 nir_address_format addr_format = vtn_mode_to_address_format(b, mode);
230 const struct glsl_type *index_type =
231 b->options->lower_ubo_ssbo_access_to_offsets ?
232 glsl_uint_type() : nir_address_format_to_glsl_type(addr_format);
233
234 instr->num_components = glsl_get_vector_elements(index_type);
235 nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components,
236 glsl_get_bit_size(index_type), NULL);
237 nir_builder_instr_insert(&b->nb, &instr->instr);
238
239 return &instr->dest.ssa;
240 }
241
242 static nir_ssa_def *
243 vtn_descriptor_load(struct vtn_builder *b, enum vtn_variable_mode mode,
244 nir_ssa_def *desc_index)
245 {
246 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN);
247
248 nir_intrinsic_instr *desc_load =
249 nir_intrinsic_instr_create(b->nb.shader,
250 nir_intrinsic_load_vulkan_descriptor);
251 desc_load->src[0] = nir_src_for_ssa(desc_index);
252 nir_intrinsic_set_desc_type(desc_load, vk_desc_type_for_mode(b, mode));
253
254 vtn_fail_if(mode != vtn_variable_mode_ubo && mode != vtn_variable_mode_ssbo,
255 "Invalid mode for load_vulkan_descriptor");
256
257 nir_address_format addr_format = vtn_mode_to_address_format(b, mode);
258 const struct glsl_type *ptr_type =
259 nir_address_format_to_glsl_type(addr_format);
260
261 desc_load->num_components = glsl_get_vector_elements(ptr_type);
262 nir_ssa_dest_init(&desc_load->instr, &desc_load->dest,
263 desc_load->num_components,
264 glsl_get_bit_size(ptr_type), NULL);
265 nir_builder_instr_insert(&b->nb, &desc_load->instr);
266
267 return &desc_load->dest.ssa;
268 }
269
270 /* Dereference the given base pointer by the access chain */
271 static struct vtn_pointer *
272 vtn_nir_deref_pointer_dereference(struct vtn_builder *b,
273 struct vtn_pointer *base,
274 struct vtn_access_chain *deref_chain)
275 {
276 struct vtn_type *type = base->type;
277 enum gl_access_qualifier access = base->access | deref_chain->access;
278 unsigned idx = 0;
279
280 nir_deref_instr *tail;
281 if (base->deref) {
282 tail = base->deref;
283 } else if (b->options->environment == NIR_SPIRV_VULKAN &&
284 vtn_pointer_is_external_block(b, base)) {
285 nir_ssa_def *block_index = base->block_index;
286
287 /* We dereferencing an external block pointer. Correctness of this
288 * operation relies on one particular line in the SPIR-V spec, section
289 * entitled "Validation Rules for Shader Capabilities":
290 *
291 * "Block and BufferBlock decorations cannot decorate a structure
292 * type that is nested at any level inside another structure type
293 * decorated with Block or BufferBlock."
294 *
295 * This means that we can detect the point where we cross over from
296 * descriptor indexing to buffer indexing by looking for the block
297 * decorated struct type. Anything before the block decorated struct
298 * type is a descriptor indexing operation and anything after the block
299 * decorated struct is a buffer offset operation.
300 */
301
302 /* Figure out the descriptor array index if any
303 *
304 * Some of the Vulkan CTS tests with hand-rolled SPIR-V have been known
305 * to forget the Block or BufferBlock decoration from time to time.
306 * It's more robust if we check for both !block_index and for the type
307 * to contain a block. This way there's a decent chance that arrays of
308 * UBOs/SSBOs will work correctly even if variable pointers are
309 * completley toast.
310 */
311 nir_ssa_def *desc_arr_idx = NULL;
312 if (!block_index || vtn_type_contains_block(b, type)) {
313 /* If our type contains a block, then we're still outside the block
314 * and we need to process enough levels of dereferences to get inside
315 * of it.
316 */
317 if (deref_chain->ptr_as_array) {
318 unsigned aoa_size = glsl_get_aoa_size(type->type);
319 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[idx],
320 MAX2(aoa_size, 1), 32);
321 idx++;
322 }
323
324 for (; idx < deref_chain->length; idx++) {
325 if (type->base_type != vtn_base_type_array) {
326 vtn_assert(type->base_type == vtn_base_type_struct);
327 break;
328 }
329
330 unsigned aoa_size = glsl_get_aoa_size(type->array_element->type);
331 nir_ssa_def *arr_offset =
332 vtn_access_link_as_ssa(b, deref_chain->link[idx],
333 MAX2(aoa_size, 1), 32);
334 if (desc_arr_idx)
335 desc_arr_idx = nir_iadd(&b->nb, desc_arr_idx, arr_offset);
336 else
337 desc_arr_idx = arr_offset;
338
339 type = type->array_element;
340 access |= type->access;
341 }
342 }
343
344 if (!block_index) {
345 vtn_assert(base->var && base->type);
346 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
347 } else if (desc_arr_idx) {
348 block_index = vtn_resource_reindex(b, base->mode,
349 block_index, desc_arr_idx);
350 }
351
352 if (idx == deref_chain->length) {
353 /* The entire deref was consumed in finding the block index. Return
354 * a pointer which just has a block index and a later access chain
355 * will dereference deeper.
356 */
357 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
358 ptr->mode = base->mode;
359 ptr->type = type;
360 ptr->block_index = block_index;
361 ptr->access = access;
362 return ptr;
363 }
364
365 /* If we got here, there's more access chain to handle and we have the
366 * final block index. Insert a descriptor load and cast to a deref to
367 * start the deref chain.
368 */
369 nir_ssa_def *desc = vtn_descriptor_load(b, base->mode, block_index);
370
371 assert(base->mode == vtn_variable_mode_ssbo ||
372 base->mode == vtn_variable_mode_ubo);
373 nir_variable_mode nir_mode =
374 base->mode == vtn_variable_mode_ssbo ? nir_var_mem_ssbo : nir_var_mem_ubo;
375
376 tail = nir_build_deref_cast(&b->nb, desc, nir_mode,
377 vtn_type_get_nir_type(b, type, base->mode),
378 base->ptr_type->stride);
379 } else {
380 assert(base->var && base->var->var);
381 tail = nir_build_deref_var(&b->nb, base->var->var);
382 if (base->ptr_type && base->ptr_type->type) {
383 tail->dest.ssa.num_components =
384 glsl_get_vector_elements(base->ptr_type->type);
385 tail->dest.ssa.bit_size = glsl_get_bit_size(base->ptr_type->type);
386 }
387 }
388
389 if (idx == 0 && deref_chain->ptr_as_array) {
390 /* We start with a deref cast to get the stride. Hopefully, we'll be
391 * able to delete that cast eventually.
392 */
393 tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->mode,
394 tail->type, base->ptr_type->stride);
395
396 nir_ssa_def *index = vtn_access_link_as_ssa(b, deref_chain->link[0], 1,
397 tail->dest.ssa.bit_size);
398 tail = nir_build_deref_ptr_as_array(&b->nb, tail, index);
399 idx++;
400 }
401
402 for (; idx < deref_chain->length; idx++) {
403 if (glsl_type_is_struct_or_ifc(type->type)) {
404 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
405 unsigned field = deref_chain->link[idx].id;
406 tail = nir_build_deref_struct(&b->nb, tail, field);
407 type = type->members[field];
408 } else {
409 nir_ssa_def *arr_index =
410 vtn_access_link_as_ssa(b, deref_chain->link[idx], 1,
411 tail->dest.ssa.bit_size);
412 tail = nir_build_deref_array(&b->nb, tail, arr_index);
413 type = type->array_element;
414 }
415
416 access |= type->access;
417 }
418
419 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
420 ptr->mode = base->mode;
421 ptr->type = type;
422 ptr->var = base->var;
423 ptr->deref = tail;
424 ptr->access = access;
425
426 return ptr;
427 }
428
429 static struct vtn_pointer *
430 vtn_ssa_offset_pointer_dereference(struct vtn_builder *b,
431 struct vtn_pointer *base,
432 struct vtn_access_chain *deref_chain)
433 {
434 nir_ssa_def *block_index = base->block_index;
435 nir_ssa_def *offset = base->offset;
436 struct vtn_type *type = base->type;
437 enum gl_access_qualifier access = base->access;
438
439 unsigned idx = 0;
440 if (base->mode == vtn_variable_mode_ubo ||
441 base->mode == vtn_variable_mode_ssbo) {
442 if (!block_index) {
443 vtn_assert(base->var && base->type);
444 nir_ssa_def *desc_arr_idx;
445 if (glsl_type_is_array(type->type)) {
446 if (deref_chain->length >= 1) {
447 desc_arr_idx =
448 vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32);
449 idx++;
450 /* This consumes a level of type */
451 type = type->array_element;
452 access |= type->access;
453 } else {
454 /* This is annoying. We've been asked for a pointer to the
455 * array of UBOs/SSBOs and not a specifc buffer. Return a
456 * pointer with a descriptor index of 0 and we'll have to do
457 * a reindex later to adjust it to the right thing.
458 */
459 desc_arr_idx = nir_imm_int(&b->nb, 0);
460 }
461 } else if (deref_chain->ptr_as_array) {
462 /* You can't have a zero-length OpPtrAccessChain */
463 vtn_assert(deref_chain->length >= 1);
464 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32);
465 } else {
466 /* We have a regular non-array SSBO. */
467 desc_arr_idx = NULL;
468 }
469 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
470 } else if (deref_chain->ptr_as_array &&
471 type->base_type == vtn_base_type_struct && type->block) {
472 /* We are doing an OpPtrAccessChain on a pointer to a struct that is
473 * decorated block. This is an interesting corner in the SPIR-V
474 * spec. One interpretation would be that they client is clearly
475 * trying to treat that block as if it's an implicit array of blocks
476 * repeated in the buffer. However, the SPIR-V spec for the
477 * OpPtrAccessChain says:
478 *
479 * "Base is treated as the address of the first element of an
480 * array, and the Element element’s address is computed to be the
481 * base for the Indexes, as per OpAccessChain."
482 *
483 * Taken literally, that would mean that your struct type is supposed
484 * to be treated as an array of such a struct and, since it's
485 * decorated block, that means an array of blocks which corresponds
486 * to an array descriptor. Therefore, we need to do a reindex
487 * operation to add the index from the first link in the access chain
488 * to the index we recieved.
489 *
490 * The downside to this interpretation (there always is one) is that
491 * this might be somewhat surprising behavior to apps if they expect
492 * the implicit array behavior described above.
493 */
494 vtn_assert(deref_chain->length >= 1);
495 nir_ssa_def *offset_index =
496 vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32);
497 idx++;
498
499 block_index = vtn_resource_reindex(b, base->mode,
500 block_index, offset_index);
501 }
502 }
503
504 if (!offset) {
505 if (base->mode == vtn_variable_mode_workgroup) {
506 /* SLM doesn't need nor have a block index */
507 vtn_assert(!block_index);
508
509 /* We need the variable for the base offset */
510 vtn_assert(base->var);
511
512 /* We need ptr_type for size and alignment */
513 vtn_assert(base->ptr_type);
514
515 /* Assign location on first use so that we don't end up bloating SLM
516 * address space for variables which are never statically used.
517 */
518 if (base->var->shared_location < 0) {
519 vtn_assert(base->ptr_type->length > 0 && base->ptr_type->align > 0);
520 b->shader->num_shared = vtn_align_u32(b->shader->num_shared,
521 base->ptr_type->align);
522 base->var->shared_location = b->shader->num_shared;
523 b->shader->num_shared += base->ptr_type->length;
524 }
525
526 offset = nir_imm_int(&b->nb, base->var->shared_location);
527 } else if (base->mode == vtn_variable_mode_push_constant) {
528 /* Push constants neither need nor have a block index */
529 vtn_assert(!block_index);
530
531 /* Start off with at the start of the push constant block. */
532 offset = nir_imm_int(&b->nb, 0);
533 } else {
534 /* The code above should have ensured a block_index when needed. */
535 vtn_assert(block_index);
536
537 /* Start off with at the start of the buffer. */
538 offset = nir_imm_int(&b->nb, 0);
539 }
540 }
541
542 if (deref_chain->ptr_as_array && idx == 0) {
543 /* We need ptr_type for the stride */
544 vtn_assert(base->ptr_type);
545
546 /* We need at least one element in the chain */
547 vtn_assert(deref_chain->length >= 1);
548
549 nir_ssa_def *elem_offset =
550 vtn_access_link_as_ssa(b, deref_chain->link[idx],
551 base->ptr_type->stride, offset->bit_size);
552 offset = nir_iadd(&b->nb, offset, elem_offset);
553 idx++;
554 }
555
556 for (; idx < deref_chain->length; idx++) {
557 switch (glsl_get_base_type(type->type)) {
558 case GLSL_TYPE_UINT:
559 case GLSL_TYPE_INT:
560 case GLSL_TYPE_UINT16:
561 case GLSL_TYPE_INT16:
562 case GLSL_TYPE_UINT8:
563 case GLSL_TYPE_INT8:
564 case GLSL_TYPE_UINT64:
565 case GLSL_TYPE_INT64:
566 case GLSL_TYPE_FLOAT:
567 case GLSL_TYPE_FLOAT16:
568 case GLSL_TYPE_DOUBLE:
569 case GLSL_TYPE_BOOL:
570 case GLSL_TYPE_ARRAY: {
571 nir_ssa_def *elem_offset =
572 vtn_access_link_as_ssa(b, deref_chain->link[idx],
573 type->stride, offset->bit_size);
574 offset = nir_iadd(&b->nb, offset, elem_offset);
575 type = type->array_element;
576 access |= type->access;
577 break;
578 }
579
580 case GLSL_TYPE_INTERFACE:
581 case GLSL_TYPE_STRUCT: {
582 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
583 unsigned member = deref_chain->link[idx].id;
584 offset = nir_iadd_imm(&b->nb, offset, type->offsets[member]);
585 type = type->members[member];
586 access |= type->access;
587 break;
588 }
589
590 default:
591 vtn_fail("Invalid type for deref");
592 }
593 }
594
595 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
596 ptr->mode = base->mode;
597 ptr->type = type;
598 ptr->block_index = block_index;
599 ptr->offset = offset;
600 ptr->access = access;
601
602 return ptr;
603 }
604
605 /* Dereference the given base pointer by the access chain */
606 static struct vtn_pointer *
607 vtn_pointer_dereference(struct vtn_builder *b,
608 struct vtn_pointer *base,
609 struct vtn_access_chain *deref_chain)
610 {
611 if (vtn_pointer_uses_ssa_offset(b, base)) {
612 return vtn_ssa_offset_pointer_dereference(b, base, deref_chain);
613 } else {
614 return vtn_nir_deref_pointer_dereference(b, base, deref_chain);
615 }
616 }
617
618 nir_deref_instr *
619 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr)
620 {
621 vtn_assert(!vtn_pointer_uses_ssa_offset(b, ptr));
622 if (!ptr->deref) {
623 struct vtn_access_chain chain = {
624 .length = 0,
625 };
626 ptr = vtn_nir_deref_pointer_dereference(b, ptr, &chain);
627 }
628
629 return ptr->deref;
630 }
631
632 static void
633 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref,
634 struct vtn_ssa_value *inout,
635 enum gl_access_qualifier access)
636 {
637 if (glsl_type_is_vector_or_scalar(deref->type)) {
638 if (load) {
639 inout->def = nir_load_deref_with_access(&b->nb, deref, access);
640 } else {
641 nir_store_deref_with_access(&b->nb, deref, inout->def, ~0, access);
642 }
643 } else if (glsl_type_is_array(deref->type) ||
644 glsl_type_is_matrix(deref->type)) {
645 unsigned elems = glsl_get_length(deref->type);
646 for (unsigned i = 0; i < elems; i++) {
647 nir_deref_instr *child =
648 nir_build_deref_array_imm(&b->nb, deref, i);
649 _vtn_local_load_store(b, load, child, inout->elems[i], access);
650 }
651 } else {
652 vtn_assert(glsl_type_is_struct_or_ifc(deref->type));
653 unsigned elems = glsl_get_length(deref->type);
654 for (unsigned i = 0; i < elems; i++) {
655 nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i);
656 _vtn_local_load_store(b, load, child, inout->elems[i], access);
657 }
658 }
659 }
660
661 nir_deref_instr *
662 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
663 {
664 struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer;
665 return vtn_pointer_to_deref(b, ptr);
666 }
667
668 /*
669 * Gets the NIR-level deref tail, which may have as a child an array deref
670 * selecting which component due to OpAccessChain supporting per-component
671 * indexing in SPIR-V.
672 */
673 static nir_deref_instr *
674 get_deref_tail(nir_deref_instr *deref)
675 {
676 if (deref->deref_type != nir_deref_type_array)
677 return deref;
678
679 nir_deref_instr *parent =
680 nir_instr_as_deref(deref->parent.ssa->parent_instr);
681
682 if (glsl_type_is_vector(parent->type))
683 return parent;
684 else
685 return deref;
686 }
687
688 struct vtn_ssa_value *
689 vtn_local_load(struct vtn_builder *b, nir_deref_instr *src,
690 enum gl_access_qualifier access)
691 {
692 nir_deref_instr *src_tail = get_deref_tail(src);
693 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
694 _vtn_local_load_store(b, true, src_tail, val, access);
695
696 if (src_tail != src) {
697 val->type = src->type;
698 val->def = nir_vector_extract(&b->nb, val->def, src->arr.index.ssa);
699 }
700
701 return val;
702 }
703
704 void
705 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
706 nir_deref_instr *dest, enum gl_access_qualifier access)
707 {
708 nir_deref_instr *dest_tail = get_deref_tail(dest);
709
710 if (dest_tail != dest) {
711 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
712 _vtn_local_load_store(b, true, dest_tail, val, access);
713
714 val->def = nir_vector_insert(&b->nb, val->def, src->def,
715 dest->arr.index.ssa);
716 _vtn_local_load_store(b, false, dest_tail, val, access);
717 } else {
718 _vtn_local_load_store(b, false, dest_tail, src, access);
719 }
720 }
721
722 nir_ssa_def *
723 vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr,
724 nir_ssa_def **index_out)
725 {
726 assert(vtn_pointer_uses_ssa_offset(b, ptr));
727 if (!ptr->offset) {
728 struct vtn_access_chain chain = {
729 .length = 0,
730 };
731 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
732 }
733 *index_out = ptr->block_index;
734 return ptr->offset;
735 }
736
737 /* Tries to compute the size of an interface block based on the strides and
738 * offsets that are provided to us in the SPIR-V source.
739 */
740 static unsigned
741 vtn_type_block_size(struct vtn_builder *b, struct vtn_type *type)
742 {
743 enum glsl_base_type base_type = glsl_get_base_type(type->type);
744 switch (base_type) {
745 case GLSL_TYPE_UINT:
746 case GLSL_TYPE_INT:
747 case GLSL_TYPE_UINT16:
748 case GLSL_TYPE_INT16:
749 case GLSL_TYPE_UINT8:
750 case GLSL_TYPE_INT8:
751 case GLSL_TYPE_UINT64:
752 case GLSL_TYPE_INT64:
753 case GLSL_TYPE_FLOAT:
754 case GLSL_TYPE_FLOAT16:
755 case GLSL_TYPE_BOOL:
756 case GLSL_TYPE_DOUBLE: {
757 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
758 glsl_get_matrix_columns(type->type);
759 if (cols > 1) {
760 vtn_assert(type->stride > 0);
761 return type->stride * cols;
762 } else {
763 unsigned type_size = glsl_get_bit_size(type->type) / 8;
764 return glsl_get_vector_elements(type->type) * type_size;
765 }
766 }
767
768 case GLSL_TYPE_STRUCT:
769 case GLSL_TYPE_INTERFACE: {
770 unsigned size = 0;
771 unsigned num_fields = glsl_get_length(type->type);
772 for (unsigned f = 0; f < num_fields; f++) {
773 unsigned field_end = type->offsets[f] +
774 vtn_type_block_size(b, type->members[f]);
775 size = MAX2(size, field_end);
776 }
777 return size;
778 }
779
780 case GLSL_TYPE_ARRAY:
781 vtn_assert(type->stride > 0);
782 vtn_assert(glsl_get_length(type->type) > 0);
783 return type->stride * glsl_get_length(type->type);
784
785 default:
786 vtn_fail("Invalid block type");
787 return 0;
788 }
789 }
790
791 static void
792 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
793 nir_ssa_def *index, nir_ssa_def *offset,
794 unsigned access_offset, unsigned access_size,
795 struct vtn_ssa_value **inout, const struct glsl_type *type,
796 enum gl_access_qualifier access)
797 {
798 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
799 instr->num_components = glsl_get_vector_elements(type);
800
801 /* Booleans usually shouldn't show up in external memory in SPIR-V.
802 * However, they do for certain older GLSLang versions and can for shared
803 * memory when we lower access chains internally.
804 */
805 const unsigned data_bit_size = glsl_type_is_boolean(type) ? 32 :
806 glsl_get_bit_size(type);
807
808 int src = 0;
809 if (!load) {
810 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
811 instr->src[src++] = nir_src_for_ssa((*inout)->def);
812 }
813
814 if (op == nir_intrinsic_load_push_constant) {
815 nir_intrinsic_set_base(instr, access_offset);
816 nir_intrinsic_set_range(instr, access_size);
817 }
818
819 if (op == nir_intrinsic_load_ubo ||
820 op == nir_intrinsic_load_ssbo ||
821 op == nir_intrinsic_store_ssbo) {
822 nir_intrinsic_set_access(instr, access);
823 }
824
825 /* With extensions like relaxed_block_layout, we really can't guarantee
826 * much more than scalar alignment.
827 */
828 if (op != nir_intrinsic_load_push_constant)
829 nir_intrinsic_set_align(instr, data_bit_size / 8, 0);
830
831 if (index)
832 instr->src[src++] = nir_src_for_ssa(index);
833
834 if (op == nir_intrinsic_load_push_constant) {
835 /* We need to subtract the offset from where the intrinsic will load the
836 * data. */
837 instr->src[src++] =
838 nir_src_for_ssa(nir_isub(&b->nb, offset,
839 nir_imm_int(&b->nb, access_offset)));
840 } else {
841 instr->src[src++] = nir_src_for_ssa(offset);
842 }
843
844 if (load) {
845 nir_ssa_dest_init(&instr->instr, &instr->dest,
846 instr->num_components, data_bit_size, NULL);
847 (*inout)->def = &instr->dest.ssa;
848 }
849
850 nir_builder_instr_insert(&b->nb, &instr->instr);
851
852 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
853 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
854 }
855
856 static void
857 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
858 nir_ssa_def *index, nir_ssa_def *offset,
859 unsigned access_offset, unsigned access_size,
860 struct vtn_type *type, enum gl_access_qualifier access,
861 struct vtn_ssa_value **inout)
862 {
863 enum glsl_base_type base_type = glsl_get_base_type(type->type);
864 switch (base_type) {
865 case GLSL_TYPE_UINT:
866 case GLSL_TYPE_INT:
867 case GLSL_TYPE_UINT16:
868 case GLSL_TYPE_INT16:
869 case GLSL_TYPE_UINT8:
870 case GLSL_TYPE_INT8:
871 case GLSL_TYPE_UINT64:
872 case GLSL_TYPE_INT64:
873 case GLSL_TYPE_FLOAT:
874 case GLSL_TYPE_FLOAT16:
875 case GLSL_TYPE_DOUBLE:
876 case GLSL_TYPE_BOOL:
877 /* This is where things get interesting. At this point, we've hit
878 * a vector, a scalar, or a matrix.
879 */
880 if (glsl_type_is_matrix(type->type)) {
881 /* Loading the whole matrix */
882 struct vtn_ssa_value *transpose;
883 unsigned num_ops, vec_width, col_stride;
884 if (type->row_major) {
885 num_ops = glsl_get_vector_elements(type->type);
886 vec_width = glsl_get_matrix_columns(type->type);
887 col_stride = type->array_element->stride;
888 if (load) {
889 const struct glsl_type *transpose_type =
890 glsl_matrix_type(base_type, vec_width, num_ops);
891 *inout = vtn_create_ssa_value(b, transpose_type);
892 } else {
893 transpose = vtn_ssa_transpose(b, *inout);
894 inout = &transpose;
895 }
896 } else {
897 num_ops = glsl_get_matrix_columns(type->type);
898 vec_width = glsl_get_vector_elements(type->type);
899 col_stride = type->stride;
900 }
901
902 for (unsigned i = 0; i < num_ops; i++) {
903 nir_ssa_def *elem_offset =
904 nir_iadd_imm(&b->nb, offset, i * col_stride);
905 _vtn_load_store_tail(b, op, load, index, elem_offset,
906 access_offset, access_size,
907 &(*inout)->elems[i],
908 glsl_vector_type(base_type, vec_width),
909 type->access | access);
910 }
911
912 if (load && type->row_major)
913 *inout = vtn_ssa_transpose(b, *inout);
914 } else {
915 unsigned elems = glsl_get_vector_elements(type->type);
916 unsigned type_size = glsl_get_bit_size(type->type) / 8;
917 if (elems == 1 || type->stride == type_size) {
918 /* This is a tightly-packed normal scalar or vector load */
919 vtn_assert(glsl_type_is_vector_or_scalar(type->type));
920 _vtn_load_store_tail(b, op, load, index, offset,
921 access_offset, access_size,
922 inout, type->type,
923 type->access | access);
924 } else {
925 /* This is a strided load. We have to load N things separately.
926 * This is the single column of a row-major matrix case.
927 */
928 vtn_assert(type->stride > type_size);
929 vtn_assert(type->stride % type_size == 0);
930
931 nir_ssa_def *per_comp[4];
932 for (unsigned i = 0; i < elems; i++) {
933 nir_ssa_def *elem_offset =
934 nir_iadd_imm(&b->nb, offset, i * type->stride);
935 struct vtn_ssa_value *comp, temp_val;
936 if (!load) {
937 temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
938 temp_val.type = glsl_scalar_type(base_type);
939 }
940 comp = &temp_val;
941 _vtn_load_store_tail(b, op, load, index, elem_offset,
942 access_offset, access_size,
943 &comp, glsl_scalar_type(base_type),
944 type->access | access);
945 per_comp[i] = comp->def;
946 }
947
948 if (load) {
949 if (*inout == NULL)
950 *inout = vtn_create_ssa_value(b, type->type);
951 (*inout)->def = nir_vec(&b->nb, per_comp, elems);
952 }
953 }
954 }
955 return;
956
957 case GLSL_TYPE_ARRAY: {
958 unsigned elems = glsl_get_length(type->type);
959 for (unsigned i = 0; i < elems; i++) {
960 nir_ssa_def *elem_off =
961 nir_iadd_imm(&b->nb, offset, i * type->stride);
962 _vtn_block_load_store(b, op, load, index, elem_off,
963 access_offset, access_size,
964 type->array_element,
965 type->array_element->access | access,
966 &(*inout)->elems[i]);
967 }
968 return;
969 }
970
971 case GLSL_TYPE_INTERFACE:
972 case GLSL_TYPE_STRUCT: {
973 unsigned elems = glsl_get_length(type->type);
974 for (unsigned i = 0; i < elems; i++) {
975 nir_ssa_def *elem_off =
976 nir_iadd_imm(&b->nb, offset, type->offsets[i]);
977 _vtn_block_load_store(b, op, load, index, elem_off,
978 access_offset, access_size,
979 type->members[i],
980 type->members[i]->access | access,
981 &(*inout)->elems[i]);
982 }
983 return;
984 }
985
986 default:
987 vtn_fail("Invalid block member type");
988 }
989 }
990
991 static struct vtn_ssa_value *
992 vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src)
993 {
994 nir_intrinsic_op op;
995 unsigned access_offset = 0, access_size = 0;
996 switch (src->mode) {
997 case vtn_variable_mode_ubo:
998 op = nir_intrinsic_load_ubo;
999 break;
1000 case vtn_variable_mode_ssbo:
1001 op = nir_intrinsic_load_ssbo;
1002 break;
1003 case vtn_variable_mode_push_constant:
1004 op = nir_intrinsic_load_push_constant;
1005 access_size = b->shader->num_uniforms;
1006 break;
1007 case vtn_variable_mode_workgroup:
1008 op = nir_intrinsic_load_shared;
1009 break;
1010 default:
1011 vtn_fail("Invalid block variable mode");
1012 }
1013
1014 nir_ssa_def *offset, *index = NULL;
1015 offset = vtn_pointer_to_offset(b, src, &index);
1016
1017 struct vtn_ssa_value *value = vtn_create_ssa_value(b, src->type->type);
1018 _vtn_block_load_store(b, op, true, index, offset,
1019 access_offset, access_size,
1020 src->type, src->access, &value);
1021 return value;
1022 }
1023
1024 static void
1025 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
1026 struct vtn_pointer *dst)
1027 {
1028 nir_intrinsic_op op;
1029 switch (dst->mode) {
1030 case vtn_variable_mode_ssbo:
1031 op = nir_intrinsic_store_ssbo;
1032 break;
1033 case vtn_variable_mode_workgroup:
1034 op = nir_intrinsic_store_shared;
1035 break;
1036 default:
1037 vtn_fail("Invalid block variable mode");
1038 }
1039
1040 nir_ssa_def *offset, *index = NULL;
1041 offset = vtn_pointer_to_offset(b, dst, &index);
1042
1043 _vtn_block_load_store(b, op, false, index, offset,
1044 0, 0, dst->type, dst->access, &src);
1045 }
1046
1047 static void
1048 _vtn_variable_load_store(struct vtn_builder *b, bool load,
1049 struct vtn_pointer *ptr,
1050 enum gl_access_qualifier access,
1051 struct vtn_ssa_value **inout)
1052 {
1053 if (ptr->mode == vtn_variable_mode_uniform) {
1054 if (ptr->type->base_type == vtn_base_type_image ||
1055 ptr->type->base_type == vtn_base_type_sampler) {
1056 /* See also our handling of OpTypeSampler and OpTypeImage */
1057 vtn_assert(load);
1058 (*inout)->def = vtn_pointer_to_ssa(b, ptr);
1059 return;
1060 } else if (ptr->type->base_type == vtn_base_type_sampled_image) {
1061 /* See also our handling of OpTypeSampledImage */
1062 vtn_assert(load);
1063 struct vtn_sampled_image si = {
1064 .image = vtn_pointer_to_deref(b, ptr),
1065 .sampler = vtn_pointer_to_deref(b, ptr),
1066 };
1067 (*inout)->def = vtn_sampled_image_to_nir_ssa(b, si);
1068 return;
1069 }
1070 }
1071
1072 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type);
1073 switch (base_type) {
1074 case GLSL_TYPE_UINT:
1075 case GLSL_TYPE_INT:
1076 case GLSL_TYPE_UINT16:
1077 case GLSL_TYPE_INT16:
1078 case GLSL_TYPE_UINT8:
1079 case GLSL_TYPE_INT8:
1080 case GLSL_TYPE_UINT64:
1081 case GLSL_TYPE_INT64:
1082 case GLSL_TYPE_FLOAT:
1083 case GLSL_TYPE_FLOAT16:
1084 case GLSL_TYPE_BOOL:
1085 case GLSL_TYPE_DOUBLE:
1086 if (glsl_type_is_vector_or_scalar(ptr->type->type)) {
1087 /* We hit a vector or scalar; go ahead and emit the load[s] */
1088 nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr);
1089 if (vtn_mode_is_cross_invocation(b, ptr->mode)) {
1090 /* If it's cross-invocation, we call nir_load/store_deref
1091 * directly. The vtn_local_load/store helpers are too clever and
1092 * do magic to avoid array derefs of vectors. That magic is both
1093 * less efficient than the direct load/store and, in the case of
1094 * stores, is broken because it creates a race condition if two
1095 * threads are writing to different components of the same vector
1096 * due to the load+insert+store it uses to emulate the array
1097 * deref.
1098 */
1099 if (load) {
1100 (*inout)->def = nir_load_deref_with_access(&b->nb, deref,
1101 ptr->type->access | access);
1102 } else {
1103 nir_store_deref_with_access(&b->nb, deref, (*inout)->def, ~0,
1104 ptr->type->access | access);
1105 }
1106 } else {
1107 if (load) {
1108 *inout = vtn_local_load(b, deref, ptr->type->access | access);
1109 } else {
1110 vtn_local_store(b, *inout, deref, ptr->type->access | access);
1111 }
1112 }
1113 return;
1114 }
1115 /* Fall through */
1116
1117 case GLSL_TYPE_INTERFACE:
1118 case GLSL_TYPE_ARRAY:
1119 case GLSL_TYPE_STRUCT: {
1120 unsigned elems = glsl_get_length(ptr->type->type);
1121 struct vtn_access_chain chain = {
1122 .length = 1,
1123 .link = {
1124 { .mode = vtn_access_mode_literal, },
1125 }
1126 };
1127 for (unsigned i = 0; i < elems; i++) {
1128 chain.link[0].id = i;
1129 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain);
1130 _vtn_variable_load_store(b, load, elem, ptr->type->access | access,
1131 &(*inout)->elems[i]);
1132 }
1133 return;
1134 }
1135
1136 default:
1137 vtn_fail("Invalid access chain type");
1138 }
1139 }
1140
1141 struct vtn_ssa_value *
1142 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src)
1143 {
1144 if (vtn_pointer_uses_ssa_offset(b, src)) {
1145 return vtn_block_load(b, src);
1146 } else {
1147 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src->type->type);
1148 _vtn_variable_load_store(b, true, src, src->access, &val);
1149 return val;
1150 }
1151 }
1152
1153 void
1154 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
1155 struct vtn_pointer *dest)
1156 {
1157 if (vtn_pointer_uses_ssa_offset(b, dest)) {
1158 vtn_assert(dest->mode == vtn_variable_mode_ssbo ||
1159 dest->mode == vtn_variable_mode_workgroup);
1160 vtn_block_store(b, src, dest);
1161 } else {
1162 _vtn_variable_load_store(b, false, dest, dest->access, &src);
1163 }
1164 }
1165
1166 static void
1167 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
1168 struct vtn_pointer *src)
1169 {
1170 vtn_assert(glsl_get_bare_type(src->type->type) ==
1171 glsl_get_bare_type(dest->type->type));
1172 enum glsl_base_type base_type = glsl_get_base_type(src->type->type);
1173 switch (base_type) {
1174 case GLSL_TYPE_UINT:
1175 case GLSL_TYPE_INT:
1176 case GLSL_TYPE_UINT16:
1177 case GLSL_TYPE_INT16:
1178 case GLSL_TYPE_UINT8:
1179 case GLSL_TYPE_INT8:
1180 case GLSL_TYPE_UINT64:
1181 case GLSL_TYPE_INT64:
1182 case GLSL_TYPE_FLOAT:
1183 case GLSL_TYPE_FLOAT16:
1184 case GLSL_TYPE_DOUBLE:
1185 case GLSL_TYPE_BOOL:
1186 /* At this point, we have a scalar, vector, or matrix so we know that
1187 * there cannot be any structure splitting still in the way. By
1188 * stopping at the matrix level rather than the vector level, we
1189 * ensure that matrices get loaded in the optimal way even if they
1190 * are storred row-major in a UBO.
1191 */
1192 vtn_variable_store(b, vtn_variable_load(b, src), dest);
1193 return;
1194
1195 case GLSL_TYPE_INTERFACE:
1196 case GLSL_TYPE_ARRAY:
1197 case GLSL_TYPE_STRUCT: {
1198 struct vtn_access_chain chain = {
1199 .length = 1,
1200 .link = {
1201 { .mode = vtn_access_mode_literal, },
1202 }
1203 };
1204 unsigned elems = glsl_get_length(src->type->type);
1205 for (unsigned i = 0; i < elems; i++) {
1206 chain.link[0].id = i;
1207 struct vtn_pointer *src_elem =
1208 vtn_pointer_dereference(b, src, &chain);
1209 struct vtn_pointer *dest_elem =
1210 vtn_pointer_dereference(b, dest, &chain);
1211
1212 _vtn_variable_copy(b, dest_elem, src_elem);
1213 }
1214 return;
1215 }
1216
1217 default:
1218 vtn_fail("Invalid access chain type");
1219 }
1220 }
1221
1222 static void
1223 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
1224 struct vtn_pointer *src)
1225 {
1226 /* TODO: At some point, we should add a special-case for when we can
1227 * just emit a copy_var intrinsic.
1228 */
1229 _vtn_variable_copy(b, dest, src);
1230 }
1231
1232 static void
1233 set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode)
1234 {
1235 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
1236 *mode = nir_var_system_value;
1237 }
1238
1239 static void
1240 vtn_get_builtin_location(struct vtn_builder *b,
1241 SpvBuiltIn builtin, int *location,
1242 nir_variable_mode *mode)
1243 {
1244 switch (builtin) {
1245 case SpvBuiltInPosition:
1246 *location = VARYING_SLOT_POS;
1247 break;
1248 case SpvBuiltInPointSize:
1249 *location = VARYING_SLOT_PSIZ;
1250 break;
1251 case SpvBuiltInClipDistance:
1252 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
1253 break;
1254 case SpvBuiltInCullDistance:
1255 *location = VARYING_SLOT_CULL_DIST0;
1256 break;
1257 case SpvBuiltInVertexId:
1258 case SpvBuiltInVertexIndex:
1259 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't
1260 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the
1261 * same as gl_VertexID, which is non-zero-based, and removes
1262 * VertexIndex. Since they're both defined to be non-zero-based, we use
1263 * SYSTEM_VALUE_VERTEX_ID for both.
1264 */
1265 *location = SYSTEM_VALUE_VERTEX_ID;
1266 set_mode_system_value(b, mode);
1267 break;
1268 case SpvBuiltInInstanceIndex:
1269 *location = SYSTEM_VALUE_INSTANCE_INDEX;
1270 set_mode_system_value(b, mode);
1271 break;
1272 case SpvBuiltInInstanceId:
1273 *location = SYSTEM_VALUE_INSTANCE_ID;
1274 set_mode_system_value(b, mode);
1275 break;
1276 case SpvBuiltInPrimitiveId:
1277 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) {
1278 vtn_assert(*mode == nir_var_shader_in);
1279 *location = VARYING_SLOT_PRIMITIVE_ID;
1280 } else if (*mode == nir_var_shader_out) {
1281 *location = VARYING_SLOT_PRIMITIVE_ID;
1282 } else {
1283 *location = SYSTEM_VALUE_PRIMITIVE_ID;
1284 set_mode_system_value(b, mode);
1285 }
1286 break;
1287 case SpvBuiltInInvocationId:
1288 *location = SYSTEM_VALUE_INVOCATION_ID;
1289 set_mode_system_value(b, mode);
1290 break;
1291 case SpvBuiltInLayer:
1292 *location = VARYING_SLOT_LAYER;
1293 if (b->shader->info.stage == MESA_SHADER_FRAGMENT)
1294 *mode = nir_var_shader_in;
1295 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY)
1296 *mode = nir_var_shader_out;
1297 else if (b->options && b->options->caps.shader_viewport_index_layer &&
1298 (b->shader->info.stage == MESA_SHADER_VERTEX ||
1299 b->shader->info.stage == MESA_SHADER_TESS_EVAL))
1300 *mode = nir_var_shader_out;
1301 else
1302 vtn_fail("invalid stage for SpvBuiltInLayer");
1303 break;
1304 case SpvBuiltInViewportIndex:
1305 *location = VARYING_SLOT_VIEWPORT;
1306 if (b->shader->info.stage == MESA_SHADER_GEOMETRY)
1307 *mode = nir_var_shader_out;
1308 else if (b->options && b->options->caps.shader_viewport_index_layer &&
1309 (b->shader->info.stage == MESA_SHADER_VERTEX ||
1310 b->shader->info.stage == MESA_SHADER_TESS_EVAL))
1311 *mode = nir_var_shader_out;
1312 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT)
1313 *mode = nir_var_shader_in;
1314 else
1315 vtn_fail("invalid stage for SpvBuiltInViewportIndex");
1316 break;
1317 case SpvBuiltInTessLevelOuter:
1318 *location = VARYING_SLOT_TESS_LEVEL_OUTER;
1319 break;
1320 case SpvBuiltInTessLevelInner:
1321 *location = VARYING_SLOT_TESS_LEVEL_INNER;
1322 break;
1323 case SpvBuiltInTessCoord:
1324 *location = SYSTEM_VALUE_TESS_COORD;
1325 set_mode_system_value(b, mode);
1326 break;
1327 case SpvBuiltInPatchVertices:
1328 *location = SYSTEM_VALUE_VERTICES_IN;
1329 set_mode_system_value(b, mode);
1330 break;
1331 case SpvBuiltInFragCoord:
1332 vtn_assert(*mode == nir_var_shader_in);
1333 if (b->options && b->options->frag_coord_is_sysval) {
1334 *mode = nir_var_system_value;
1335 *location = SYSTEM_VALUE_FRAG_COORD;
1336 } else {
1337 *location = VARYING_SLOT_POS;
1338 }
1339 break;
1340 case SpvBuiltInPointCoord:
1341 *location = VARYING_SLOT_PNTC;
1342 vtn_assert(*mode == nir_var_shader_in);
1343 break;
1344 case SpvBuiltInFrontFacing:
1345 *location = SYSTEM_VALUE_FRONT_FACE;
1346 set_mode_system_value(b, mode);
1347 break;
1348 case SpvBuiltInSampleId:
1349 *location = SYSTEM_VALUE_SAMPLE_ID;
1350 set_mode_system_value(b, mode);
1351 break;
1352 case SpvBuiltInSamplePosition:
1353 *location = SYSTEM_VALUE_SAMPLE_POS;
1354 set_mode_system_value(b, mode);
1355 break;
1356 case SpvBuiltInSampleMask:
1357 if (*mode == nir_var_shader_out) {
1358 *location = FRAG_RESULT_SAMPLE_MASK;
1359 } else {
1360 *location = SYSTEM_VALUE_SAMPLE_MASK_IN;
1361 set_mode_system_value(b, mode);
1362 }
1363 break;
1364 case SpvBuiltInFragDepth:
1365 *location = FRAG_RESULT_DEPTH;
1366 vtn_assert(*mode == nir_var_shader_out);
1367 break;
1368 case SpvBuiltInHelperInvocation:
1369 *location = SYSTEM_VALUE_HELPER_INVOCATION;
1370 set_mode_system_value(b, mode);
1371 break;
1372 case SpvBuiltInNumWorkgroups:
1373 *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
1374 set_mode_system_value(b, mode);
1375 break;
1376 case SpvBuiltInWorkgroupSize:
1377 *location = SYSTEM_VALUE_LOCAL_GROUP_SIZE;
1378 set_mode_system_value(b, mode);
1379 break;
1380 case SpvBuiltInWorkgroupId:
1381 *location = SYSTEM_VALUE_WORK_GROUP_ID;
1382 set_mode_system_value(b, mode);
1383 break;
1384 case SpvBuiltInLocalInvocationId:
1385 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
1386 set_mode_system_value(b, mode);
1387 break;
1388 case SpvBuiltInLocalInvocationIndex:
1389 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1390 set_mode_system_value(b, mode);
1391 break;
1392 case SpvBuiltInGlobalInvocationId:
1393 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1394 set_mode_system_value(b, mode);
1395 break;
1396 case SpvBuiltInGlobalLinearId:
1397 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX;
1398 set_mode_system_value(b, mode);
1399 break;
1400 case SpvBuiltInGlobalOffset:
1401 *location = SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID;
1402 set_mode_system_value(b, mode);
1403 break;
1404 case SpvBuiltInBaseVertex:
1405 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same
1406 * semantic as Vulkan BaseVertex (SYSTEM_VALUE_FIRST_VERTEX).
1407 */
1408 if (b->options->environment == NIR_SPIRV_OPENGL)
1409 *location = SYSTEM_VALUE_BASE_VERTEX;
1410 else
1411 *location = SYSTEM_VALUE_FIRST_VERTEX;
1412 set_mode_system_value(b, mode);
1413 break;
1414 case SpvBuiltInBaseInstance:
1415 *location = SYSTEM_VALUE_BASE_INSTANCE;
1416 set_mode_system_value(b, mode);
1417 break;
1418 case SpvBuiltInDrawIndex:
1419 *location = SYSTEM_VALUE_DRAW_ID;
1420 set_mode_system_value(b, mode);
1421 break;
1422 case SpvBuiltInSubgroupSize:
1423 *location = SYSTEM_VALUE_SUBGROUP_SIZE;
1424 set_mode_system_value(b, mode);
1425 break;
1426 case SpvBuiltInSubgroupId:
1427 *location = SYSTEM_VALUE_SUBGROUP_ID;
1428 set_mode_system_value(b, mode);
1429 break;
1430 case SpvBuiltInSubgroupLocalInvocationId:
1431 *location = SYSTEM_VALUE_SUBGROUP_INVOCATION;
1432 set_mode_system_value(b, mode);
1433 break;
1434 case SpvBuiltInNumSubgroups:
1435 *location = SYSTEM_VALUE_NUM_SUBGROUPS;
1436 set_mode_system_value(b, mode);
1437 break;
1438 case SpvBuiltInDeviceIndex:
1439 *location = SYSTEM_VALUE_DEVICE_INDEX;
1440 set_mode_system_value(b, mode);
1441 break;
1442 case SpvBuiltInViewIndex:
1443 if (b->options && b->options->view_index_is_input) {
1444 *location = VARYING_SLOT_VIEW_INDEX;
1445 vtn_assert(*mode == nir_var_shader_in);
1446 } else {
1447 *location = SYSTEM_VALUE_VIEW_INDEX;
1448 set_mode_system_value(b, mode);
1449 }
1450 break;
1451 case SpvBuiltInSubgroupEqMask:
1452 *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK,
1453 set_mode_system_value(b, mode);
1454 break;
1455 case SpvBuiltInSubgroupGeMask:
1456 *location = SYSTEM_VALUE_SUBGROUP_GE_MASK,
1457 set_mode_system_value(b, mode);
1458 break;
1459 case SpvBuiltInSubgroupGtMask:
1460 *location = SYSTEM_VALUE_SUBGROUP_GT_MASK,
1461 set_mode_system_value(b, mode);
1462 break;
1463 case SpvBuiltInSubgroupLeMask:
1464 *location = SYSTEM_VALUE_SUBGROUP_LE_MASK,
1465 set_mode_system_value(b, mode);
1466 break;
1467 case SpvBuiltInSubgroupLtMask:
1468 *location = SYSTEM_VALUE_SUBGROUP_LT_MASK,
1469 set_mode_system_value(b, mode);
1470 break;
1471 case SpvBuiltInFragStencilRefEXT:
1472 *location = FRAG_RESULT_STENCIL;
1473 vtn_assert(*mode == nir_var_shader_out);
1474 break;
1475 case SpvBuiltInWorkDim:
1476 *location = SYSTEM_VALUE_WORK_DIM;
1477 set_mode_system_value(b, mode);
1478 break;
1479 case SpvBuiltInGlobalSize:
1480 *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE;
1481 set_mode_system_value(b, mode);
1482 break;
1483 case SpvBuiltInBaryCoordNoPerspAMD:
1484 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL;
1485 set_mode_system_value(b, mode);
1486 break;
1487 case SpvBuiltInBaryCoordNoPerspCentroidAMD:
1488 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID;
1489 set_mode_system_value(b, mode);
1490 break;
1491 case SpvBuiltInBaryCoordNoPerspSampleAMD:
1492 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE;
1493 set_mode_system_value(b, mode);
1494 break;
1495 case SpvBuiltInBaryCoordSmoothAMD:
1496 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL;
1497 set_mode_system_value(b, mode);
1498 break;
1499 case SpvBuiltInBaryCoordSmoothCentroidAMD:
1500 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID;
1501 set_mode_system_value(b, mode);
1502 break;
1503 case SpvBuiltInBaryCoordSmoothSampleAMD:
1504 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE;
1505 set_mode_system_value(b, mode);
1506 break;
1507 case SpvBuiltInBaryCoordPullModelAMD:
1508 *location = SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL;
1509 set_mode_system_value(b, mode);
1510 break;
1511 default:
1512 vtn_fail("Unsupported builtin: %s (%u)",
1513 spirv_builtin_to_string(builtin), builtin);
1514 }
1515 }
1516
1517 static void
1518 apply_var_decoration(struct vtn_builder *b,
1519 struct nir_variable_data *var_data,
1520 const struct vtn_decoration *dec)
1521 {
1522 switch (dec->decoration) {
1523 case SpvDecorationRelaxedPrecision:
1524 break; /* FIXME: Do nothing with this for now. */
1525 case SpvDecorationNoPerspective:
1526 var_data->interpolation = INTERP_MODE_NOPERSPECTIVE;
1527 break;
1528 case SpvDecorationFlat:
1529 var_data->interpolation = INTERP_MODE_FLAT;
1530 break;
1531 case SpvDecorationExplicitInterpAMD:
1532 var_data->interpolation = INTERP_MODE_EXPLICIT;
1533 break;
1534 case SpvDecorationCentroid:
1535 var_data->centroid = true;
1536 break;
1537 case SpvDecorationSample:
1538 var_data->sample = true;
1539 break;
1540 case SpvDecorationInvariant:
1541 var_data->invariant = true;
1542 break;
1543 case SpvDecorationConstant:
1544 var_data->read_only = true;
1545 break;
1546 case SpvDecorationNonReadable:
1547 var_data->access |= ACCESS_NON_READABLE;
1548 break;
1549 case SpvDecorationNonWritable:
1550 var_data->read_only = true;
1551 var_data->access |= ACCESS_NON_WRITEABLE;
1552 break;
1553 case SpvDecorationRestrict:
1554 var_data->access |= ACCESS_RESTRICT;
1555 break;
1556 case SpvDecorationAliased:
1557 var_data->access &= ~ACCESS_RESTRICT;
1558 break;
1559 case SpvDecorationVolatile:
1560 var_data->access |= ACCESS_VOLATILE;
1561 break;
1562 case SpvDecorationCoherent:
1563 var_data->access |= ACCESS_COHERENT;
1564 break;
1565 case SpvDecorationComponent:
1566 var_data->location_frac = dec->operands[0];
1567 break;
1568 case SpvDecorationIndex:
1569 var_data->index = dec->operands[0];
1570 break;
1571 case SpvDecorationBuiltIn: {
1572 SpvBuiltIn builtin = dec->operands[0];
1573
1574 nir_variable_mode mode = var_data->mode;
1575 vtn_get_builtin_location(b, builtin, &var_data->location, &mode);
1576 var_data->mode = mode;
1577
1578 switch (builtin) {
1579 case SpvBuiltInTessLevelOuter:
1580 case SpvBuiltInTessLevelInner:
1581 case SpvBuiltInClipDistance:
1582 case SpvBuiltInCullDistance:
1583 var_data->compact = true;
1584 break;
1585 default:
1586 break;
1587 }
1588 }
1589
1590 case SpvDecorationSpecId:
1591 case SpvDecorationRowMajor:
1592 case SpvDecorationColMajor:
1593 case SpvDecorationMatrixStride:
1594 case SpvDecorationUniform:
1595 case SpvDecorationUniformId:
1596 case SpvDecorationLinkageAttributes:
1597 break; /* Do nothing with these here */
1598
1599 case SpvDecorationPatch:
1600 var_data->patch = true;
1601 break;
1602
1603 case SpvDecorationLocation:
1604 vtn_fail("Handled above");
1605
1606 case SpvDecorationBlock:
1607 case SpvDecorationBufferBlock:
1608 case SpvDecorationArrayStride:
1609 case SpvDecorationGLSLShared:
1610 case SpvDecorationGLSLPacked:
1611 break; /* These can apply to a type but we don't care about them */
1612
1613 case SpvDecorationBinding:
1614 case SpvDecorationDescriptorSet:
1615 case SpvDecorationNoContraction:
1616 case SpvDecorationInputAttachmentIndex:
1617 vtn_warn("Decoration not allowed for variable or structure member: %s",
1618 spirv_decoration_to_string(dec->decoration));
1619 break;
1620
1621 case SpvDecorationXfbBuffer:
1622 var_data->explicit_xfb_buffer = true;
1623 var_data->xfb.buffer = dec->operands[0];
1624 var_data->always_active_io = true;
1625 break;
1626 case SpvDecorationXfbStride:
1627 var_data->explicit_xfb_stride = true;
1628 var_data->xfb.stride = dec->operands[0];
1629 break;
1630 case SpvDecorationOffset:
1631 var_data->explicit_offset = true;
1632 var_data->offset = dec->operands[0];
1633 break;
1634
1635 case SpvDecorationStream:
1636 var_data->stream = dec->operands[0];
1637 break;
1638
1639 case SpvDecorationCPacked:
1640 case SpvDecorationSaturatedConversion:
1641 case SpvDecorationFuncParamAttr:
1642 case SpvDecorationFPRoundingMode:
1643 case SpvDecorationFPFastMathMode:
1644 case SpvDecorationAlignment:
1645 if (b->shader->info.stage != MESA_SHADER_KERNEL) {
1646 vtn_warn("Decoration only allowed for CL-style kernels: %s",
1647 spirv_decoration_to_string(dec->decoration));
1648 }
1649 break;
1650
1651 case SpvDecorationUserSemantic:
1652 case SpvDecorationUserTypeGOOGLE:
1653 /* User semantic decorations can safely be ignored by the driver. */
1654 break;
1655
1656 case SpvDecorationRestrictPointerEXT:
1657 case SpvDecorationAliasedPointerEXT:
1658 /* TODO: We should actually plumb alias information through NIR. */
1659 break;
1660
1661 default:
1662 vtn_fail_with_decoration("Unhandled decoration", dec->decoration);
1663 }
1664 }
1665
1666 static void
1667 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1668 const struct vtn_decoration *dec, void *out_is_patch)
1669 {
1670 if (dec->decoration == SpvDecorationPatch) {
1671 *((bool *) out_is_patch) = true;
1672 }
1673 }
1674
1675 static void
1676 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1677 const struct vtn_decoration *dec, void *void_var)
1678 {
1679 struct vtn_variable *vtn_var = void_var;
1680
1681 /* Handle decorations that apply to a vtn_variable as a whole */
1682 switch (dec->decoration) {
1683 case SpvDecorationBinding:
1684 vtn_var->binding = dec->operands[0];
1685 vtn_var->explicit_binding = true;
1686 return;
1687 case SpvDecorationDescriptorSet:
1688 vtn_var->descriptor_set = dec->operands[0];
1689 return;
1690 case SpvDecorationInputAttachmentIndex:
1691 vtn_var->input_attachment_index = dec->operands[0];
1692 return;
1693 case SpvDecorationPatch:
1694 vtn_var->patch = true;
1695 break;
1696 case SpvDecorationOffset:
1697 vtn_var->offset = dec->operands[0];
1698 break;
1699 case SpvDecorationNonWritable:
1700 vtn_var->access |= ACCESS_NON_WRITEABLE;
1701 break;
1702 case SpvDecorationNonReadable:
1703 vtn_var->access |= ACCESS_NON_READABLE;
1704 break;
1705 case SpvDecorationVolatile:
1706 vtn_var->access |= ACCESS_VOLATILE;
1707 break;
1708 case SpvDecorationCoherent:
1709 vtn_var->access |= ACCESS_COHERENT;
1710 break;
1711 case SpvDecorationCounterBuffer:
1712 /* Counter buffer decorations can safely be ignored by the driver. */
1713 return;
1714 default:
1715 break;
1716 }
1717
1718 if (val->value_type == vtn_value_type_pointer) {
1719 assert(val->pointer->var == void_var);
1720 assert(member == -1);
1721 } else {
1722 assert(val->value_type == vtn_value_type_type);
1723 }
1724
1725 /* Location is odd. If applied to a split structure, we have to walk the
1726 * whole thing and accumulate the location. It's easier to handle as a
1727 * special case.
1728 */
1729 if (dec->decoration == SpvDecorationLocation) {
1730 unsigned location = dec->operands[0];
1731 if (b->shader->info.stage == MESA_SHADER_FRAGMENT &&
1732 vtn_var->mode == vtn_variable_mode_output) {
1733 location += FRAG_RESULT_DATA0;
1734 } else if (b->shader->info.stage == MESA_SHADER_VERTEX &&
1735 vtn_var->mode == vtn_variable_mode_input) {
1736 location += VERT_ATTRIB_GENERIC0;
1737 } else if (vtn_var->mode == vtn_variable_mode_input ||
1738 vtn_var->mode == vtn_variable_mode_output) {
1739 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1740 } else if (vtn_var->mode != vtn_variable_mode_uniform) {
1741 vtn_warn("Location must be on input, output, uniform, sampler or "
1742 "image variable");
1743 return;
1744 }
1745
1746 if (vtn_var->var->num_members == 0) {
1747 /* This handles the member and lone variable cases */
1748 vtn_var->var->data.location = location;
1749 } else {
1750 /* This handles the structure member case */
1751 assert(vtn_var->var->members);
1752
1753 if (member == -1)
1754 vtn_var->base_location = location;
1755 else
1756 vtn_var->var->members[member].location = location;
1757 }
1758
1759 return;
1760 } else {
1761 if (vtn_var->var) {
1762 if (vtn_var->var->num_members == 0) {
1763 /* We call this function on types as well as variables and not all
1764 * struct types get split so we can end up having stray member
1765 * decorations; just ignore them.
1766 */
1767 if (member == -1)
1768 apply_var_decoration(b, &vtn_var->var->data, dec);
1769 } else if (member >= 0) {
1770 /* Member decorations must come from a type */
1771 assert(val->value_type == vtn_value_type_type);
1772 apply_var_decoration(b, &vtn_var->var->members[member], dec);
1773 } else {
1774 unsigned length =
1775 glsl_get_length(glsl_without_array(vtn_var->type->type));
1776 for (unsigned i = 0; i < length; i++)
1777 apply_var_decoration(b, &vtn_var->var->members[i], dec);
1778 }
1779 } else {
1780 /* A few variables, those with external storage, have no actual
1781 * nir_variables associated with them. Fortunately, all decorations
1782 * we care about for those variables are on the type only.
1783 */
1784 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo ||
1785 vtn_var->mode == vtn_variable_mode_ssbo ||
1786 vtn_var->mode == vtn_variable_mode_push_constant);
1787 }
1788 }
1789 }
1790
1791 enum vtn_variable_mode
1792 vtn_storage_class_to_mode(struct vtn_builder *b,
1793 SpvStorageClass class,
1794 struct vtn_type *interface_type,
1795 nir_variable_mode *nir_mode_out)
1796 {
1797 enum vtn_variable_mode mode;
1798 nir_variable_mode nir_mode;
1799 switch (class) {
1800 case SpvStorageClassUniform:
1801 /* Assume it's an UBO if we lack the interface_type. */
1802 if (!interface_type || interface_type->block) {
1803 mode = vtn_variable_mode_ubo;
1804 nir_mode = nir_var_mem_ubo;
1805 } else if (interface_type->buffer_block) {
1806 mode = vtn_variable_mode_ssbo;
1807 nir_mode = nir_var_mem_ssbo;
1808 } else {
1809 /* Default-block uniforms, coming from gl_spirv */
1810 mode = vtn_variable_mode_uniform;
1811 nir_mode = nir_var_uniform;
1812 }
1813 break;
1814 case SpvStorageClassStorageBuffer:
1815 mode = vtn_variable_mode_ssbo;
1816 nir_mode = nir_var_mem_ssbo;
1817 break;
1818 case SpvStorageClassPhysicalStorageBuffer:
1819 mode = vtn_variable_mode_phys_ssbo;
1820 nir_mode = nir_var_mem_global;
1821 break;
1822 case SpvStorageClassUniformConstant:
1823 if (b->shader->info.stage == MESA_SHADER_KERNEL) {
1824 if (b->options->constant_as_global) {
1825 mode = vtn_variable_mode_cross_workgroup;
1826 nir_mode = nir_var_mem_global;
1827 } else {
1828 mode = vtn_variable_mode_ubo;
1829 nir_mode = nir_var_mem_ubo;
1830 }
1831 } else {
1832 mode = vtn_variable_mode_uniform;
1833 nir_mode = nir_var_uniform;
1834 }
1835 break;
1836 case SpvStorageClassPushConstant:
1837 mode = vtn_variable_mode_push_constant;
1838 nir_mode = nir_var_uniform;
1839 break;
1840 case SpvStorageClassInput:
1841 mode = vtn_variable_mode_input;
1842 nir_mode = nir_var_shader_in;
1843 break;
1844 case SpvStorageClassOutput:
1845 mode = vtn_variable_mode_output;
1846 nir_mode = nir_var_shader_out;
1847 break;
1848 case SpvStorageClassPrivate:
1849 mode = vtn_variable_mode_private;
1850 nir_mode = nir_var_shader_temp;
1851 break;
1852 case SpvStorageClassFunction:
1853 mode = vtn_variable_mode_function;
1854 nir_mode = nir_var_function_temp;
1855 break;
1856 case SpvStorageClassWorkgroup:
1857 mode = vtn_variable_mode_workgroup;
1858 nir_mode = nir_var_mem_shared;
1859 break;
1860 case SpvStorageClassAtomicCounter:
1861 mode = vtn_variable_mode_atomic_counter;
1862 nir_mode = nir_var_uniform;
1863 break;
1864 case SpvStorageClassCrossWorkgroup:
1865 mode = vtn_variable_mode_cross_workgroup;
1866 nir_mode = nir_var_mem_global;
1867 break;
1868 case SpvStorageClassImage:
1869 mode = vtn_variable_mode_image;
1870 nir_mode = nir_var_mem_ubo;
1871 break;
1872 case SpvStorageClassGeneric:
1873 default:
1874 vtn_fail("Unhandled variable storage class: %s (%u)",
1875 spirv_storageclass_to_string(class), class);
1876 }
1877
1878 if (nir_mode_out)
1879 *nir_mode_out = nir_mode;
1880
1881 return mode;
1882 }
1883
1884 nir_address_format
1885 vtn_mode_to_address_format(struct vtn_builder *b, enum vtn_variable_mode mode)
1886 {
1887 switch (mode) {
1888 case vtn_variable_mode_ubo:
1889 return b->options->ubo_addr_format;
1890
1891 case vtn_variable_mode_ssbo:
1892 return b->options->ssbo_addr_format;
1893
1894 case vtn_variable_mode_phys_ssbo:
1895 return b->options->phys_ssbo_addr_format;
1896
1897 case vtn_variable_mode_push_constant:
1898 return b->options->push_const_addr_format;
1899
1900 case vtn_variable_mode_workgroup:
1901 return b->options->shared_addr_format;
1902
1903 case vtn_variable_mode_cross_workgroup:
1904 return b->options->global_addr_format;
1905
1906 case vtn_variable_mode_function:
1907 if (b->physical_ptrs)
1908 return b->options->temp_addr_format;
1909 /* Fall through. */
1910
1911 case vtn_variable_mode_private:
1912 case vtn_variable_mode_uniform:
1913 case vtn_variable_mode_atomic_counter:
1914 case vtn_variable_mode_input:
1915 case vtn_variable_mode_output:
1916 case vtn_variable_mode_image:
1917 return nir_address_format_logical;
1918 }
1919
1920 unreachable("Invalid variable mode");
1921 }
1922
1923 nir_ssa_def *
1924 vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr)
1925 {
1926 if (vtn_pointer_uses_ssa_offset(b, ptr)) {
1927 /* This pointer needs to have a pointer type with actual storage */
1928 vtn_assert(ptr->ptr_type);
1929 vtn_assert(ptr->ptr_type->type);
1930
1931 if (!ptr->offset) {
1932 /* If we don't have an offset then we must be a pointer to the variable
1933 * itself.
1934 */
1935 vtn_assert(!ptr->offset && !ptr->block_index);
1936
1937 struct vtn_access_chain chain = {
1938 .length = 0,
1939 };
1940 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
1941 }
1942
1943 vtn_assert(ptr->offset);
1944 if (ptr->block_index) {
1945 vtn_assert(ptr->mode == vtn_variable_mode_ubo ||
1946 ptr->mode == vtn_variable_mode_ssbo);
1947 return nir_vec2(&b->nb, ptr->block_index, ptr->offset);
1948 } else {
1949 vtn_assert(ptr->mode == vtn_variable_mode_workgroup);
1950 return ptr->offset;
1951 }
1952 } else {
1953 if (vtn_pointer_is_external_block(b, ptr) &&
1954 vtn_type_contains_block(b, ptr->type) &&
1955 ptr->mode != vtn_variable_mode_phys_ssbo) {
1956 /* In this case, we're looking for a block index and not an actual
1957 * deref.
1958 *
1959 * For PhysicalStorageBuffer pointers, we don't have a block index
1960 * at all because we get the pointer directly from the client. This
1961 * assumes that there will never be a SSBO binding variable using the
1962 * PhysicalStorageBuffer storage class. This assumption appears
1963 * to be correct according to the Vulkan spec because the table,
1964 * "Shader Resource and Storage Class Correspondence," the only the
1965 * Uniform storage class with BufferBlock or the StorageBuffer
1966 * storage class with Block can be used.
1967 */
1968 if (!ptr->block_index) {
1969 /* If we don't have a block_index then we must be a pointer to the
1970 * variable itself.
1971 */
1972 vtn_assert(!ptr->deref);
1973
1974 struct vtn_access_chain chain = {
1975 .length = 0,
1976 };
1977 ptr = vtn_nir_deref_pointer_dereference(b, ptr, &chain);
1978 }
1979
1980 return ptr->block_index;
1981 } else {
1982 return &vtn_pointer_to_deref(b, ptr)->dest.ssa;
1983 }
1984 }
1985 }
1986
1987 struct vtn_pointer *
1988 vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa,
1989 struct vtn_type *ptr_type)
1990 {
1991 vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
1992
1993 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
1994 struct vtn_type *without_array =
1995 vtn_type_without_array(ptr_type->deref);
1996
1997 nir_variable_mode nir_mode;
1998 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class,
1999 without_array, &nir_mode);
2000 ptr->type = ptr_type->deref;
2001 ptr->ptr_type = ptr_type;
2002
2003 if (vtn_pointer_uses_ssa_offset(b, ptr)) {
2004 /* This pointer type needs to have actual storage */
2005 vtn_assert(ptr_type->type);
2006 if (ptr->mode == vtn_variable_mode_ubo ||
2007 ptr->mode == vtn_variable_mode_ssbo) {
2008 vtn_assert(ssa->num_components == 2);
2009 ptr->block_index = nir_channel(&b->nb, ssa, 0);
2010 ptr->offset = nir_channel(&b->nb, ssa, 1);
2011 } else {
2012 vtn_assert(ssa->num_components == 1);
2013 ptr->block_index = NULL;
2014 ptr->offset = ssa;
2015 }
2016 } else {
2017 const struct glsl_type *deref_type =
2018 vtn_type_get_nir_type(b, ptr_type->deref, ptr->mode);
2019 if (!vtn_pointer_is_external_block(b, ptr)) {
2020 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode,
2021 deref_type, ptr_type->stride);
2022 } else if (vtn_type_contains_block(b, ptr->type) &&
2023 ptr->mode != vtn_variable_mode_phys_ssbo) {
2024 /* This is a pointer to somewhere in an array of blocks, not a
2025 * pointer to somewhere inside the block. Set the block index
2026 * instead of making a cast.
2027 */
2028 ptr->block_index = ssa;
2029 } else {
2030 /* This is a pointer to something internal or a pointer inside a
2031 * block. It's just a regular cast.
2032 *
2033 * For PhysicalStorageBuffer pointers, we don't have a block index
2034 * at all because we get the pointer directly from the client. This
2035 * assumes that there will never be a SSBO binding variable using the
2036 * PhysicalStorageBuffer storage class. This assumption appears
2037 * to be correct according to the Vulkan spec because the table,
2038 * "Shader Resource and Storage Class Correspondence," the only the
2039 * Uniform storage class with BufferBlock or the StorageBuffer
2040 * storage class with Block can be used.
2041 */
2042 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode,
2043 deref_type, ptr_type->stride);
2044 ptr->deref->dest.ssa.num_components =
2045 glsl_get_vector_elements(ptr_type->type);
2046 ptr->deref->dest.ssa.bit_size = glsl_get_bit_size(ptr_type->type);
2047 }
2048 }
2049
2050 return ptr;
2051 }
2052
2053 static bool
2054 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
2055 {
2056 if (var->patch || !glsl_type_is_array(var->type->type))
2057 return false;
2058
2059 if (var->mode == vtn_variable_mode_input) {
2060 return stage == MESA_SHADER_TESS_CTRL ||
2061 stage == MESA_SHADER_TESS_EVAL ||
2062 stage == MESA_SHADER_GEOMETRY;
2063 }
2064
2065 if (var->mode == vtn_variable_mode_output)
2066 return stage == MESA_SHADER_TESS_CTRL;
2067
2068 return false;
2069 }
2070
2071 static void
2072 assign_missing_member_locations(struct vtn_variable *var)
2073 {
2074 unsigned length =
2075 glsl_get_length(glsl_without_array(var->type->type));
2076 int location = var->base_location;
2077
2078 for (unsigned i = 0; i < length; i++) {
2079 /* From the Vulkan spec:
2080 *
2081 * “If the structure type is a Block but without a Location, then each
2082 * of its members must have a Location decoration.”
2083 *
2084 */
2085 if (var->type->block) {
2086 assert(var->base_location != -1 ||
2087 var->var->members[i].location != -1);
2088 }
2089
2090 /* From the Vulkan spec:
2091 *
2092 * “Any member with its own Location decoration is assigned that
2093 * location. Each remaining member is assigned the location after the
2094 * immediately preceding member in declaration order.”
2095 */
2096 if (var->var->members[i].location != -1)
2097 location = var->var->members[i].location;
2098 else
2099 var->var->members[i].location = location;
2100
2101 /* Below we use type instead of interface_type, because interface_type
2102 * is only available when it is a Block. This code also supports
2103 * input/outputs that are just structs
2104 */
2105 const struct glsl_type *member_type =
2106 glsl_get_struct_field(glsl_without_array(var->type->type), i);
2107
2108 location +=
2109 glsl_count_attribute_slots(member_type,
2110 false /* is_gl_vertex_input */);
2111 }
2112 }
2113
2114
2115 static void
2116 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val,
2117 struct vtn_type *ptr_type, SpvStorageClass storage_class,
2118 nir_constant *const_initializer, nir_variable *var_initializer)
2119 {
2120 vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
2121 struct vtn_type *type = ptr_type->deref;
2122
2123 struct vtn_type *without_array = vtn_type_without_array(ptr_type->deref);
2124
2125 enum vtn_variable_mode mode;
2126 nir_variable_mode nir_mode;
2127 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode);
2128
2129 switch (mode) {
2130 case vtn_variable_mode_ubo:
2131 /* There's no other way to get vtn_variable_mode_ubo */
2132 vtn_assert(without_array->block);
2133 b->shader->info.num_ubos++;
2134 break;
2135 case vtn_variable_mode_ssbo:
2136 if (storage_class == SpvStorageClassStorageBuffer &&
2137 !without_array->block) {
2138 if (b->variable_pointers) {
2139 vtn_fail("Variables in the StorageBuffer storage class must "
2140 "have a struct type with the Block decoration");
2141 } else {
2142 /* If variable pointers are not present, it's still malformed
2143 * SPIR-V but we can parse it and do the right thing anyway.
2144 * Since some of the 8-bit storage tests have bugs in this are,
2145 * just make it a warning for now.
2146 */
2147 vtn_warn("Variables in the StorageBuffer storage class must "
2148 "have a struct type with the Block decoration");
2149 }
2150 }
2151 b->shader->info.num_ssbos++;
2152 break;
2153 case vtn_variable_mode_uniform:
2154 if (without_array->base_type == vtn_base_type_image) {
2155 if (glsl_type_is_image(without_array->glsl_image))
2156 b->shader->info.num_images++;
2157 else if (glsl_type_is_sampler(without_array->glsl_image))
2158 b->shader->info.num_textures++;
2159 }
2160 break;
2161 case vtn_variable_mode_push_constant:
2162 b->shader->num_uniforms = vtn_type_block_size(b, type);
2163 break;
2164
2165 case vtn_variable_mode_image:
2166 vtn_fail("Cannot create a variable with the Image storage class");
2167 break;
2168
2169 case vtn_variable_mode_phys_ssbo:
2170 vtn_fail("Cannot create a variable with the "
2171 "PhysicalStorageBuffer storage class");
2172 break;
2173
2174 default:
2175 /* No tallying is needed */
2176 break;
2177 }
2178
2179 struct vtn_variable *var = rzalloc(b, struct vtn_variable);
2180 var->type = type;
2181 var->mode = mode;
2182 var->base_location = -1;
2183
2184 val->pointer = rzalloc(b, struct vtn_pointer);
2185 val->pointer->mode = var->mode;
2186 val->pointer->type = var->type;
2187 val->pointer->ptr_type = ptr_type;
2188 val->pointer->var = var;
2189 val->pointer->access = var->type->access;
2190
2191 switch (var->mode) {
2192 case vtn_variable_mode_function:
2193 case vtn_variable_mode_private:
2194 case vtn_variable_mode_uniform:
2195 case vtn_variable_mode_atomic_counter:
2196 /* For these, we create the variable normally */
2197 var->var = rzalloc(b->shader, nir_variable);
2198 var->var->name = ralloc_strdup(var->var, val->name);
2199 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2200 var->var->data.mode = nir_mode;
2201 var->var->data.location = -1;
2202 var->var->interface_type = NULL;
2203 break;
2204
2205 case vtn_variable_mode_ubo:
2206 case vtn_variable_mode_ssbo:
2207 var->var = rzalloc(b->shader, nir_variable);
2208 var->var->name = ralloc_strdup(var->var, val->name);
2209
2210 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2211 var->var->interface_type = var->var->type;
2212
2213 var->var->data.mode = nir_mode;
2214 var->var->data.location = -1;
2215
2216 break;
2217
2218 case vtn_variable_mode_workgroup:
2219 /* Create the variable normally */
2220 var->var = rzalloc(b->shader, nir_variable);
2221 var->var->name = ralloc_strdup(var->var, val->name);
2222 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2223 var->var->data.mode = nir_var_mem_shared;
2224 break;
2225
2226 case vtn_variable_mode_input:
2227 case vtn_variable_mode_output: {
2228 /* In order to know whether or not we're a per-vertex inout, we need
2229 * the patch qualifier. This means walking the variable decorations
2230 * early before we actually create any variables. Not a big deal.
2231 *
2232 * GLSLang really likes to place decorations in the most interior
2233 * thing it possibly can. In particular, if you have a struct, it
2234 * will place the patch decorations on the struct members. This
2235 * should be handled by the variable splitting below just fine.
2236 *
2237 * If you have an array-of-struct, things get even more weird as it
2238 * will place the patch decorations on the struct even though it's
2239 * inside an array and some of the members being patch and others not
2240 * makes no sense whatsoever. Since the only sensible thing is for
2241 * it to be all or nothing, we'll call it patch if any of the members
2242 * are declared patch.
2243 */
2244 var->patch = false;
2245 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
2246 if (glsl_type_is_array(var->type->type) &&
2247 glsl_type_is_struct_or_ifc(without_array->type)) {
2248 vtn_foreach_decoration(b, vtn_value(b, without_array->id,
2249 vtn_value_type_type),
2250 var_is_patch_cb, &var->patch);
2251 }
2252
2253 /* For inputs and outputs, we immediately split structures. This
2254 * is for a couple of reasons. For one, builtins may all come in
2255 * a struct and we really want those split out into separate
2256 * variables. For another, interpolation qualifiers can be
2257 * applied to members of the top-level struct ane we need to be
2258 * able to preserve that information.
2259 */
2260
2261 struct vtn_type *per_vertex_type = var->type;
2262 if (is_per_vertex_inout(var, b->shader->info.stage)) {
2263 /* In Geometry shaders (and some tessellation), inputs come
2264 * in per-vertex arrays. However, some builtins come in
2265 * non-per-vertex, hence the need for the is_array check. In
2266 * any case, there are no non-builtin arrays allowed so this
2267 * check should be sufficient.
2268 */
2269 per_vertex_type = var->type->array_element;
2270 }
2271
2272 var->var = rzalloc(b->shader, nir_variable);
2273 var->var->name = ralloc_strdup(var->var, val->name);
2274 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2275 var->var->data.mode = nir_mode;
2276 var->var->data.patch = var->patch;
2277
2278 /* Figure out the interface block type. */
2279 struct vtn_type *iface_type = per_vertex_type;
2280 if (var->mode == vtn_variable_mode_output &&
2281 (b->shader->info.stage == MESA_SHADER_VERTEX ||
2282 b->shader->info.stage == MESA_SHADER_TESS_EVAL ||
2283 b->shader->info.stage == MESA_SHADER_GEOMETRY)) {
2284 /* For vertex data outputs, we can end up with arrays of blocks for
2285 * transform feedback where each array element corresponds to a
2286 * different XFB output buffer.
2287 */
2288 while (iface_type->base_type == vtn_base_type_array)
2289 iface_type = iface_type->array_element;
2290 }
2291 if (iface_type->base_type == vtn_base_type_struct && iface_type->block)
2292 var->var->interface_type = vtn_type_get_nir_type(b, iface_type,
2293 var->mode);
2294
2295 if (per_vertex_type->base_type == vtn_base_type_struct &&
2296 per_vertex_type->block) {
2297 /* It's a struct. Set it up as per-member. */
2298 var->var->num_members = glsl_get_length(per_vertex_type->type);
2299 var->var->members = rzalloc_array(var->var, struct nir_variable_data,
2300 var->var->num_members);
2301
2302 for (unsigned i = 0; i < var->var->num_members; i++) {
2303 var->var->members[i].mode = nir_mode;
2304 var->var->members[i].patch = var->patch;
2305 var->var->members[i].location = -1;
2306 }
2307 }
2308
2309 /* For inputs and outputs, we need to grab locations and builtin
2310 * information from the per-vertex type.
2311 */
2312 vtn_foreach_decoration(b, vtn_value(b, per_vertex_type->id,
2313 vtn_value_type_type),
2314 var_decoration_cb, var);
2315 break;
2316 }
2317
2318 case vtn_variable_mode_push_constant:
2319 case vtn_variable_mode_cross_workgroup:
2320 /* These don't need actual variables. */
2321 break;
2322
2323 case vtn_variable_mode_image:
2324 case vtn_variable_mode_phys_ssbo:
2325 unreachable("Should have been caught before");
2326 }
2327
2328 /* We can only have one type of initializer */
2329 assert(!(const_initializer && var_initializer));
2330 if (const_initializer) {
2331 var->var->constant_initializer =
2332 nir_constant_clone(const_initializer, var->var);
2333 }
2334 if (var_initializer)
2335 var->var->pointer_initializer = var_initializer;
2336
2337 if (var->mode == vtn_variable_mode_uniform ||
2338 var->mode == vtn_variable_mode_ssbo) {
2339 /* SSBOs and images are assumed to not alias in the Simple, GLSL and Vulkan memory models */
2340 var->var->data.access |= b->mem_model != SpvMemoryModelOpenCL ? ACCESS_RESTRICT : 0;
2341 }
2342
2343 vtn_foreach_decoration(b, val, var_decoration_cb, var);
2344 vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer);
2345
2346 /* Propagate access flags from the OpVariable decorations. */
2347 val->pointer->access |= var->access;
2348
2349 if ((var->mode == vtn_variable_mode_input ||
2350 var->mode == vtn_variable_mode_output) &&
2351 var->var->members) {
2352 assign_missing_member_locations(var);
2353 }
2354
2355 if (var->mode == vtn_variable_mode_uniform ||
2356 var->mode == vtn_variable_mode_ubo ||
2357 var->mode == vtn_variable_mode_ssbo ||
2358 var->mode == vtn_variable_mode_atomic_counter) {
2359 /* XXX: We still need the binding information in the nir_variable
2360 * for these. We should fix that.
2361 */
2362 var->var->data.binding = var->binding;
2363 var->var->data.explicit_binding = var->explicit_binding;
2364 var->var->data.descriptor_set = var->descriptor_set;
2365 var->var->data.index = var->input_attachment_index;
2366 var->var->data.offset = var->offset;
2367
2368 if (glsl_type_is_image(glsl_without_array(var->var->type)))
2369 var->var->data.image.format = without_array->image_format;
2370 }
2371
2372 if (var->mode == vtn_variable_mode_function) {
2373 vtn_assert(var->var != NULL && var->var->members == NULL);
2374 nir_function_impl_add_variable(b->nb.impl, var->var);
2375 } else if (var->var) {
2376 nir_shader_add_variable(b->shader, var->var);
2377 } else {
2378 vtn_assert(vtn_pointer_is_external_block(b, val->pointer));
2379 }
2380 }
2381
2382 static void
2383 vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode,
2384 struct vtn_type *dst_type,
2385 struct vtn_type *src_type)
2386 {
2387 if (dst_type->id == src_type->id)
2388 return;
2389
2390 if (vtn_types_compatible(b, dst_type, src_type)) {
2391 /* Early versions of GLSLang would re-emit types unnecessarily and you
2392 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have
2393 * mismatched source and destination types.
2394 *
2395 * https://github.com/KhronosGroup/glslang/issues/304
2396 * https://github.com/KhronosGroup/glslang/issues/307
2397 * https://bugs.freedesktop.org/show_bug.cgi?id=104338
2398 * https://bugs.freedesktop.org/show_bug.cgi?id=104424
2399 */
2400 vtn_warn("Source and destination types of %s do not have the same "
2401 "ID (but are compatible): %u vs %u",
2402 spirv_op_to_string(opcode), dst_type->id, src_type->id);
2403 return;
2404 }
2405
2406 vtn_fail("Source and destination types of %s do not match: %s vs. %s",
2407 spirv_op_to_string(opcode),
2408 glsl_get_type_name(dst_type->type),
2409 glsl_get_type_name(src_type->type));
2410 }
2411
2412 static nir_ssa_def *
2413 nir_shrink_zero_pad_vec(nir_builder *b, nir_ssa_def *val,
2414 unsigned num_components)
2415 {
2416 if (val->num_components == num_components)
2417 return val;
2418
2419 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS];
2420 for (unsigned i = 0; i < num_components; i++) {
2421 if (i < val->num_components)
2422 comps[i] = nir_channel(b, val, i);
2423 else
2424 comps[i] = nir_imm_intN_t(b, 0, val->bit_size);
2425 }
2426 return nir_vec(b, comps, num_components);
2427 }
2428
2429 static nir_ssa_def *
2430 nir_sloppy_bitcast(nir_builder *b, nir_ssa_def *val,
2431 const struct glsl_type *type)
2432 {
2433 const unsigned num_components = glsl_get_vector_elements(type);
2434 const unsigned bit_size = glsl_get_bit_size(type);
2435
2436 /* First, zero-pad to ensure that the value is big enough that when we
2437 * bit-cast it, we don't loose anything.
2438 */
2439 if (val->bit_size < bit_size) {
2440 const unsigned src_num_components_needed =
2441 vtn_align_u32(val->num_components, bit_size / val->bit_size);
2442 val = nir_shrink_zero_pad_vec(b, val, src_num_components_needed);
2443 }
2444
2445 val = nir_bitcast_vector(b, val, bit_size);
2446
2447 return nir_shrink_zero_pad_vec(b, val, num_components);
2448 }
2449
2450 static bool
2451 vtn_get_mem_operands(struct vtn_builder *b, const uint32_t *w, unsigned count,
2452 unsigned *idx, SpvMemoryAccessMask *access, unsigned *alignment,
2453 SpvScope *dest_scope, SpvScope *src_scope)
2454 {
2455 *access = 0;
2456 *alignment = 0;
2457 if (*idx >= count)
2458 return false;
2459
2460 *access = w[(*idx)++];
2461 if (*access & SpvMemoryAccessAlignedMask) {
2462 vtn_assert(*idx < count);
2463 *alignment = w[(*idx)++];
2464 }
2465
2466 if (*access & SpvMemoryAccessMakePointerAvailableMask) {
2467 vtn_assert(*idx < count);
2468 vtn_assert(dest_scope);
2469 *dest_scope = vtn_constant_uint(b, w[(*idx)++]);
2470 }
2471
2472 if (*access & SpvMemoryAccessMakePointerVisibleMask) {
2473 vtn_assert(*idx < count);
2474 vtn_assert(src_scope);
2475 *src_scope = vtn_constant_uint(b, w[(*idx)++]);
2476 }
2477
2478 return true;
2479 }
2480
2481 SpvMemorySemanticsMask
2482 vtn_mode_to_memory_semantics(enum vtn_variable_mode mode)
2483 {
2484 switch (mode) {
2485 case vtn_variable_mode_ssbo:
2486 case vtn_variable_mode_phys_ssbo:
2487 return SpvMemorySemanticsUniformMemoryMask;
2488 case vtn_variable_mode_workgroup:
2489 return SpvMemorySemanticsWorkgroupMemoryMask;
2490 case vtn_variable_mode_cross_workgroup:
2491 return SpvMemorySemanticsCrossWorkgroupMemoryMask;
2492 case vtn_variable_mode_atomic_counter:
2493 return SpvMemorySemanticsAtomicCounterMemoryMask;
2494 case vtn_variable_mode_image:
2495 return SpvMemorySemanticsImageMemoryMask;
2496 case vtn_variable_mode_output:
2497 return SpvMemorySemanticsOutputMemoryMask;
2498 default:
2499 return SpvMemorySemanticsMaskNone;
2500 }
2501 }
2502
2503 static void
2504 vtn_emit_make_visible_barrier(struct vtn_builder *b, SpvMemoryAccessMask access,
2505 SpvScope scope, enum vtn_variable_mode mode)
2506 {
2507 if (!(access & SpvMemoryAccessMakePointerVisibleMask))
2508 return;
2509
2510 vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeVisibleMask |
2511 SpvMemorySemanticsAcquireMask |
2512 vtn_mode_to_memory_semantics(mode));
2513 }
2514
2515 static void
2516 vtn_emit_make_available_barrier(struct vtn_builder *b, SpvMemoryAccessMask access,
2517 SpvScope scope, enum vtn_variable_mode mode)
2518 {
2519 if (!(access & SpvMemoryAccessMakePointerAvailableMask))
2520 return;
2521
2522 vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeAvailableMask |
2523 SpvMemorySemanticsReleaseMask |
2524 vtn_mode_to_memory_semantics(mode));
2525 }
2526
2527 void
2528 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
2529 const uint32_t *w, unsigned count)
2530 {
2531 switch (opcode) {
2532 case SpvOpUndef: {
2533 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef);
2534 val->type = vtn_get_type(b, w[1]);
2535 break;
2536 }
2537
2538 case SpvOpVariable: {
2539 struct vtn_type *ptr_type = vtn_get_type(b, w[1]);
2540
2541 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer);
2542
2543 SpvStorageClass storage_class = w[3];
2544 nir_constant *const_initializer = NULL;
2545 nir_variable *var_initializer = NULL;
2546 if (count > 4) {
2547 struct vtn_value *init = vtn_untyped_value(b, w[4]);
2548 switch (init->value_type) {
2549 case vtn_value_type_constant:
2550 const_initializer = init->constant;
2551 break;
2552 case vtn_value_type_pointer:
2553 var_initializer = init->pointer->var->var;
2554 break;
2555 default:
2556 vtn_fail("SPIR-V variable initializer %u must be constant or pointer",
2557 w[4]);
2558 }
2559 }
2560
2561 vtn_create_variable(b, val, ptr_type, storage_class, const_initializer, var_initializer);
2562
2563 break;
2564 }
2565
2566 case SpvOpConstantSampler: {
2567 /* Synthesize a pointer-to-sampler type, create a variable of that type,
2568 * and give the variable a constant initializer with the sampler params */
2569 struct vtn_type *sampler_type = vtn_value(b, w[1], vtn_value_type_type)->type;
2570 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer);
2571
2572 struct vtn_type *ptr_type = rzalloc(b, struct vtn_type);
2573 ptr_type = rzalloc(b, struct vtn_type);
2574 ptr_type->base_type = vtn_base_type_pointer;
2575 ptr_type->deref = sampler_type;
2576 ptr_type->storage_class = SpvStorageClassUniform;
2577
2578 ptr_type->type = nir_address_format_to_glsl_type(
2579 vtn_mode_to_address_format(b, vtn_variable_mode_function));
2580
2581 vtn_create_variable(b, val, ptr_type, ptr_type->storage_class, NULL, NULL);
2582
2583 nir_variable *nir_var = val->pointer->var->var;
2584 nir_var->data.sampler.is_inline_sampler = true;
2585 nir_var->data.sampler.addressing_mode = w[3];
2586 nir_var->data.sampler.normalized_coordinates = w[4];
2587 nir_var->data.sampler.filter_mode = w[5];
2588
2589 break;
2590 }
2591
2592 case SpvOpAccessChain:
2593 case SpvOpPtrAccessChain:
2594 case SpvOpInBoundsAccessChain:
2595 case SpvOpInBoundsPtrAccessChain: {
2596 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4);
2597 enum gl_access_qualifier access = 0;
2598 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain || opcode == SpvOpInBoundsPtrAccessChain);
2599
2600 unsigned idx = 0;
2601 for (int i = 4; i < count; i++) {
2602 struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
2603 if (link_val->value_type == vtn_value_type_constant) {
2604 chain->link[idx].mode = vtn_access_mode_literal;
2605 chain->link[idx].id = vtn_constant_int(b, w[i]);
2606 } else {
2607 chain->link[idx].mode = vtn_access_mode_id;
2608 chain->link[idx].id = w[i];
2609 }
2610 idx++;
2611 }
2612
2613 struct vtn_type *ptr_type = vtn_get_type(b, w[1]);
2614 struct vtn_pointer *base =
2615 vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
2616 struct vtn_pointer *ptr = vtn_pointer_dereference(b, base, chain);
2617 ptr->ptr_type = ptr_type;
2618 ptr->access |= access;
2619 vtn_push_pointer(b, w[2], ptr);
2620 break;
2621 }
2622
2623 case SpvOpCopyMemory: {
2624 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer);
2625 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer);
2626
2627 vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref);
2628
2629 vtn_variable_copy(b, dest->pointer, src->pointer);
2630 break;
2631 }
2632
2633 case SpvOpLoad: {
2634 struct vtn_type *res_type = vtn_get_type(b, w[1]);
2635 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer);
2636 struct vtn_pointer *src = src_val->pointer;
2637
2638 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref);
2639
2640 unsigned idx = 4, alignment;
2641 SpvMemoryAccessMask access;
2642 SpvScope scope;
2643 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, NULL, &scope);
2644
2645 vtn_emit_make_visible_barrier(b, access, scope, src->mode);
2646
2647 vtn_push_ssa_value(b, w[2], vtn_variable_load(b, src));
2648 break;
2649 }
2650
2651 case SpvOpStore: {
2652 struct vtn_value *dest_val = vtn_value(b, w[1], vtn_value_type_pointer);
2653 struct vtn_pointer *dest = dest_val->pointer;
2654 struct vtn_value *src_val = vtn_untyped_value(b, w[2]);
2655
2656 /* OpStore requires us to actually have a storage type */
2657 vtn_fail_if(dest->type->type == NULL,
2658 "Invalid destination type for OpStore");
2659
2660 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL &&
2661 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) {
2662 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but
2663 * would then store them to a local variable as bool. Work around
2664 * the issue by doing an implicit conversion.
2665 *
2666 * https://github.com/KhronosGroup/glslang/issues/170
2667 * https://bugs.freedesktop.org/show_bug.cgi?id=104424
2668 */
2669 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type "
2670 "OpTypeBool. Doing an implicit conversion to work around "
2671 "the problem.");
2672 struct vtn_ssa_value *bool_ssa =
2673 vtn_create_ssa_value(b, dest->type->type);
2674 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def);
2675 vtn_variable_store(b, bool_ssa, dest);
2676 break;
2677 }
2678
2679 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type);
2680
2681 unsigned idx = 3, alignment;
2682 SpvMemoryAccessMask access;
2683 SpvScope scope;
2684 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, &scope, NULL);
2685
2686 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
2687 vtn_variable_store(b, src, dest);
2688
2689 vtn_emit_make_available_barrier(b, access, scope, dest->mode);
2690 break;
2691 }
2692
2693 case SpvOpArrayLength: {
2694 struct vtn_pointer *ptr =
2695 vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
2696 const uint32_t field = w[4];
2697
2698 vtn_fail_if(ptr->type->base_type != vtn_base_type_struct,
2699 "OpArrayLength must take a pointer to a structure type");
2700 vtn_fail_if(field != ptr->type->length - 1 ||
2701 ptr->type->members[field]->base_type != vtn_base_type_array,
2702 "OpArrayLength must reference the last memeber of the "
2703 "structure and that must be an array");
2704
2705 const uint32_t offset = ptr->type->offsets[field];
2706 const uint32_t stride = ptr->type->members[field]->stride;
2707
2708 if (!ptr->block_index) {
2709 struct vtn_access_chain chain = {
2710 .length = 0,
2711 };
2712 ptr = vtn_pointer_dereference(b, ptr, &chain);
2713 vtn_assert(ptr->block_index);
2714 }
2715
2716 nir_intrinsic_instr *instr =
2717 nir_intrinsic_instr_create(b->nb.shader,
2718 nir_intrinsic_get_buffer_size);
2719 instr->src[0] = nir_src_for_ssa(ptr->block_index);
2720 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
2721 nir_builder_instr_insert(&b->nb, &instr->instr);
2722 nir_ssa_def *buf_size = &instr->dest.ssa;
2723
2724 /* array_length = max(buffer_size - offset, 0) / stride */
2725 nir_ssa_def *array_length =
2726 nir_idiv(&b->nb,
2727 nir_imax(&b->nb,
2728 nir_isub(&b->nb,
2729 buf_size,
2730 nir_imm_int(&b->nb, offset)),
2731 nir_imm_int(&b->nb, 0u)),
2732 nir_imm_int(&b->nb, stride));
2733
2734 vtn_push_nir_ssa(b, w[2], array_length);
2735 break;
2736 }
2737
2738 case SpvOpConvertPtrToU: {
2739 struct vtn_type *u_type = vtn_get_type(b, w[1]);
2740 struct vtn_type *ptr_type = vtn_get_value_type(b, w[3]);
2741
2742 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer ||
2743 ptr_type->type == NULL,
2744 "OpConvertPtrToU can only be used on physical pointers");
2745
2746 vtn_fail_if(u_type->base_type != vtn_base_type_vector &&
2747 u_type->base_type != vtn_base_type_scalar,
2748 "OpConvertPtrToU can only be used to cast to a vector or "
2749 "scalar type");
2750
2751 /* The pointer will be converted to an SSA value automatically */
2752 nir_ssa_def *ptr = vtn_get_nir_ssa(b, w[3]);
2753 nir_ssa_def *u = nir_sloppy_bitcast(&b->nb, ptr, u_type->type);
2754 vtn_push_nir_ssa(b, w[2], u);
2755 break;
2756 }
2757
2758 case SpvOpConvertUToPtr: {
2759 struct vtn_type *ptr_type = vtn_get_type(b, w[1]);
2760 struct vtn_type *u_type = vtn_get_value_type(b, w[3]);
2761
2762 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer ||
2763 ptr_type->type == NULL,
2764 "OpConvertUToPtr can only be used on physical pointers");
2765
2766 vtn_fail_if(u_type->base_type != vtn_base_type_vector &&
2767 u_type->base_type != vtn_base_type_scalar,
2768 "OpConvertUToPtr can only be used to cast from a vector or "
2769 "scalar type");
2770
2771 nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]);
2772 nir_ssa_def *ptr = nir_sloppy_bitcast(&b->nb, u, ptr_type->type);
2773 vtn_push_pointer(b, w[2], vtn_pointer_from_ssa(b, ptr, ptr_type));
2774 break;
2775 }
2776
2777 case SpvOpCopyMemorySized:
2778 default:
2779 vtn_fail_with_opcode("Unhandled opcode", opcode);
2780 }
2781 }