projects / mesa.git / blob
? search:


812ac62b39aec70ab3cff8337a0750aed6395e95
[mesa.git] / src / compiler / spirv / vtn_variables.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Jason Ekstrand (jason@jlekstrand.net)
25 *
26 */
27
28 #include "vtn_private.h"
29
30 static struct vtn_access_chain *
31 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
32 unsigned new_ids)
33 {
34 struct vtn_access_chain *chain;
35
36 unsigned new_len = old->length + new_ids;
37 chain = ralloc_size(b, sizeof(*chain) + new_len * sizeof(chain->link[0]));
38
39 chain->var = old->var;
40 chain->length = new_len;
41
42 for (unsigned i = 0; i < old->length; i++)
43 chain->link[i] = old->link[i];
44
45 return chain;
46 }
47
48 static nir_ssa_def *
49 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
50 unsigned stride)
51 {
52 assert(stride > 0);
53 if (link.mode == vtn_access_mode_literal) {
54 return nir_imm_int(&b->nb, link.id * stride);
55 } else if (stride == 1) {
56 return vtn_ssa_value(b, link.id)->def;
57 } else {
58 return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
59 nir_imm_int(&b->nb, stride));
60 }
61 }
62
63 static struct vtn_type *
64 vtn_access_chain_tail_type(struct vtn_builder *b,
65 struct vtn_access_chain *chain)
66 {
67 struct vtn_type *type = chain->var->type;
68 for (unsigned i = 0; i < chain->length; i++) {
69 if (glsl_type_is_struct(type->type)) {
70 assert(chain->link[i].mode == vtn_access_mode_literal);
71 type = type->members[chain->link[i].id];
72 } else {
73 type = type->array_element;
74 }
75 }
76 return type;
77 }
78
79 /* Crawls a chain of array derefs and rewrites the types so that the
80 * lengths stay the same but the terminal type is the one given by
81 * tail_type. This is useful for split structures.
82 */
83 static void
84 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type)
85 {
86 deref->type = type;
87 if (deref->child) {
88 assert(deref->child->deref_type == nir_deref_type_array);
89 assert(glsl_type_is_array(deref->type));
90 rewrite_deref_types(deref->child, glsl_get_array_element(type));
91 }
92 }
93
94 nir_deref_var *
95 vtn_access_chain_to_deref(struct vtn_builder *b, struct vtn_access_chain *chain)
96 {
97 nir_deref_var *deref_var;
98 if (chain->var->var) {
99 deref_var = nir_deref_var_create(b, chain->var->var);
100 } else {
101 assert(chain->var->members);
102 /* Create the deref_var manually. It will get filled out later. */
103 deref_var = rzalloc(b, nir_deref_var);
104 deref_var->deref.deref_type = nir_deref_type_var;
105 }
106
107 struct vtn_type *deref_type = chain->var->type;
108 nir_deref *tail = &deref_var->deref;
109 nir_variable **members = chain->var->members;
110
111 for (unsigned i = 0; i < chain->length; i++) {
112 enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
113 switch (base_type) {
114 case GLSL_TYPE_UINT:
115 case GLSL_TYPE_INT:
116 case GLSL_TYPE_FLOAT:
117 case GLSL_TYPE_DOUBLE:
118 case GLSL_TYPE_BOOL:
119 case GLSL_TYPE_ARRAY: {
120 deref_type = deref_type->array_element;
121
122 nir_deref_array *deref_arr = nir_deref_array_create(b);
123 deref_arr->deref.type = deref_type->type;
124
125 if (chain->link[i].mode == vtn_access_mode_literal) {
126 deref_arr->deref_array_type = nir_deref_array_type_direct;
127 deref_arr->base_offset = chain->link[i].id;
128 } else {
129 assert(chain->link[i].mode == vtn_access_mode_id);
130 deref_arr->deref_array_type = nir_deref_array_type_indirect;
131 deref_arr->base_offset = 0;
132 deref_arr->indirect =
133 nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
134 }
135 tail->child = &deref_arr->deref;
136 tail = tail->child;
137 break;
138 }
139
140 case GLSL_TYPE_STRUCT: {
141 assert(chain->link[i].mode == vtn_access_mode_literal);
142 unsigned idx = chain->link[i].id;
143 deref_type = deref_type->members[idx];
144 if (members) {
145 /* This is a pre-split structure. */
146 deref_var->var = members[idx];
147 rewrite_deref_types(&deref_var->deref, members[idx]->type);
148 assert(tail->type == deref_type->type);
149 members = NULL;
150 } else {
151 nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
152 deref_struct->deref.type = deref_type->type;
153 tail->child = &deref_struct->deref;
154 tail = tail->child;
155 }
156 break;
157 }
158 default:
159 unreachable("Invalid type for deref");
160 }
161 }
162
163 assert(members == NULL);
164 return deref_var;
165 }
166
167 static void
168 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref,
169 nir_deref *tail, struct vtn_ssa_value *inout)
170 {
171 /* The deref tail may contain a deref to select a component of a vector (in
172 * other words, it might not be an actual tail) so we have to save it away
173 * here since we overwrite it later.
174 */
175 nir_deref *old_child = tail->child;
176
177 if (glsl_type_is_vector_or_scalar(tail->type)) {
178 /* Terminate the deref chain in case there is one more link to pick
179 * off a component of the vector.
180 */
181 tail->child = NULL;
182
183 nir_intrinsic_op op = load ? nir_intrinsic_load_var :
184 nir_intrinsic_store_var;
185
186 nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op);
187 intrin->variables[0] =
188 nir_deref_as_var(nir_copy_deref(intrin, &deref->deref));
189 intrin->num_components = glsl_get_vector_elements(tail->type);
190
191 if (load) {
192 nir_ssa_dest_init(&intrin->instr, &intrin->dest,
193 intrin->num_components,
194 glsl_get_bit_size(tail->type),
195 NULL);
196 inout->def = &intrin->dest.ssa;
197 } else {
198 nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1);
199 intrin->src[0] = nir_src_for_ssa(inout->def);
200 }
201
202 nir_builder_instr_insert(&b->nb, &intrin->instr);
203 } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY ||
204 glsl_type_is_matrix(tail->type)) {
205 unsigned elems = glsl_get_length(tail->type);
206 nir_deref_array *deref_arr = nir_deref_array_create(b);
207 deref_arr->deref_array_type = nir_deref_array_type_direct;
208 deref_arr->deref.type = glsl_get_array_element(tail->type);
209 tail->child = &deref_arr->deref;
210 for (unsigned i = 0; i < elems; i++) {
211 deref_arr->base_offset = i;
212 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
213 }
214 } else {
215 assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT);
216 unsigned elems = glsl_get_length(tail->type);
217 nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0);
218 tail->child = &deref_struct->deref;
219 for (unsigned i = 0; i < elems; i++) {
220 deref_struct->index = i;
221 deref_struct->deref.type = glsl_get_struct_field(tail->type, i);
222 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
223 }
224 }
225
226 tail->child = old_child;
227 }
228
229 nir_deref_var *
230 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
231 {
232 struct vtn_access_chain *chain =
233 vtn_value(b, id, vtn_value_type_access_chain)->access_chain;
234
235 return vtn_access_chain_to_deref(b, chain);
236 }
237
238 /*
239 * Gets the NIR-level deref tail, which may have as a child an array deref
240 * selecting which component due to OpAccessChain supporting per-component
241 * indexing in SPIR-V.
242 */
243 static nir_deref *
244 get_deref_tail(nir_deref_var *deref)
245 {
246 nir_deref *cur = &deref->deref;
247 while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child)
248 cur = cur->child;
249
250 return cur;
251 }
252
253 struct vtn_ssa_value *
254 vtn_local_load(struct vtn_builder *b, nir_deref_var *src)
255 {
256 nir_deref *src_tail = get_deref_tail(src);
257 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
258 _vtn_local_load_store(b, true, src, src_tail, val);
259
260 if (src_tail->child) {
261 nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child);
262 assert(vec_deref->deref.child == NULL);
263 val->type = vec_deref->deref.type;
264 if (vec_deref->deref_array_type == nir_deref_array_type_direct)
265 val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset);
266 else
267 val->def = vtn_vector_extract_dynamic(b, val->def,
268 vec_deref->indirect.ssa);
269 }
270
271 return val;
272 }
273
274 void
275 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
276 nir_deref_var *dest)
277 {
278 nir_deref *dest_tail = get_deref_tail(dest);
279
280 if (dest_tail->child) {
281 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
282 _vtn_local_load_store(b, true, dest, dest_tail, val);
283 nir_deref_array *deref = nir_deref_as_array(dest_tail->child);
284 assert(deref->deref.child == NULL);
285 if (deref->deref_array_type == nir_deref_array_type_direct)
286 val->def = vtn_vector_insert(b, val->def, src->def,
287 deref->base_offset);
288 else
289 val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
290 deref->indirect.ssa);
291 _vtn_local_load_store(b, false, dest, dest_tail, val);
292 } else {
293 _vtn_local_load_store(b, false, dest, dest_tail, src);
294 }
295 }
296
297 static nir_ssa_def *
298 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain,
299 struct vtn_type **type, unsigned *chain_idx)
300 {
301 /* Push constants have no explicit binding */
302 if (chain->var->mode == vtn_variable_mode_push_constant) {
303 *chain_idx = 0;
304 *type = chain->var->type;
305 return NULL;
306 }
307
308 nir_ssa_def *array_index;
309 if (glsl_type_is_array(chain->var->type->type)) {
310 assert(chain->length > 0);
311 array_index = vtn_access_link_as_ssa(b, chain->link[0], 1);
312 *chain_idx = 1;
313 *type = chain->var->type->array_element;
314 } else {
315 array_index = nir_imm_int(&b->nb, 0);
316 *chain_idx = 0;
317 *type = chain->var->type;
318 }
319
320 nir_intrinsic_instr *instr =
321 nir_intrinsic_instr_create(b->nb.shader,
322 nir_intrinsic_vulkan_resource_index);
323 instr->src[0] = nir_src_for_ssa(array_index);
324 nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set);
325 nir_intrinsic_set_binding(instr, chain->var->binding);
326
327 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
328 nir_builder_instr_insert(&b->nb, &instr->instr);
329
330 return &instr->dest.ssa;
331 }
332
333 nir_ssa_def *
334 vtn_access_chain_to_offset(struct vtn_builder *b,
335 struct vtn_access_chain *chain,
336 nir_ssa_def **index_out, struct vtn_type **type_out,
337 unsigned *end_idx_out, bool stop_at_matrix)
338 {
339 unsigned idx = 0;
340 struct vtn_type *type;
341 *index_out = get_vulkan_resource_index(b, chain, &type, &idx);
342
343 nir_ssa_def *offset = nir_imm_int(&b->nb, 0);
344 for (; idx < chain->length; idx++) {
345 enum glsl_base_type base_type = glsl_get_base_type(type->type);
346 switch (base_type) {
347 case GLSL_TYPE_UINT:
348 case GLSL_TYPE_INT:
349 case GLSL_TYPE_FLOAT:
350 case GLSL_TYPE_DOUBLE:
351 case GLSL_TYPE_BOOL:
352 /* Some users may not want matrix or vector derefs */
353 if (stop_at_matrix)
354 goto end;
355 /* Fall through */
356
357 case GLSL_TYPE_ARRAY:
358 offset = nir_iadd(&b->nb, offset,
359 vtn_access_link_as_ssa(b, chain->link[idx],
360 type->stride));
361
362 type = type->array_element;
363 break;
364
365 case GLSL_TYPE_STRUCT: {
366 assert(chain->link[idx].mode == vtn_access_mode_literal);
367 unsigned member = chain->link[idx].id;
368 offset = nir_iadd(&b->nb, offset,
369 nir_imm_int(&b->nb, type->offsets[member]));
370 type = type->members[member];
371 break;
372 }
373
374 default:
375 unreachable("Invalid type for deref");
376 }
377 }
378
379 end:
380 *type_out = type;
381 if (end_idx_out)
382 *end_idx_out = idx;
383
384 return offset;
385 }
386
387 static void
388 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
389 nir_ssa_def *index, nir_ssa_def *offset,
390 struct vtn_ssa_value **inout, const struct glsl_type *type)
391 {
392 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
393 instr->num_components = glsl_get_vector_elements(type);
394
395 int src = 0;
396 if (!load) {
397 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
398 instr->src[src++] = nir_src_for_ssa((*inout)->def);
399 }
400
401 /* We set the base and size for push constant load to the entire push
402 * constant block for now.
403 */
404 if (op == nir_intrinsic_load_push_constant) {
405 nir_intrinsic_set_base(instr, 0);
406 nir_intrinsic_set_range(instr, 128);
407 }
408
409 if (index)
410 instr->src[src++] = nir_src_for_ssa(index);
411
412 instr->src[src++] = nir_src_for_ssa(offset);
413
414 if (load) {
415 nir_ssa_dest_init(&instr->instr, &instr->dest,
416 instr->num_components,
417 glsl_get_bit_size(type), NULL);
418 (*inout)->def = &instr->dest.ssa;
419 }
420
421 nir_builder_instr_insert(&b->nb, &instr->instr);
422
423 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
424 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
425 }
426
427 static void
428 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
429 nir_ssa_def *index, nir_ssa_def *offset,
430 struct vtn_access_chain *chain, unsigned chain_idx,
431 struct vtn_type *type, struct vtn_ssa_value **inout)
432 {
433 if (chain && chain_idx >= chain->length)
434 chain = NULL;
435
436 if (load && chain == NULL && *inout == NULL)
437 *inout = vtn_create_ssa_value(b, type->type);
438
439 enum glsl_base_type base_type = glsl_get_base_type(type->type);
440 switch (base_type) {
441 case GLSL_TYPE_UINT:
442 case GLSL_TYPE_INT:
443 case GLSL_TYPE_FLOAT:
444 case GLSL_TYPE_BOOL:
445 /* This is where things get interesting. At this point, we've hit
446 * a vector, a scalar, or a matrix.
447 */
448 if (glsl_type_is_matrix(type->type)) {
449 if (chain == NULL) {
450 /* Loading the whole matrix */
451 struct vtn_ssa_value *transpose;
452 unsigned num_ops, vec_width;
453 if (type->row_major) {
454 num_ops = glsl_get_vector_elements(type->type);
455 vec_width = glsl_get_matrix_columns(type->type);
456 if (load) {
457 const struct glsl_type *transpose_type =
458 glsl_matrix_type(base_type, vec_width, num_ops);
459 *inout = vtn_create_ssa_value(b, transpose_type);
460 } else {
461 transpose = vtn_ssa_transpose(b, *inout);
462 inout = &transpose;
463 }
464 } else {
465 num_ops = glsl_get_matrix_columns(type->type);
466 vec_width = glsl_get_vector_elements(type->type);
467 }
468
469 for (unsigned i = 0; i < num_ops; i++) {
470 nir_ssa_def *elem_offset =
471 nir_iadd(&b->nb, offset,
472 nir_imm_int(&b->nb, i * type->stride));
473 _vtn_load_store_tail(b, op, load, index, elem_offset,
474 &(*inout)->elems[i],
475 glsl_vector_type(base_type, vec_width));
476 }
477
478 if (load && type->row_major)
479 *inout = vtn_ssa_transpose(b, *inout);
480 } else if (type->row_major) {
481 /* Row-major but with an access chiain. */
482 nir_ssa_def *col_offset =
483 vtn_access_link_as_ssa(b, chain->link[chain_idx],
484 type->array_element->stride);
485 offset = nir_iadd(&b->nb, offset, col_offset);
486
487 if (chain_idx + 1 < chain->length) {
488 /* Picking off a single element */
489 nir_ssa_def *row_offset =
490 vtn_access_link_as_ssa(b, chain->link[chain_idx + 1],
491 type->stride);
492 offset = nir_iadd(&b->nb, offset, row_offset);
493 if (load)
494 *inout = vtn_create_ssa_value(b, glsl_scalar_type(base_type));
495 _vtn_load_store_tail(b, op, load, index, offset, inout,
496 glsl_scalar_type(base_type));
497 } else {
498 /* Grabbing a column; picking one element off each row */
499 unsigned num_comps = glsl_get_vector_elements(type->type);
500 const struct glsl_type *column_type =
501 glsl_get_column_type(type->type);
502
503 nir_ssa_def *comps[4];
504 for (unsigned i = 0; i < num_comps; i++) {
505 nir_ssa_def *elem_offset =
506 nir_iadd(&b->nb, offset,
507 nir_imm_int(&b->nb, i * type->stride));
508
509 struct vtn_ssa_value *comp, temp_val;
510 if (!load) {
511 temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
512 temp_val.type = glsl_scalar_type(base_type);
513 }
514 comp = &temp_val;
515 _vtn_load_store_tail(b, op, load, index, elem_offset,
516 &comp, glsl_scalar_type(base_type));
517 comps[i] = comp->def;
518 }
519
520 if (load) {
521 if (*inout == NULL)
522 *inout = vtn_create_ssa_value(b, column_type);
523
524 (*inout)->def = nir_vec(&b->nb, comps, num_comps);
525 }
526 }
527 } else {
528 /* Column-major with a deref. Fall through to array case. */
529 nir_ssa_def *col_offset =
530 vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
531 offset = nir_iadd(&b->nb, offset, col_offset);
532
533 _vtn_block_load_store(b, op, load, index, offset,
534 chain, chain_idx + 1,
535 type->array_element, inout);
536 }
537 } else if (chain == NULL) {
538 /* Single whole vector */
539 assert(glsl_type_is_vector_or_scalar(type->type));
540 _vtn_load_store_tail(b, op, load, index, offset, inout, type->type);
541 } else {
542 /* Single component of a vector. Fall through to array case. */
543 nir_ssa_def *elem_offset =
544 vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
545 offset = nir_iadd(&b->nb, offset, elem_offset);
546
547 _vtn_block_load_store(b, op, load, index, offset, NULL, 0,
548 type->array_element, inout);
549 }
550 return;
551
552 case GLSL_TYPE_ARRAY: {
553 unsigned elems = glsl_get_length(type->type);
554 for (unsigned i = 0; i < elems; i++) {
555 nir_ssa_def *elem_off =
556 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
557 _vtn_block_load_store(b, op, load, index, elem_off, NULL, 0,
558 type->array_element, &(*inout)->elems[i]);
559 }
560 return;
561 }
562
563 case GLSL_TYPE_STRUCT: {
564 unsigned elems = glsl_get_length(type->type);
565 for (unsigned i = 0; i < elems; i++) {
566 nir_ssa_def *elem_off =
567 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
568 _vtn_block_load_store(b, op, load, index, elem_off, NULL, 0,
569 type->members[i], &(*inout)->elems[i]);
570 }
571 return;
572 }
573
574 default:
575 unreachable("Invalid block member type");
576 }
577 }
578
579 static struct vtn_ssa_value *
580 vtn_block_load(struct vtn_builder *b, struct vtn_access_chain *src)
581 {
582 nir_intrinsic_op op;
583 switch (src->var->mode) {
584 case vtn_variable_mode_ubo:
585 op = nir_intrinsic_load_ubo;
586 break;
587 case vtn_variable_mode_ssbo:
588 op = nir_intrinsic_load_ssbo;
589 break;
590 case vtn_variable_mode_push_constant:
591 op = nir_intrinsic_load_push_constant;
592 break;
593 default:
594 assert(!"Invalid block variable mode");
595 }
596
597 nir_ssa_def *offset, *index = NULL;
598 struct vtn_type *type;
599 unsigned chain_idx;
600 offset = vtn_access_chain_to_offset(b, src, &index, &type, &chain_idx, true);
601
602 struct vtn_ssa_value *value = NULL;
603 _vtn_block_load_store(b, op, true, index, offset,
604 src, chain_idx, type, &value);
605 return value;
606 }
607
608 static void
609 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
610 struct vtn_access_chain *dst)
611 {
612 nir_ssa_def *offset, *index = NULL;
613 struct vtn_type *type;
614 unsigned chain_idx;
615 offset = vtn_access_chain_to_offset(b, dst, &index, &type, &chain_idx, true);
616
617 _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset,
618 dst, chain_idx, type, &src);
619 }
620
621 static bool
622 vtn_variable_is_external_block(struct vtn_variable *var)
623 {
624 return var->mode == vtn_variable_mode_ssbo ||
625 var->mode == vtn_variable_mode_ubo ||
626 var->mode == vtn_variable_mode_push_constant;
627 }
628
629 static void
630 _vtn_variable_load_store(struct vtn_builder *b, bool load,
631 struct vtn_access_chain *chain,
632 struct vtn_type *tail_type,
633 struct vtn_ssa_value **inout)
634 {
635 enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
636 switch (base_type) {
637 case GLSL_TYPE_UINT:
638 case GLSL_TYPE_INT:
639 case GLSL_TYPE_FLOAT:
640 case GLSL_TYPE_BOOL:
641 /* At this point, we have a scalar, vector, or matrix so we know that
642 * there cannot be any structure splitting still in the way. By
643 * stopping at the matrix level rather than the vector level, we
644 * ensure that matrices get loaded in the optimal way even if they
645 * are storred row-major in a UBO.
646 */
647 if (load) {
648 *inout = vtn_local_load(b, vtn_access_chain_to_deref(b, chain));
649 } else {
650 vtn_local_store(b, *inout, vtn_access_chain_to_deref(b, chain));
651 }
652 return;
653
654 case GLSL_TYPE_ARRAY:
655 case GLSL_TYPE_STRUCT: {
656 struct vtn_access_chain *new_chain =
657 vtn_access_chain_extend(b, chain, 1);
658 new_chain->link[chain->length].mode = vtn_access_mode_literal;
659 unsigned elems = glsl_get_length(tail_type->type);
660 if (load) {
661 assert(*inout == NULL);
662 *inout = rzalloc(b, struct vtn_ssa_value);
663 (*inout)->type = tail_type->type;
664 (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
665 }
666 for (unsigned i = 0; i < elems; i++) {
667 new_chain->link[chain->length].id = i;
668 struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
669 tail_type->array_element : tail_type->members[i];
670 _vtn_variable_load_store(b, load, new_chain, elem_type,
671 &(*inout)->elems[i]);
672 }
673 return;
674 }
675
676 default:
677 unreachable("Invalid access chain type");
678 }
679 }
680
681 struct vtn_ssa_value *
682 vtn_variable_load(struct vtn_builder *b, struct vtn_access_chain *src)
683 {
684 if (vtn_variable_is_external_block(src->var)) {
685 return vtn_block_load(b, src);
686 } else {
687 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
688 struct vtn_ssa_value *val = NULL;
689 _vtn_variable_load_store(b, true, src, tail_type, &val);
690 return val;
691 }
692 }
693
694 void
695 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
696 struct vtn_access_chain *dest)
697 {
698 if (vtn_variable_is_external_block(dest->var)) {
699 assert(dest->var->mode == vtn_variable_mode_ssbo);
700 vtn_block_store(b, src, dest);
701 } else {
702 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, dest);
703 _vtn_variable_load_store(b, false, dest, tail_type, &src);
704 }
705 }
706
707 static void
708 _vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
709 struct vtn_access_chain *src, struct vtn_type *tail_type)
710 {
711 enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
712 switch (base_type) {
713 case GLSL_TYPE_UINT:
714 case GLSL_TYPE_INT:
715 case GLSL_TYPE_FLOAT:
716 case GLSL_TYPE_BOOL:
717 /* At this point, we have a scalar, vector, or matrix so we know that
718 * there cannot be any structure splitting still in the way. By
719 * stopping at the matrix level rather than the vector level, we
720 * ensure that matrices get loaded in the optimal way even if they
721 * are storred row-major in a UBO.
722 */
723 vtn_variable_store(b, vtn_variable_load(b, src), dest);
724 return;
725
726 case GLSL_TYPE_ARRAY:
727 case GLSL_TYPE_STRUCT: {
728 struct vtn_access_chain *new_src, *new_dest;
729 new_src = vtn_access_chain_extend(b, src, 1);
730 new_dest = vtn_access_chain_extend(b, dest, 1);
731 new_src->link[src->length].mode = vtn_access_mode_literal;
732 new_dest->link[dest->length].mode = vtn_access_mode_literal;
733 unsigned elems = glsl_get_length(tail_type->type);
734 for (unsigned i = 0; i < elems; i++) {
735 new_src->link[src->length].id = i;
736 new_dest->link[dest->length].id = i;
737 struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
738 tail_type->array_element : tail_type->members[i];
739 _vtn_variable_copy(b, new_dest, new_src, elem_type);
740 }
741 return;
742 }
743
744 default:
745 unreachable("Invalid access chain type");
746 }
747 }
748
749 static void
750 vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
751 struct vtn_access_chain *src)
752 {
753 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
754 assert(vtn_access_chain_tail_type(b, dest)->type == tail_type->type);
755
756 /* TODO: At some point, we should add a special-case for when we can
757 * just emit a copy_var intrinsic.
758 */
759 _vtn_variable_copy(b, dest, src, tail_type);
760 }
761
762 static void
763 set_mode_system_value(nir_variable_mode *mode)
764 {
765 assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
766 *mode = nir_var_system_value;
767 }
768
769 static void
770 vtn_get_builtin_location(struct vtn_builder *b,
771 SpvBuiltIn builtin, int *location,
772 nir_variable_mode *mode)
773 {
774 switch (builtin) {
775 case SpvBuiltInPosition:
776 *location = VARYING_SLOT_POS;
777 break;
778 case SpvBuiltInPointSize:
779 *location = VARYING_SLOT_PSIZ;
780 break;
781 case SpvBuiltInClipDistance:
782 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
783 break;
784 case SpvBuiltInCullDistance:
785 /* XXX figure this out */
786 break;
787 case SpvBuiltInVertexIndex:
788 *location = SYSTEM_VALUE_VERTEX_ID;
789 set_mode_system_value(mode);
790 break;
791 case SpvBuiltInVertexId:
792 /* Vulkan defines VertexID to be zero-based and reserves the new
793 * builtin keyword VertexIndex to indicate the non-zero-based value.
794 */
795 *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
796 set_mode_system_value(mode);
797 break;
798 case SpvBuiltInInstanceIndex:
799 *location = SYSTEM_VALUE_INSTANCE_INDEX;
800 set_mode_system_value(mode);
801 break;
802 case SpvBuiltInInstanceId:
803 *location = SYSTEM_VALUE_INSTANCE_ID;
804 set_mode_system_value(mode);
805 break;
806 case SpvBuiltInPrimitiveId:
807 *location = VARYING_SLOT_PRIMITIVE_ID;
808 *mode = nir_var_shader_out;
809 break;
810 case SpvBuiltInInvocationId:
811 *location = SYSTEM_VALUE_INVOCATION_ID;
812 set_mode_system_value(mode);
813 break;
814 case SpvBuiltInLayer:
815 *location = VARYING_SLOT_LAYER;
816 *mode = nir_var_shader_out;
817 break;
818 case SpvBuiltInViewportIndex:
819 *location = VARYING_SLOT_VIEWPORT;
820 if (b->shader->stage == MESA_SHADER_GEOMETRY)
821 *mode = nir_var_shader_out;
822 else if (b->shader->stage == MESA_SHADER_FRAGMENT)
823 *mode = nir_var_shader_in;
824 else
825 unreachable("invalid stage for SpvBuiltInViewportIndex");
826 break;
827 case SpvBuiltInTessLevelOuter:
828 case SpvBuiltInTessLevelInner:
829 case SpvBuiltInTessCoord:
830 case SpvBuiltInPatchVertices:
831 unreachable("no tessellation support");
832 case SpvBuiltInFragCoord:
833 *location = VARYING_SLOT_POS;
834 assert(*mode == nir_var_shader_in);
835 break;
836 case SpvBuiltInPointCoord:
837 *location = VARYING_SLOT_PNTC;
838 assert(*mode == nir_var_shader_in);
839 break;
840 case SpvBuiltInFrontFacing:
841 *location = VARYING_SLOT_FACE;
842 assert(*mode == nir_var_shader_in);
843 break;
844 case SpvBuiltInSampleId:
845 *location = SYSTEM_VALUE_SAMPLE_ID;
846 set_mode_system_value(mode);
847 break;
848 case SpvBuiltInSamplePosition:
849 *location = SYSTEM_VALUE_SAMPLE_POS;
850 set_mode_system_value(mode);
851 break;
852 case SpvBuiltInSampleMask:
853 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; /* XXX out? */
854 set_mode_system_value(mode);
855 break;
856 case SpvBuiltInFragDepth:
857 *location = FRAG_RESULT_DEPTH;
858 assert(*mode == nir_var_shader_out);
859 break;
860 case SpvBuiltInNumWorkgroups:
861 *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
862 set_mode_system_value(mode);
863 break;
864 case SpvBuiltInWorkgroupSize:
865 /* This should already be handled */
866 unreachable("unsupported builtin");
867 break;
868 case SpvBuiltInWorkgroupId:
869 *location = SYSTEM_VALUE_WORK_GROUP_ID;
870 set_mode_system_value(mode);
871 break;
872 case SpvBuiltInLocalInvocationId:
873 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
874 set_mode_system_value(mode);
875 break;
876 case SpvBuiltInLocalInvocationIndex:
877 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
878 set_mode_system_value(mode);
879 break;
880 case SpvBuiltInGlobalInvocationId:
881 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
882 set_mode_system_value(mode);
883 break;
884 case SpvBuiltInHelperInvocation:
885 default:
886 unreachable("unsupported builtin");
887 }
888 }
889
890 static void
891 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
892 const struct vtn_decoration *dec, void *void_var)
893 {
894 struct vtn_variable *vtn_var = void_var;
895
896 /* Handle decorations that apply to a vtn_variable as a whole */
897 switch (dec->decoration) {
898 case SpvDecorationBinding:
899 vtn_var->binding = dec->literals[0];
900 return;
901 case SpvDecorationDescriptorSet:
902 vtn_var->descriptor_set = dec->literals[0];
903 return;
904 default:
905 break;
906 }
907
908 /* Now we handle decorations that apply to a particular nir_variable */
909 nir_variable *nir_var = vtn_var->var;
910 if (val->value_type == vtn_value_type_access_chain) {
911 assert(val->access_chain->length == 0);
912 assert(val->access_chain->var == void_var);
913 assert(member == -1);
914 } else {
915 assert(val->value_type == vtn_value_type_type);
916 if (member != -1)
917 nir_var = vtn_var->members[member];
918 }
919
920 /* Location is odd in that it can apply in three different cases: To a
921 * non-split variable, to a whole split variable, or to one structure
922 * member of a split variable.
923 */
924 if (dec->decoration == SpvDecorationLocation) {
925 unsigned location = dec->literals[0];
926 bool is_vertex_input;
927 if (b->shader->stage == MESA_SHADER_FRAGMENT &&
928 vtn_var->mode == vtn_variable_mode_output) {
929 is_vertex_input = false;
930 location += FRAG_RESULT_DATA0;
931 } else if (b->shader->stage == MESA_SHADER_VERTEX &&
932 vtn_var->mode == vtn_variable_mode_input) {
933 is_vertex_input = true;
934 location += VERT_ATTRIB_GENERIC0;
935 } else if (vtn_var->mode == vtn_variable_mode_input ||
936 vtn_var->mode == vtn_variable_mode_output) {
937 is_vertex_input = false;
938 location += VARYING_SLOT_VAR0;
939 } else {
940 assert(!"Location must be on input or output variable");
941 }
942
943 if (nir_var) {
944 /* This handles the member and lone variable cases */
945 nir_var->data.location = location;
946 nir_var->data.explicit_location = true;
947 } else {
948 /* This handles the structure member case */
949 assert(vtn_var->members);
950 unsigned length =
951 glsl_get_length(glsl_without_array(vtn_var->type->type));
952 for (unsigned i = 0; i < length; i++) {
953 vtn_var->members[i]->data.location = location;
954 vtn_var->members[i]->data.explicit_location = true;
955 location +=
956 glsl_count_attribute_slots(vtn_var->members[i]->interface_type,
957 is_vertex_input);
958 }
959 }
960 return;
961 }
962
963 if (nir_var == NULL)
964 return;
965
966 switch (dec->decoration) {
967 case SpvDecorationRelaxedPrecision:
968 break; /* FIXME: Do nothing with this for now. */
969 case SpvDecorationNoPerspective:
970 nir_var->data.interpolation = INTERP_QUALIFIER_NOPERSPECTIVE;
971 break;
972 case SpvDecorationFlat:
973 nir_var->data.interpolation = INTERP_QUALIFIER_FLAT;
974 break;
975 case SpvDecorationCentroid:
976 nir_var->data.centroid = true;
977 break;
978 case SpvDecorationSample:
979 nir_var->data.sample = true;
980 break;
981 case SpvDecorationInvariant:
982 nir_var->data.invariant = true;
983 break;
984 case SpvDecorationConstant:
985 assert(nir_var->constant_initializer != NULL);
986 nir_var->data.read_only = true;
987 break;
988 case SpvDecorationNonWritable:
989 nir_var->data.read_only = true;
990 break;
991 case SpvDecorationComponent:
992 nir_var->data.location_frac = dec->literals[0];
993 break;
994 case SpvDecorationIndex:
995 nir_var->data.explicit_index = true;
996 nir_var->data.index = dec->literals[0];
997 break;
998 case SpvDecorationBuiltIn: {
999 SpvBuiltIn builtin = dec->literals[0];
1000
1001 if (builtin == SpvBuiltInWorkgroupSize) {
1002 /* This shouldn't be a builtin. It's actually a constant. */
1003 nir_var->data.mode = nir_var_global;
1004 nir_var->data.read_only = true;
1005
1006 nir_constant *c = rzalloc(nir_var, nir_constant);
1007 c->value.u[0] = b->shader->info.cs.local_size[0];
1008 c->value.u[1] = b->shader->info.cs.local_size[1];
1009 c->value.u[2] = b->shader->info.cs.local_size[2];
1010 nir_var->constant_initializer = c;
1011 break;
1012 }
1013
1014 nir_variable_mode mode = nir_var->data.mode;
1015 vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode);
1016 nir_var->data.explicit_location = true;
1017 nir_var->data.mode = mode;
1018
1019 if (builtin == SpvBuiltInFragCoord || builtin == SpvBuiltInSamplePosition)
1020 nir_var->data.origin_upper_left = b->origin_upper_left;
1021
1022 if (builtin == SpvBuiltInFragCoord)
1023 nir_var->data.pixel_center_integer = b->pixel_center_integer;
1024 break;
1025 }
1026
1027 case SpvDecorationSpecId:
1028 case SpvDecorationRowMajor:
1029 case SpvDecorationColMajor:
1030 case SpvDecorationMatrixStride:
1031 case SpvDecorationRestrict:
1032 case SpvDecorationAliased:
1033 case SpvDecorationVolatile:
1034 case SpvDecorationCoherent:
1035 case SpvDecorationNonReadable:
1036 case SpvDecorationUniform:
1037 case SpvDecorationStream:
1038 case SpvDecorationOffset:
1039 case SpvDecorationLinkageAttributes:
1040 break; /* Do nothing with these here */
1041
1042 case SpvDecorationPatch:
1043 unreachable("Tessellation not yet supported");
1044
1045 case SpvDecorationLocation:
1046 unreachable("Handled above");
1047
1048 case SpvDecorationBlock:
1049 case SpvDecorationBufferBlock:
1050 case SpvDecorationArrayStride:
1051 case SpvDecorationGLSLShared:
1052 case SpvDecorationGLSLPacked:
1053 break; /* These can apply to a type but we don't care about them */
1054
1055 case SpvDecorationBinding:
1056 case SpvDecorationDescriptorSet:
1057 case SpvDecorationNoContraction:
1058 case SpvDecorationInputAttachmentIndex:
1059 unreachable("Decoration not allowed for variable or structure member");
1060
1061 case SpvDecorationXfbBuffer:
1062 case SpvDecorationXfbStride:
1063 unreachable("Vulkan does not have transform feedback");
1064
1065 case SpvDecorationCPacked:
1066 case SpvDecorationSaturatedConversion:
1067 case SpvDecorationFuncParamAttr:
1068 case SpvDecorationFPRoundingMode:
1069 case SpvDecorationFPFastMathMode:
1070 case SpvDecorationAlignment:
1071 unreachable("Decoraiton only allowed for CL-style kernels");
1072 }
1073 }
1074
1075 /* Tries to compute the size of an interface block based on the strides and
1076 * offsets that are provided to us in the SPIR-V source.
1077 */
1078 static unsigned
1079 vtn_type_block_size(struct vtn_type *type)
1080 {
1081 enum glsl_base_type base_type = glsl_get_base_type(type->type);
1082 switch (base_type) {
1083 case GLSL_TYPE_UINT:
1084 case GLSL_TYPE_INT:
1085 case GLSL_TYPE_FLOAT:
1086 case GLSL_TYPE_BOOL:
1087 case GLSL_TYPE_DOUBLE: {
1088 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
1089 glsl_get_matrix_columns(type->type);
1090 if (cols > 1) {
1091 assert(type->stride > 0);
1092 return type->stride * cols;
1093 } else if (base_type == GLSL_TYPE_DOUBLE) {
1094 return glsl_get_vector_elements(type->type) * 8;
1095 } else {
1096 return glsl_get_vector_elements(type->type) * 4;
1097 }
1098 }
1099
1100 case GLSL_TYPE_STRUCT:
1101 case GLSL_TYPE_INTERFACE: {
1102 unsigned size = 0;
1103 unsigned num_fields = glsl_get_length(type->type);
1104 for (unsigned f = 0; f < num_fields; f++) {
1105 unsigned field_end = type->offsets[f] +
1106 vtn_type_block_size(type->members[f]);
1107 size = MAX2(size, field_end);
1108 }
1109 return size;
1110 }
1111
1112 case GLSL_TYPE_ARRAY:
1113 assert(type->stride > 0);
1114 assert(glsl_get_length(type->type) > 0);
1115 return type->stride * glsl_get_length(type->type);
1116
1117 default:
1118 assert(!"Invalid block type");
1119 return 0;
1120 }
1121 }
1122
1123 void
1124 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1125 const uint32_t *w, unsigned count)
1126 {
1127 switch (opcode) {
1128 case SpvOpVariable: {
1129 struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1130 var->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1131
1132 var->chain.var = var;
1133 var->chain.length = 0;
1134
1135 struct vtn_value *val =
1136 vtn_push_value(b, w[2], vtn_value_type_access_chain);
1137 val->access_chain = &var->chain;
1138
1139 struct vtn_type *without_array = var->type;
1140 while(glsl_type_is_array(without_array->type))
1141 without_array = without_array->array_element;
1142
1143 nir_variable_mode nir_mode;
1144 switch ((SpvStorageClass)w[3]) {
1145 case SpvStorageClassUniform:
1146 case SpvStorageClassUniformConstant:
1147 if (without_array->block) {
1148 var->mode = vtn_variable_mode_ubo;
1149 b->shader->info.num_ubos++;
1150 } else if (without_array->buffer_block) {
1151 var->mode = vtn_variable_mode_ssbo;
1152 b->shader->info.num_ssbos++;
1153 } else if (glsl_type_is_image(without_array->type)) {
1154 var->mode = vtn_variable_mode_image;
1155 nir_mode = nir_var_uniform;
1156 b->shader->info.num_images++;
1157 } else if (glsl_type_is_sampler(without_array->type)) {
1158 var->mode = vtn_variable_mode_sampler;
1159 nir_mode = nir_var_uniform;
1160 b->shader->info.num_textures++;
1161 } else {
1162 assert(!"Invalid uniform variable type");
1163 }
1164 break;
1165 case SpvStorageClassPushConstant:
1166 var->mode = vtn_variable_mode_push_constant;
1167 assert(b->shader->num_uniforms == 0);
1168 b->shader->num_uniforms = vtn_type_block_size(var->type) * 4;
1169 break;
1170 case SpvStorageClassInput:
1171 var->mode = vtn_variable_mode_input;
1172 nir_mode = nir_var_shader_in;
1173 break;
1174 case SpvStorageClassOutput:
1175 var->mode = vtn_variable_mode_output;
1176 nir_mode = nir_var_shader_out;
1177 break;
1178 case SpvStorageClassPrivate:
1179 var->mode = vtn_variable_mode_global;
1180 nir_mode = nir_var_global;
1181 break;
1182 case SpvStorageClassFunction:
1183 var->mode = vtn_variable_mode_local;
1184 nir_mode = nir_var_local;
1185 break;
1186 case SpvStorageClassWorkgroup:
1187 var->mode = vtn_variable_mode_workgroup;
1188 nir_mode = nir_var_shared;
1189 break;
1190 case SpvStorageClassCrossWorkgroup:
1191 case SpvStorageClassGeneric:
1192 case SpvStorageClassAtomicCounter:
1193 default:
1194 unreachable("Unhandled variable storage class");
1195 }
1196
1197 switch (var->mode) {
1198 case vtn_variable_mode_local:
1199 case vtn_variable_mode_global:
1200 case vtn_variable_mode_image:
1201 case vtn_variable_mode_sampler:
1202 case vtn_variable_mode_workgroup:
1203 /* For these, we create the variable normally */
1204 var->var = rzalloc(b->shader, nir_variable);
1205 var->var->name = ralloc_strdup(var->var, val->name);
1206 var->var->type = var->type->type;
1207 var->var->data.mode = nir_mode;
1208
1209 switch (var->mode) {
1210 case vtn_variable_mode_image:
1211 case vtn_variable_mode_sampler:
1212 var->var->interface_type = without_array->type;
1213 break;
1214 default:
1215 var->var->interface_type = NULL;
1216 break;
1217 }
1218 break;
1219
1220 case vtn_variable_mode_input:
1221 case vtn_variable_mode_output: {
1222 /* For inputs and outputs, we immediately split structures. This
1223 * is for a couple of reasons. For one, builtins may all come in
1224 * a struct and we really want those split out into separate
1225 * variables. For another, interpolation qualifiers can be
1226 * applied to members of the top-level struct ane we need to be
1227 * able to preserve that information.
1228 */
1229
1230 int array_length = -1;
1231 struct vtn_type *interface_type = var->type;
1232 if (b->shader->stage == MESA_SHADER_GEOMETRY &&
1233 glsl_type_is_array(var->type->type)) {
1234 /* In Geometry shaders (and some tessellation), inputs come
1235 * in per-vertex arrays. However, some builtins come in
1236 * non-per-vertex, hence the need for the is_array check. In
1237 * any case, there are no non-builtin arrays allowed so this
1238 * check should be sufficient.
1239 */
1240 interface_type = var->type->array_element;
1241 array_length = glsl_get_length(var->type->type);
1242 }
1243
1244 if (glsl_type_is_struct(interface_type->type)) {
1245 /* It's a struct. Split it. */
1246 unsigned num_members = glsl_get_length(interface_type->type);
1247 var->members = ralloc_array(b, nir_variable *, num_members);
1248
1249 for (unsigned i = 0; i < num_members; i++) {
1250 const struct glsl_type *mtype = interface_type->members[i]->type;
1251 if (array_length >= 0)
1252 mtype = glsl_array_type(mtype, array_length);
1253
1254 var->members[i] = rzalloc(b->shader, nir_variable);
1255 var->members[i]->name =
1256 ralloc_asprintf(var->members[i], "%s.%d", val->name, i);
1257 var->members[i]->type = mtype;
1258 var->members[i]->interface_type =
1259 interface_type->members[i]->type;
1260 var->members[i]->data.mode = nir_mode;
1261 }
1262 } else {
1263 var->var = rzalloc(b->shader, nir_variable);
1264 var->var->name = ralloc_strdup(var->var, val->name);
1265 var->var->type = var->type->type;
1266 var->var->interface_type = interface_type->type;
1267 var->var->data.mode = nir_mode;
1268 }
1269
1270 /* For inputs and outputs, we need to grab locations and builtin
1271 * information from the interface type.
1272 */
1273 vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var);
1274 break;
1275
1276 case vtn_variable_mode_param:
1277 unreachable("Not created through OpVariable");
1278 }
1279
1280 case vtn_variable_mode_ubo:
1281 case vtn_variable_mode_ssbo:
1282 case vtn_variable_mode_push_constant:
1283 /* These don't need actual variables. */
1284 break;
1285 }
1286
1287 if (count > 4) {
1288 assert(count == 5);
1289 nir_constant *constant =
1290 vtn_value(b, w[4], vtn_value_type_constant)->constant;
1291 var->var->constant_initializer =
1292 nir_constant_clone(constant, var->var);
1293 }
1294
1295 vtn_foreach_decoration(b, val, var_decoration_cb, var);
1296
1297 if (var->mode == vtn_variable_mode_image ||
1298 var->mode == vtn_variable_mode_sampler) {
1299 /* XXX: We still need the binding information in the nir_variable
1300 * for these. We should fix that.
1301 */
1302 var->var->data.binding = var->binding;
1303 var->var->data.descriptor_set = var->descriptor_set;
1304
1305 if (var->mode == vtn_variable_mode_image)
1306 var->var->data.image.format = without_array->image_format;
1307 }
1308
1309 if (var->mode == vtn_variable_mode_local) {
1310 assert(var->members == NULL && var->var != NULL);
1311 nir_function_impl_add_variable(b->impl, var->var);
1312 } else if (var->var) {
1313 nir_shader_add_variable(b->shader, var->var);
1314 } else if (var->members) {
1315 unsigned count = glsl_get_length(without_array->type);
1316 for (unsigned i = 0; i < count; i++) {
1317 assert(var->members[i]->data.mode != nir_var_local);
1318 nir_shader_add_variable(b->shader, var->members[i]);
1319 }
1320 } else {
1321 assert(var->mode == vtn_variable_mode_ubo ||
1322 var->mode == vtn_variable_mode_ssbo ||
1323 var->mode == vtn_variable_mode_push_constant);
1324 }
1325 break;
1326 }
1327
1328 case SpvOpAccessChain:
1329 case SpvOpInBoundsAccessChain: {
1330 struct vtn_access_chain *base, *chain;
1331 struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1332 if (base_val->value_type == vtn_value_type_sampled_image) {
1333 /* This is rather insane. SPIR-V allows you to use OpSampledImage
1334 * to combine an array of images with a single sampler to get an
1335 * array of sampled images that all share the same sampler.
1336 * Fortunately, this means that we can more-or-less ignore the
1337 * sampler when crawling the access chain, but it does leave us
1338 * with this rather awkward little special-case.
1339 */
1340 base = base_val->sampled_image->image;
1341 } else {
1342 assert(base_val->value_type == vtn_value_type_access_chain);
1343 base = base_val->access_chain;
1344 }
1345
1346 chain = vtn_access_chain_extend(b, base, count - 4);
1347
1348 unsigned idx = base->length;
1349 for (int i = 4; i < count; i++) {
1350 struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1351 if (link_val->value_type == vtn_value_type_constant) {
1352 chain->link[idx].mode = vtn_access_mode_literal;
1353 chain->link[idx].id = link_val->constant->value.u[0];
1354 } else {
1355 chain->link[idx].mode = vtn_access_mode_id;
1356 chain->link[idx].id = w[i];
1357 }
1358 idx++;
1359 }
1360
1361 if (base_val->value_type == vtn_value_type_sampled_image) {
1362 struct vtn_value *val =
1363 vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1364 val->sampled_image = ralloc(b, struct vtn_sampled_image);
1365 val->sampled_image->image = chain;
1366 val->sampled_image->sampler = base_val->sampled_image->sampler;
1367 } else {
1368 struct vtn_value *val =
1369 vtn_push_value(b, w[2], vtn_value_type_access_chain);
1370 val->access_chain = chain;
1371 }
1372 break;
1373 }
1374
1375 case SpvOpCopyMemory: {
1376 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_access_chain);
1377 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_access_chain);
1378
1379 vtn_variable_copy(b, dest->access_chain, src->access_chain);
1380 break;
1381 }
1382
1383 case SpvOpLoad: {
1384 struct vtn_access_chain *src =
1385 vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1386
1387 if (src->var->mode == vtn_variable_mode_image ||
1388 src->var->mode == vtn_variable_mode_sampler) {
1389 vtn_push_value(b, w[2], vtn_value_type_access_chain)->access_chain = src;
1390 return;
1391 }
1392
1393 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1394 val->ssa = vtn_variable_load(b, src);
1395 break;
1396 }
1397
1398 case SpvOpStore: {
1399 struct vtn_access_chain *dest =
1400 vtn_value(b, w[1], vtn_value_type_access_chain)->access_chain;
1401 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1402 vtn_variable_store(b, src, dest);
1403 break;
1404 }
1405
1406 case SpvOpArrayLength: {
1407 struct vtn_access_chain *chain =
1408 vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1409
1410 const uint32_t offset = chain->var->type->offsets[w[4]];
1411 const uint32_t stride = chain->var->type->members[w[4]]->stride;
1412
1413 unsigned chain_idx;
1414 struct vtn_type *type;
1415 nir_ssa_def *index =
1416 get_vulkan_resource_index(b, chain, &type, &chain_idx);
1417
1418 nir_intrinsic_instr *instr =
1419 nir_intrinsic_instr_create(b->nb.shader,
1420 nir_intrinsic_get_buffer_size);
1421 instr->src[0] = nir_src_for_ssa(index);
1422 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1423 nir_builder_instr_insert(&b->nb, &instr->instr);
1424 nir_ssa_def *buf_size = &instr->dest.ssa;
1425
1426 /* array_length = max(buffer_size - offset, 0) / stride */
1427 nir_ssa_def *array_length =
1428 nir_idiv(&b->nb,
1429 nir_imax(&b->nb,
1430 nir_isub(&b->nb,
1431 buf_size,
1432 nir_imm_int(&b->nb, offset)),
1433 nir_imm_int(&b->nb, 0u)),
1434 nir_imm_int(&b->nb, stride));
1435
1436 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1437 val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
1438 val->ssa->def = array_length;
1439 break;
1440 }
1441
1442 case SpvOpCopyMemorySized:
1443 default:
1444 unreachable("Unhandled opcode");
1445 }
1446 }