src/compiler/spirv/vtn_variables.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 #include "vtn_private.h"
  29 #include "spirv_info.h"
  30 #include "nir_deref.h"
  31
  32 static struct vtn_access_chain *
  33 vtn_access_chain_create(struct vtn_builder *b, unsigned length)
  34 {
  35    struct vtn_access_chain *chain;
  36
  37    /* Subtract 1 from the length since there's already one built in */
  38    size_t size = sizeof(*chain) +
  39                  (MAX2(length, 1) - 1) * sizeof(chain->link[0]);
  40    chain = rzalloc_size(b, size);
  41    chain->length = length;
  42
  43    return chain;
  44 }
  45
  46 static struct vtn_access_chain *
  47 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
  48                         unsigned new_ids)
  49 {
  50    struct vtn_access_chain *chain;
  51
  52    unsigned old_len = old ? old->length : 0;
  53    chain = vtn_access_chain_create(b, old_len + new_ids);
  54
  55    for (unsigned i = 0; i < old_len; i++)
  56       chain->link[i] = old->link[i];
  57
  58    return chain;
  59 }
  60
  61 static bool
  62 vtn_pointer_uses_ssa_offset(struct vtn_builder *b,
  63                             struct vtn_pointer *ptr)
  64 {
  65    return ptr->mode == vtn_variable_mode_ubo ||
  66           ptr->mode == vtn_variable_mode_ssbo ||
  67           ptr->mode == vtn_variable_mode_push_constant ||
  68           (ptr->mode == vtn_variable_mode_workgroup &&
  69            b->options->lower_workgroup_access_to_offsets);
  70 }
  71
  72 static bool
  73 vtn_pointer_is_external_block(struct vtn_builder *b,
  74                               struct vtn_pointer *ptr)
  75 {
  76    return ptr->mode == vtn_variable_mode_ssbo ||
  77           ptr->mode == vtn_variable_mode_ubo ||
  78           ptr->mode == vtn_variable_mode_push_constant ||
  79           (ptr->mode == vtn_variable_mode_workgroup &&
  80            b->options->lower_workgroup_access_to_offsets);
  81 }
  82
  83 /* Dereference the given base pointer by the access chain */
  84 static struct vtn_pointer *
  85 vtn_access_chain_pointer_dereference(struct vtn_builder *b,
  86                                      struct vtn_pointer *base,
  87                                      struct vtn_access_chain *deref_chain)
  88 {
  89    struct vtn_access_chain *chain =
  90       vtn_access_chain_extend(b, base->chain, deref_chain->length);
  91    struct vtn_type *type = base->type;
  92
  93    /* OpPtrAccessChain is only allowed on things which support variable
  94     * pointers.  For everything else, the client is expected to just pass us
  95     * the right access chain.
  96     */
  97    vtn_assert(!deref_chain->ptr_as_array);
  98
  99    unsigned start = base->chain ? base->chain->length : 0;
 100    for (unsigned i = 0; i < deref_chain->length; i++) {
 101       chain->link[start + i] = deref_chain->link[i];
 102
 103       if (glsl_type_is_struct(type->type)) {
 104          vtn_assert(deref_chain->link[i].mode == vtn_access_mode_literal);
 105          type = type->members[deref_chain->link[i].id];
 106       } else {
 107          type = type->array_element;
 108       }
 109    }
 110
 111    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
 112    ptr->mode = base->mode;
 113    ptr->type = type;
 114    ptr->var = base->var;
 115    ptr->deref = base->deref;
 116    ptr->chain = chain;
 117
 118    return ptr;
 119 }
 120
 121 static nir_ssa_def *
 122 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
 123                        unsigned stride)
 124 {
 125    vtn_assert(stride > 0);
 126    if (link.mode == vtn_access_mode_literal) {
 127       return nir_imm_int(&b->nb, link.id * stride);
 128    } else if (stride == 1) {
 129        nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def;
 130        if (ssa->bit_size != 32)
 131           ssa = nir_u2u32(&b->nb, ssa);
 132       return ssa;
 133    } else {
 134       nir_ssa_def *src0 = vtn_ssa_value(b, link.id)->def;
 135       if (src0->bit_size != 32)
 136          src0 = nir_u2u32(&b->nb, src0);
 137       return nir_imul(&b->nb, src0, nir_imm_int(&b->nb, stride));
 138    }
 139 }
 140
 141 static nir_ssa_def *
 142 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var,
 143                             nir_ssa_def *desc_array_index)
 144 {
 145    if (!desc_array_index) {
 146       vtn_assert(glsl_type_is_struct(var->type->type));
 147       desc_array_index = nir_imm_int(&b->nb, 0);
 148    }
 149
 150    nir_intrinsic_instr *instr =
 151       nir_intrinsic_instr_create(b->nb.shader,
 152                                  nir_intrinsic_vulkan_resource_index);
 153    instr->src[0] = nir_src_for_ssa(desc_array_index);
 154    nir_intrinsic_set_desc_set(instr, var->descriptor_set);
 155    nir_intrinsic_set_binding(instr, var->binding);
 156
 157    nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 158    nir_builder_instr_insert(&b->nb, &instr->instr);
 159
 160    return &instr->dest.ssa;
 161 }
 162
 163 static nir_ssa_def *
 164 vtn_resource_reindex(struct vtn_builder *b, nir_ssa_def *base_index,
 165                      nir_ssa_def *offset_index)
 166 {
 167    nir_intrinsic_instr *instr =
 168       nir_intrinsic_instr_create(b->nb.shader,
 169                                  nir_intrinsic_vulkan_resource_reindex);
 170    instr->src[0] = nir_src_for_ssa(base_index);
 171    instr->src[1] = nir_src_for_ssa(offset_index);
 172
 173    nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 174    nir_builder_instr_insert(&b->nb, &instr->instr);
 175
 176    return &instr->dest.ssa;
 177 }
 178
 179 static struct vtn_pointer *
 180 vtn_ssa_offset_pointer_dereference(struct vtn_builder *b,
 181                                    struct vtn_pointer *base,
 182                                    struct vtn_access_chain *deref_chain)
 183 {
 184    nir_ssa_def *block_index = base->block_index;
 185    nir_ssa_def *offset = base->offset;
 186    struct vtn_type *type = base->type;
 187
 188    unsigned idx = 0;
 189    if (base->mode == vtn_variable_mode_ubo ||
 190        base->mode == vtn_variable_mode_ssbo) {
 191       if (!block_index) {
 192          vtn_assert(base->var && base->type);
 193          nir_ssa_def *desc_arr_idx;
 194          if (glsl_type_is_array(type->type)) {
 195             if (deref_chain->length >= 1) {
 196                desc_arr_idx =
 197                   vtn_access_link_as_ssa(b, deref_chain->link[0], 1);
 198                idx++;
 199                /* This consumes a level of type */
 200                type = type->array_element;
 201             } else {
 202                /* This is annoying.  We've been asked for a pointer to the
 203                 * array of UBOs/SSBOs and not a specifc buffer.  Return a
 204                 * pointer with a descriptor index of 0 and we'll have to do
 205                 * a reindex later to adjust it to the right thing.
 206                 */
 207                desc_arr_idx = nir_imm_int(&b->nb, 0);
 208             }
 209          } else if (deref_chain->ptr_as_array) {
 210             /* You can't have a zero-length OpPtrAccessChain */
 211             vtn_assert(deref_chain->length >= 1);
 212             desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[0], 1);
 213          } else {
 214             /* We have a regular non-array SSBO. */
 215             desc_arr_idx = NULL;
 216          }
 217          block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
 218       } else if (deref_chain->ptr_as_array &&
 219                  type->base_type == vtn_base_type_struct && type->block) {
 220          /* We are doing an OpPtrAccessChain on a pointer to a struct that is
 221           * decorated block.  This is an interesting corner in the SPIR-V
 222           * spec.  One interpretation would be that they client is clearly
 223           * trying to treat that block as if it's an implicit array of blocks
 224           * repeated in the buffer.  However, the SPIR-V spec for the
 225           * OpPtrAccessChain says:
 226           *
 227           *    "Base is treated as the address of the first element of an
 228           *    array, and the Element element’s address is computed to be the
 229           *    base for the Indexes, as per OpAccessChain."
 230           *
 231           * Taken literally, that would mean that your struct type is supposed
 232           * to be treated as an array of such a struct and, since it's
 233           * decorated block, that means an array of blocks which corresponds
 234           * to an array descriptor.  Therefore, we need to do a reindex
 235           * operation to add the index from the first link in the access chain
 236           * to the index we recieved.
 237           *
 238           * The downside to this interpretation (there always is one) is that
 239           * this might be somewhat surprising behavior to apps if they expect
 240           * the implicit array behavior described above.
 241           */
 242          vtn_assert(deref_chain->length >= 1);
 243          nir_ssa_def *offset_index =
 244             vtn_access_link_as_ssa(b, deref_chain->link[0], 1);
 245          idx++;
 246
 247          block_index = vtn_resource_reindex(b, block_index, offset_index);
 248       }
 249    }
 250
 251    if (!offset) {
 252       if (base->mode == vtn_variable_mode_workgroup) {
 253          /* SLM doesn't need nor have a block index */
 254          vtn_assert(!block_index);
 255
 256          /* We need the variable for the base offset */
 257          vtn_assert(base->var);
 258
 259          /* We need ptr_type for size and alignment */
 260          vtn_assert(base->ptr_type);
 261
 262          /* Assign location on first use so that we don't end up bloating SLM
 263           * address space for variables which are never statically used.
 264           */
 265          if (base->var->shared_location < 0) {
 266             vtn_assert(base->ptr_type->length > 0 && base->ptr_type->align > 0);
 267             b->shader->num_shared = vtn_align_u32(b->shader->num_shared,
 268                                                   base->ptr_type->align);
 269             base->var->shared_location = b->shader->num_shared;
 270             b->shader->num_shared += base->ptr_type->length;
 271          }
 272
 273          offset = nir_imm_int(&b->nb, base->var->shared_location);
 274       } else if (base->mode == vtn_variable_mode_push_constant) {
 275          /* Push constants neither need nor have a block index */
 276          vtn_assert(!block_index);
 277
 278          /* Start off with at the start of the push constant block. */
 279          offset = nir_imm_int(&b->nb, 0);
 280       } else {
 281          /* The code above should have ensured a block_index when needed. */
 282          vtn_assert(block_index);
 283
 284          /* Start off with at the start of the buffer. */
 285          offset = nir_imm_int(&b->nb, 0);
 286       }
 287    }
 288
 289    if (deref_chain->ptr_as_array && idx == 0) {
 290       /* We need ptr_type for the stride */
 291       vtn_assert(base->ptr_type);
 292
 293       /* We need at least one element in the chain */
 294       vtn_assert(deref_chain->length >= 1);
 295
 296       nir_ssa_def *elem_offset =
 297          vtn_access_link_as_ssa(b, deref_chain->link[idx],
 298                                 base->ptr_type->stride);
 299       offset = nir_iadd(&b->nb, offset, elem_offset);
 300       idx++;
 301    }
 302
 303    for (; idx < deref_chain->length; idx++) {
 304       switch (glsl_get_base_type(type->type)) {
 305       case GLSL_TYPE_UINT:
 306       case GLSL_TYPE_INT:
 307       case GLSL_TYPE_UINT16:
 308       case GLSL_TYPE_INT16:
 309       case GLSL_TYPE_UINT8:
 310       case GLSL_TYPE_INT8:
 311       case GLSL_TYPE_UINT64:
 312       case GLSL_TYPE_INT64:
 313       case GLSL_TYPE_FLOAT:
 314       case GLSL_TYPE_FLOAT16:
 315       case GLSL_TYPE_DOUBLE:
 316       case GLSL_TYPE_BOOL:
 317       case GLSL_TYPE_ARRAY: {
 318          nir_ssa_def *elem_offset =
 319             vtn_access_link_as_ssa(b, deref_chain->link[idx], type->stride);
 320          offset = nir_iadd(&b->nb, offset, elem_offset);
 321          type = type->array_element;
 322          break;
 323       }
 324
 325       case GLSL_TYPE_STRUCT: {
 326          vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
 327          unsigned member = deref_chain->link[idx].id;
 328          nir_ssa_def *mem_offset = nir_imm_int(&b->nb, type->offsets[member]);
 329          offset = nir_iadd(&b->nb, offset, mem_offset);
 330          type = type->members[member];
 331          break;
 332       }
 333
 334       default:
 335          vtn_fail("Invalid type for deref");
 336       }
 337    }
 338
 339    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
 340    ptr->mode = base->mode;
 341    ptr->type = type;
 342    ptr->block_index = block_index;
 343    ptr->offset = offset;
 344
 345    return ptr;
 346 }
 347
 348 /* Dereference the given base pointer by the access chain */
 349 static struct vtn_pointer *
 350 vtn_pointer_dereference(struct vtn_builder *b,
 351                         struct vtn_pointer *base,
 352                         struct vtn_access_chain *deref_chain)
 353 {
 354    if (vtn_pointer_uses_ssa_offset(b, base)) {
 355       return vtn_ssa_offset_pointer_dereference(b, base, deref_chain);
 356    } else {
 357       return vtn_access_chain_pointer_dereference(b, base, deref_chain);
 358    }
 359 }
 360
 361 struct vtn_pointer *
 362 vtn_pointer_for_variable(struct vtn_builder *b,
 363                          struct vtn_variable *var, struct vtn_type *ptr_type)
 364 {
 365    struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer);
 366
 367    pointer->mode = var->mode;
 368    pointer->type = var->type;
 369    vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
 370    vtn_assert(ptr_type->deref->type == var->type->type);
 371    pointer->ptr_type = ptr_type;
 372    pointer->var = var;
 373
 374    return pointer;
 375 }
 376
 377 /* Returns an atomic_uint type based on the original uint type. The returned
 378  * type will be equivalent to the original one but will have an atomic_uint
 379  * type as leaf instead of an uint.
 380  *
 381  * Manages uint scalars, arrays, and arrays of arrays of any nested depth.
 382  */
 383 static const struct glsl_type *
 384 repair_atomic_type(const struct glsl_type *type)
 385 {
 386    assert(glsl_get_base_type(glsl_without_array(type)) == GLSL_TYPE_UINT);
 387    assert(glsl_type_is_scalar(glsl_without_array(type)));
 388
 389    if (glsl_type_is_array(type)) {
 390       const struct glsl_type *atomic =
 391          repair_atomic_type(glsl_get_array_element(type));
 392
 393       return glsl_array_type(atomic, glsl_get_length(type));
 394    } else {
 395       return glsl_atomic_uint_type();
 396    }
 397 }
 398
 399 nir_deref_instr *
 400 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr)
 401 {
 402    /* Do on-the-fly copy propagation for samplers. */
 403    if (ptr->var && ptr->var->copy_prop_sampler)
 404       return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler);
 405
 406    nir_deref_instr *tail;
 407    if (ptr->deref) {
 408       tail = ptr->deref;
 409    } else {
 410       assert(ptr->var && ptr->var->var);
 411       tail = nir_build_deref_var(&b->nb, ptr->var->var);
 412    }
 413
 414    /* Raw variable access */
 415    if (!ptr->chain)
 416       return tail;
 417
 418    struct vtn_access_chain *chain = ptr->chain;
 419    vtn_assert(chain);
 420
 421    for (unsigned i = 0; i < chain->length; i++) {
 422       if (glsl_type_is_struct(tail->type)) {
 423          vtn_assert(chain->link[i].mode == vtn_access_mode_literal);
 424          unsigned idx = chain->link[i].id;
 425          tail = nir_build_deref_struct(&b->nb, tail, idx);
 426       } else {
 427          nir_ssa_def *index;
 428          if (chain->link[i].mode == vtn_access_mode_literal) {
 429             index = nir_imm_int(&b->nb, chain->link[i].id);
 430          } else {
 431             vtn_assert(chain->link[i].mode == vtn_access_mode_id);
 432             index = vtn_ssa_value(b, chain->link[i].id)->def;
 433          }
 434          tail = nir_build_deref_array(&b->nb, tail, index);
 435       }
 436    }
 437
 438    return tail;
 439 }
 440
 441 static void
 442 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref,
 443                       struct vtn_ssa_value *inout)
 444 {
 445    if (glsl_type_is_vector_or_scalar(deref->type)) {
 446       if (load) {
 447          inout->def = nir_load_deref(&b->nb, deref);
 448       } else {
 449          nir_store_deref(&b->nb, deref, inout->def, ~0);
 450       }
 451    } else if (glsl_type_is_array(deref->type) ||
 452               glsl_type_is_matrix(deref->type)) {
 453       unsigned elems = glsl_get_length(deref->type);
 454       for (unsigned i = 0; i < elems; i++) {
 455          nir_deref_instr *child =
 456             nir_build_deref_array(&b->nb, deref, nir_imm_int(&b->nb, i));
 457          _vtn_local_load_store(b, load, child, inout->elems[i]);
 458       }
 459    } else {
 460       vtn_assert(glsl_type_is_struct(deref->type));
 461       unsigned elems = glsl_get_length(deref->type);
 462       for (unsigned i = 0; i < elems; i++) {
 463          nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i);
 464          _vtn_local_load_store(b, load, child, inout->elems[i]);
 465       }
 466    }
 467 }
 468
 469 nir_deref_instr *
 470 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
 471 {
 472    struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer;
 473    return vtn_pointer_to_deref(b, ptr);
 474 }
 475
 476 /*
 477  * Gets the NIR-level deref tail, which may have as a child an array deref
 478  * selecting which component due to OpAccessChain supporting per-component
 479  * indexing in SPIR-V.
 480  */
 481 static nir_deref_instr *
 482 get_deref_tail(nir_deref_instr *deref)
 483 {
 484    if (deref->deref_type != nir_deref_type_array)
 485       return deref;
 486
 487    nir_deref_instr *parent =
 488       nir_instr_as_deref(deref->parent.ssa->parent_instr);
 489
 490    if (glsl_type_is_vector(parent->type))
 491       return parent;
 492    else
 493       return deref;
 494 }
 495
 496 struct vtn_ssa_value *
 497 vtn_local_load(struct vtn_builder *b, nir_deref_instr *src)
 498 {
 499    nir_deref_instr *src_tail = get_deref_tail(src);
 500    struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
 501    _vtn_local_load_store(b, true, src_tail, val);
 502
 503    if (src_tail != src) {
 504       val->type = src->type;
 505       nir_const_value *const_index = nir_src_as_const_value(src->arr.index);
 506       if (const_index)
 507          val->def = vtn_vector_extract(b, val->def, const_index->u32[0]);
 508       else
 509          val->def = vtn_vector_extract_dynamic(b, val->def, src->arr.index.ssa);
 510    }
 511
 512    return val;
 513 }
 514
 515 void
 516 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 517                 nir_deref_instr *dest)
 518 {
 519    nir_deref_instr *dest_tail = get_deref_tail(dest);
 520
 521    if (dest_tail != dest) {
 522       struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
 523       _vtn_local_load_store(b, true, dest_tail, val);
 524
 525       nir_const_value *const_index = nir_src_as_const_value(dest->arr.index);
 526       if (const_index)
 527          val->def = vtn_vector_insert(b, val->def, src->def,
 528                                       const_index->u32[0]);
 529       else
 530          val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
 531                                               dest->arr.index.ssa);
 532       _vtn_local_load_store(b, false, dest_tail, val);
 533    } else {
 534       _vtn_local_load_store(b, false, dest_tail, src);
 535    }
 536 }
 537
 538 nir_ssa_def *
 539 vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr,
 540                       nir_ssa_def **index_out)
 541 {
 542    assert(vtn_pointer_uses_ssa_offset(b, ptr));
 543    if (!ptr->offset) {
 544       struct vtn_access_chain chain = {
 545          .length = 0,
 546       };
 547       ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
 548    }
 549    *index_out = ptr->block_index;
 550    return ptr->offset;
 551 }
 552
 553 /* Tries to compute the size of an interface block based on the strides and
 554  * offsets that are provided to us in the SPIR-V source.
 555  */
 556 static unsigned
 557 vtn_type_block_size(struct vtn_builder *b, struct vtn_type *type)
 558 {
 559    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 560    switch (base_type) {
 561    case GLSL_TYPE_UINT:
 562    case GLSL_TYPE_INT:
 563    case GLSL_TYPE_UINT16:
 564    case GLSL_TYPE_INT16:
 565    case GLSL_TYPE_UINT8:
 566    case GLSL_TYPE_INT8:
 567    case GLSL_TYPE_UINT64:
 568    case GLSL_TYPE_INT64:
 569    case GLSL_TYPE_FLOAT:
 570    case GLSL_TYPE_FLOAT16:
 571    case GLSL_TYPE_BOOL:
 572    case GLSL_TYPE_DOUBLE: {
 573       unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
 574                                         glsl_get_matrix_columns(type->type);
 575       if (cols > 1) {
 576          vtn_assert(type->stride > 0);
 577          return type->stride * cols;
 578       } else {
 579          unsigned type_size = glsl_get_bit_size(type->type) / 8;
 580          return glsl_get_vector_elements(type->type) * type_size;
 581       }
 582    }
 583
 584    case GLSL_TYPE_STRUCT:
 585    case GLSL_TYPE_INTERFACE: {
 586       unsigned size = 0;
 587       unsigned num_fields = glsl_get_length(type->type);
 588       for (unsigned f = 0; f < num_fields; f++) {
 589          unsigned field_end = type->offsets[f] +
 590                               vtn_type_block_size(b, type->members[f]);
 591          size = MAX2(size, field_end);
 592       }
 593       return size;
 594    }
 595
 596    case GLSL_TYPE_ARRAY:
 597       vtn_assert(type->stride > 0);
 598       vtn_assert(glsl_get_length(type->type) > 0);
 599       return type->stride * glsl_get_length(type->type);
 600
 601    default:
 602       vtn_fail("Invalid block type");
 603       return 0;
 604    }
 605 }
 606
 607 static void
 608 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 609                      nir_ssa_def *index, nir_ssa_def *offset,
 610                      unsigned access_offset, unsigned access_size,
 611                      struct vtn_ssa_value **inout, const struct glsl_type *type)
 612 {
 613    nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
 614    instr->num_components = glsl_get_vector_elements(type);
 615
 616    int src = 0;
 617    if (!load) {
 618       nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
 619       instr->src[src++] = nir_src_for_ssa((*inout)->def);
 620    }
 621
 622    if (op == nir_intrinsic_load_push_constant) {
 623       nir_intrinsic_set_base(instr, access_offset);
 624       nir_intrinsic_set_range(instr, access_size);
 625    }
 626
 627    if (index)
 628       instr->src[src++] = nir_src_for_ssa(index);
 629
 630    if (op == nir_intrinsic_load_push_constant) {
 631       /* We need to subtract the offset from where the intrinsic will load the
 632        * data. */
 633       instr->src[src++] =
 634          nir_src_for_ssa(nir_isub(&b->nb, offset,
 635                                   nir_imm_int(&b->nb, access_offset)));
 636    } else {
 637       instr->src[src++] = nir_src_for_ssa(offset);
 638    }
 639
 640    if (load) {
 641       nir_ssa_dest_init(&instr->instr, &instr->dest,
 642                         instr->num_components,
 643                         glsl_get_bit_size(type), NULL);
 644       (*inout)->def = &instr->dest.ssa;
 645    }
 646
 647    nir_builder_instr_insert(&b->nb, &instr->instr);
 648
 649    if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
 650       (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
 651 }
 652
 653 static void
 654 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 655                       nir_ssa_def *index, nir_ssa_def *offset,
 656                       unsigned access_offset, unsigned access_size,
 657                       struct vtn_type *type, struct vtn_ssa_value **inout)
 658 {
 659    if (load && *inout == NULL)
 660       *inout = vtn_create_ssa_value(b, type->type);
 661
 662    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 663    switch (base_type) {
 664    case GLSL_TYPE_UINT:
 665    case GLSL_TYPE_INT:
 666    case GLSL_TYPE_UINT16:
 667    case GLSL_TYPE_INT16:
 668    case GLSL_TYPE_UINT8:
 669    case GLSL_TYPE_INT8:
 670    case GLSL_TYPE_UINT64:
 671    case GLSL_TYPE_INT64:
 672    case GLSL_TYPE_FLOAT:
 673    case GLSL_TYPE_FLOAT16:
 674    case GLSL_TYPE_DOUBLE:
 675    case GLSL_TYPE_BOOL:
 676       /* This is where things get interesting.  At this point, we've hit
 677        * a vector, a scalar, or a matrix.
 678        */
 679       if (glsl_type_is_matrix(type->type)) {
 680          /* Loading the whole matrix */
 681          struct vtn_ssa_value *transpose;
 682          unsigned num_ops, vec_width, col_stride;
 683          if (type->row_major) {
 684             num_ops = glsl_get_vector_elements(type->type);
 685             vec_width = glsl_get_matrix_columns(type->type);
 686             col_stride = type->array_element->stride;
 687             if (load) {
 688                const struct glsl_type *transpose_type =
 689                   glsl_matrix_type(base_type, vec_width, num_ops);
 690                *inout = vtn_create_ssa_value(b, transpose_type);
 691             } else {
 692                transpose = vtn_ssa_transpose(b, *inout);
 693                inout = &transpose;
 694             }
 695          } else {
 696             num_ops = glsl_get_matrix_columns(type->type);
 697             vec_width = glsl_get_vector_elements(type->type);
 698             col_stride = type->stride;
 699          }
 700
 701          for (unsigned i = 0; i < num_ops; i++) {
 702             nir_ssa_def *elem_offset =
 703                nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * col_stride));
 704             _vtn_load_store_tail(b, op, load, index, elem_offset,
 705                                  access_offset, access_size,
 706                                  &(*inout)->elems[i],
 707                                  glsl_vector_type(base_type, vec_width));
 708          }
 709
 710          if (load && type->row_major)
 711             *inout = vtn_ssa_transpose(b, *inout);
 712       } else {
 713          unsigned elems = glsl_get_vector_elements(type->type);
 714          unsigned type_size = glsl_get_bit_size(type->type) / 8;
 715          if (elems == 1 || type->stride == type_size) {
 716             /* This is a tightly-packed normal scalar or vector load */
 717             vtn_assert(glsl_type_is_vector_or_scalar(type->type));
 718             _vtn_load_store_tail(b, op, load, index, offset,
 719                                  access_offset, access_size,
 720                                  inout, type->type);
 721          } else {
 722             /* This is a strided load.  We have to load N things separately.
 723              * This is the single column of a row-major matrix case.
 724              */
 725             vtn_assert(type->stride > type_size);
 726             vtn_assert(type->stride % type_size == 0);
 727
 728             nir_ssa_def *per_comp[4];
 729             for (unsigned i = 0; i < elems; i++) {
 730                nir_ssa_def *elem_offset =
 731                   nir_iadd(&b->nb, offset,
 732                                    nir_imm_int(&b->nb, i * type->stride));
 733                struct vtn_ssa_value *comp, temp_val;
 734                if (!load) {
 735                   temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
 736                   temp_val.type = glsl_scalar_type(base_type);
 737                }
 738                comp = &temp_val;
 739                _vtn_load_store_tail(b, op, load, index, elem_offset,
 740                                     access_offset, access_size,
 741                                     &comp, glsl_scalar_type(base_type));
 742                per_comp[i] = comp->def;
 743             }
 744
 745             if (load) {
 746                if (*inout == NULL)
 747                   *inout = vtn_create_ssa_value(b, type->type);
 748                (*inout)->def = nir_vec(&b->nb, per_comp, elems);
 749             }
 750          }
 751       }
 752       return;
 753
 754    case GLSL_TYPE_ARRAY: {
 755       unsigned elems = glsl_get_length(type->type);
 756       for (unsigned i = 0; i < elems; i++) {
 757          nir_ssa_def *elem_off =
 758             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
 759          _vtn_block_load_store(b, op, load, index, elem_off,
 760                                access_offset, access_size,
 761                                type->array_element, &(*inout)->elems[i]);
 762       }
 763       return;
 764    }
 765
 766    case GLSL_TYPE_STRUCT: {
 767       unsigned elems = glsl_get_length(type->type);
 768       for (unsigned i = 0; i < elems; i++) {
 769          nir_ssa_def *elem_off =
 770             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
 771          _vtn_block_load_store(b, op, load, index, elem_off,
 772                                access_offset, access_size,
 773                                type->members[i], &(*inout)->elems[i]);
 774       }
 775       return;
 776    }
 777
 778    default:
 779       vtn_fail("Invalid block member type");
 780    }
 781 }
 782
 783 static struct vtn_ssa_value *
 784 vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src)
 785 {
 786    nir_intrinsic_op op;
 787    unsigned access_offset = 0, access_size = 0;
 788    switch (src->mode) {
 789    case vtn_variable_mode_ubo:
 790       op = nir_intrinsic_load_ubo;
 791       break;
 792    case vtn_variable_mode_ssbo:
 793       op = nir_intrinsic_load_ssbo;
 794       break;
 795    case vtn_variable_mode_push_constant:
 796       op = nir_intrinsic_load_push_constant;
 797       access_size = b->shader->num_uniforms;
 798       break;
 799    case vtn_variable_mode_workgroup:
 800       op = nir_intrinsic_load_shared;
 801       break;
 802    default:
 803       vtn_fail("Invalid block variable mode");
 804    }
 805
 806    nir_ssa_def *offset, *index = NULL;
 807    offset = vtn_pointer_to_offset(b, src, &index);
 808
 809    struct vtn_ssa_value *value = NULL;
 810    _vtn_block_load_store(b, op, true, index, offset,
 811                          access_offset, access_size,
 812                          src->type, &value);
 813    return value;
 814 }
 815
 816 static void
 817 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 818                 struct vtn_pointer *dst)
 819 {
 820    nir_intrinsic_op op;
 821    switch (dst->mode) {
 822    case vtn_variable_mode_ssbo:
 823       op = nir_intrinsic_store_ssbo;
 824       break;
 825    case vtn_variable_mode_workgroup:
 826       op = nir_intrinsic_store_shared;
 827       break;
 828    default:
 829       vtn_fail("Invalid block variable mode");
 830    }
 831
 832    nir_ssa_def *offset, *index = NULL;
 833    offset = vtn_pointer_to_offset(b, dst, &index);
 834
 835    _vtn_block_load_store(b, op, false, index, offset,
 836                          0, 0, dst->type, &src);
 837 }
 838
 839 static void
 840 _vtn_variable_load_store(struct vtn_builder *b, bool load,
 841                          struct vtn_pointer *ptr,
 842                          struct vtn_ssa_value **inout)
 843 {
 844    enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type);
 845    switch (base_type) {
 846    case GLSL_TYPE_UINT:
 847    case GLSL_TYPE_INT:
 848    case GLSL_TYPE_UINT16:
 849    case GLSL_TYPE_INT16:
 850    case GLSL_TYPE_UINT8:
 851    case GLSL_TYPE_INT8:
 852    case GLSL_TYPE_UINT64:
 853    case GLSL_TYPE_INT64:
 854    case GLSL_TYPE_FLOAT:
 855    case GLSL_TYPE_FLOAT16:
 856    case GLSL_TYPE_BOOL:
 857    case GLSL_TYPE_DOUBLE:
 858       /* At this point, we have a scalar, vector, or matrix so we know that
 859        * there cannot be any structure splitting still in the way.  By
 860        * stopping at the matrix level rather than the vector level, we
 861        * ensure that matrices get loaded in the optimal way even if they
 862        * are storred row-major in a UBO.
 863        */
 864       if (load) {
 865          *inout = vtn_local_load(b, vtn_pointer_to_deref(b, ptr));
 866       } else {
 867          vtn_local_store(b, *inout, vtn_pointer_to_deref(b, ptr));
 868       }
 869       return;
 870
 871    case GLSL_TYPE_ARRAY:
 872    case GLSL_TYPE_STRUCT: {
 873       unsigned elems = glsl_get_length(ptr->type->type);
 874       if (load) {
 875          vtn_assert(*inout == NULL);
 876          *inout = rzalloc(b, struct vtn_ssa_value);
 877          (*inout)->type = ptr->type->type;
 878          (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
 879       }
 880
 881       struct vtn_access_chain chain = {
 882          .length = 1,
 883          .link = {
 884             { .mode = vtn_access_mode_literal, },
 885          }
 886       };
 887       for (unsigned i = 0; i < elems; i++) {
 888          chain.link[0].id = i;
 889          struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain);
 890          _vtn_variable_load_store(b, load, elem, &(*inout)->elems[i]);
 891       }
 892       return;
 893    }
 894
 895    default:
 896       vtn_fail("Invalid access chain type");
 897    }
 898 }
 899
 900 struct vtn_ssa_value *
 901 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src)
 902 {
 903    if (vtn_pointer_is_external_block(b, src)) {
 904       return vtn_block_load(b, src);
 905    } else {
 906       struct vtn_ssa_value *val = NULL;
 907       _vtn_variable_load_store(b, true, src, &val);
 908       return val;
 909    }
 910 }
 911
 912 void
 913 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 914                    struct vtn_pointer *dest)
 915 {
 916    if (vtn_pointer_is_external_block(b, dest)) {
 917       vtn_assert(dest->mode == vtn_variable_mode_ssbo ||
 918                  dest->mode == vtn_variable_mode_workgroup);
 919       vtn_block_store(b, src, dest);
 920    } else {
 921       _vtn_variable_load_store(b, false, dest, &src);
 922    }
 923 }
 924
 925 static void
 926 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
 927                    struct vtn_pointer *src)
 928 {
 929    vtn_assert(src->type->type == dest->type->type);
 930    enum glsl_base_type base_type = glsl_get_base_type(src->type->type);
 931    switch (base_type) {
 932    case GLSL_TYPE_UINT:
 933    case GLSL_TYPE_INT:
 934    case GLSL_TYPE_UINT16:
 935    case GLSL_TYPE_INT16:
 936    case GLSL_TYPE_UINT8:
 937    case GLSL_TYPE_INT8:
 938    case GLSL_TYPE_UINT64:
 939    case GLSL_TYPE_INT64:
 940    case GLSL_TYPE_FLOAT:
 941    case GLSL_TYPE_FLOAT16:
 942    case GLSL_TYPE_DOUBLE:
 943    case GLSL_TYPE_BOOL:
 944       /* At this point, we have a scalar, vector, or matrix so we know that
 945        * there cannot be any structure splitting still in the way.  By
 946        * stopping at the matrix level rather than the vector level, we
 947        * ensure that matrices get loaded in the optimal way even if they
 948        * are storred row-major in a UBO.
 949        */
 950       vtn_variable_store(b, vtn_variable_load(b, src), dest);
 951       return;
 952
 953    case GLSL_TYPE_ARRAY:
 954    case GLSL_TYPE_STRUCT: {
 955       struct vtn_access_chain chain = {
 956          .length = 1,
 957          .link = {
 958             { .mode = vtn_access_mode_literal, },
 959          }
 960       };
 961       unsigned elems = glsl_get_length(src->type->type);
 962       for (unsigned i = 0; i < elems; i++) {
 963          chain.link[0].id = i;
 964          struct vtn_pointer *src_elem =
 965             vtn_pointer_dereference(b, src, &chain);
 966          struct vtn_pointer *dest_elem =
 967             vtn_pointer_dereference(b, dest, &chain);
 968
 969          _vtn_variable_copy(b, dest_elem, src_elem);
 970       }
 971       return;
 972    }
 973
 974    default:
 975       vtn_fail("Invalid access chain type");
 976    }
 977 }
 978
 979 static void
 980 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
 981                   struct vtn_pointer *src)
 982 {
 983    /* TODO: At some point, we should add a special-case for when we can
 984     * just emit a copy_var intrinsic.
 985     */
 986    _vtn_variable_copy(b, dest, src);
 987 }
 988
 989 static void
 990 set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode)
 991 {
 992    vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
 993    *mode = nir_var_system_value;
 994 }
 995
 996 static void
 997 vtn_get_builtin_location(struct vtn_builder *b,
 998                          SpvBuiltIn builtin, int *location,
 999                          nir_variable_mode *mode)
1000 {
1001    switch (builtin) {
1002    case SpvBuiltInPosition:
1003       *location = VARYING_SLOT_POS;
1004       break;
1005    case SpvBuiltInPointSize:
1006       *location = VARYING_SLOT_PSIZ;
1007       break;
1008    case SpvBuiltInClipDistance:
1009       *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
1010       break;
1011    case SpvBuiltInCullDistance:
1012       *location = VARYING_SLOT_CULL_DIST0;
1013       break;
1014    case SpvBuiltInVertexIndex:
1015       *location = SYSTEM_VALUE_VERTEX_ID;
1016       set_mode_system_value(b, mode);
1017       break;
1018    case SpvBuiltInVertexId:
1019       /* Vulkan defines VertexID to be zero-based and reserves the new
1020        * builtin keyword VertexIndex to indicate the non-zero-based value.
1021        */
1022       *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
1023       set_mode_system_value(b, mode);
1024       break;
1025    case SpvBuiltInInstanceIndex:
1026       *location = SYSTEM_VALUE_INSTANCE_INDEX;
1027       set_mode_system_value(b, mode);
1028       break;
1029    case SpvBuiltInInstanceId:
1030       *location = SYSTEM_VALUE_INSTANCE_ID;
1031       set_mode_system_value(b, mode);
1032       break;
1033    case SpvBuiltInPrimitiveId:
1034       if (b->shader->info.stage == MESA_SHADER_FRAGMENT) {
1035          vtn_assert(*mode == nir_var_shader_in);
1036          *location = VARYING_SLOT_PRIMITIVE_ID;
1037       } else if (*mode == nir_var_shader_out) {
1038          *location = VARYING_SLOT_PRIMITIVE_ID;
1039       } else {
1040          *location = SYSTEM_VALUE_PRIMITIVE_ID;
1041          set_mode_system_value(b, mode);
1042       }
1043       break;
1044    case SpvBuiltInInvocationId:
1045       *location = SYSTEM_VALUE_INVOCATION_ID;
1046       set_mode_system_value(b, mode);
1047       break;
1048    case SpvBuiltInLayer:
1049       *location = VARYING_SLOT_LAYER;
1050       if (b->shader->info.stage == MESA_SHADER_FRAGMENT)
1051          *mode = nir_var_shader_in;
1052       else if (b->shader->info.stage == MESA_SHADER_GEOMETRY)
1053          *mode = nir_var_shader_out;
1054       else if (b->options && b->options->caps.shader_viewport_index_layer &&
1055                (b->shader->info.stage == MESA_SHADER_VERTEX ||
1056                 b->shader->info.stage == MESA_SHADER_TESS_EVAL))
1057          *mode = nir_var_shader_out;
1058       else
1059          vtn_fail("invalid stage for SpvBuiltInLayer");
1060       break;
1061    case SpvBuiltInViewportIndex:
1062       *location = VARYING_SLOT_VIEWPORT;
1063       if (b->shader->info.stage == MESA_SHADER_GEOMETRY)
1064          *mode = nir_var_shader_out;
1065       else if (b->options && b->options->caps.shader_viewport_index_layer &&
1066                (b->shader->info.stage == MESA_SHADER_VERTEX ||
1067                 b->shader->info.stage == MESA_SHADER_TESS_EVAL))
1068          *mode = nir_var_shader_out;
1069       else if (b->shader->info.stage == MESA_SHADER_FRAGMENT)
1070          *mode = nir_var_shader_in;
1071       else
1072          vtn_fail("invalid stage for SpvBuiltInViewportIndex");
1073       break;
1074    case SpvBuiltInTessLevelOuter:
1075       *location = VARYING_SLOT_TESS_LEVEL_OUTER;
1076       break;
1077    case SpvBuiltInTessLevelInner:
1078       *location = VARYING_SLOT_TESS_LEVEL_INNER;
1079       break;
1080    case SpvBuiltInTessCoord:
1081       *location = SYSTEM_VALUE_TESS_COORD;
1082       set_mode_system_value(b, mode);
1083       break;
1084    case SpvBuiltInPatchVertices:
1085       *location = SYSTEM_VALUE_VERTICES_IN;
1086       set_mode_system_value(b, mode);
1087       break;
1088    case SpvBuiltInFragCoord:
1089       *location = VARYING_SLOT_POS;
1090       vtn_assert(*mode == nir_var_shader_in);
1091       break;
1092    case SpvBuiltInPointCoord:
1093       *location = VARYING_SLOT_PNTC;
1094       vtn_assert(*mode == nir_var_shader_in);
1095       break;
1096    case SpvBuiltInFrontFacing:
1097       *location = SYSTEM_VALUE_FRONT_FACE;
1098       set_mode_system_value(b, mode);
1099       break;
1100    case SpvBuiltInSampleId:
1101       *location = SYSTEM_VALUE_SAMPLE_ID;
1102       set_mode_system_value(b, mode);
1103       break;
1104    case SpvBuiltInSamplePosition:
1105       *location = SYSTEM_VALUE_SAMPLE_POS;
1106       set_mode_system_value(b, mode);
1107       break;
1108    case SpvBuiltInSampleMask:
1109       if (*mode == nir_var_shader_out) {
1110          *location = FRAG_RESULT_SAMPLE_MASK;
1111       } else {
1112          *location = SYSTEM_VALUE_SAMPLE_MASK_IN;
1113          set_mode_system_value(b, mode);
1114       }
1115       break;
1116    case SpvBuiltInFragDepth:
1117       *location = FRAG_RESULT_DEPTH;
1118       vtn_assert(*mode == nir_var_shader_out);
1119       break;
1120    case SpvBuiltInHelperInvocation:
1121       *location = SYSTEM_VALUE_HELPER_INVOCATION;
1122       set_mode_system_value(b, mode);
1123       break;
1124    case SpvBuiltInNumWorkgroups:
1125       *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
1126       set_mode_system_value(b, mode);
1127       break;
1128    case SpvBuiltInWorkgroupSize:
1129       *location = SYSTEM_VALUE_LOCAL_GROUP_SIZE;
1130       set_mode_system_value(b, mode);
1131       break;
1132    case SpvBuiltInWorkgroupId:
1133       *location = SYSTEM_VALUE_WORK_GROUP_ID;
1134       set_mode_system_value(b, mode);
1135       break;
1136    case SpvBuiltInLocalInvocationId:
1137       *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
1138       set_mode_system_value(b, mode);
1139       break;
1140    case SpvBuiltInLocalInvocationIndex:
1141       *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1142       set_mode_system_value(b, mode);
1143       break;
1144    case SpvBuiltInGlobalInvocationId:
1145       *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1146       set_mode_system_value(b, mode);
1147       break;
1148    case SpvBuiltInBaseVertex:
1149       /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same
1150        * semantic as SPIR-V BaseVertex (SYSTEM_VALUE_FIRST_VERTEX).
1151        */
1152       *location = SYSTEM_VALUE_FIRST_VERTEX;
1153       set_mode_system_value(b, mode);
1154       break;
1155    case SpvBuiltInBaseInstance:
1156       *location = SYSTEM_VALUE_BASE_INSTANCE;
1157       set_mode_system_value(b, mode);
1158       break;
1159    case SpvBuiltInDrawIndex:
1160       *location = SYSTEM_VALUE_DRAW_ID;
1161       set_mode_system_value(b, mode);
1162       break;
1163    case SpvBuiltInSubgroupSize:
1164       *location = SYSTEM_VALUE_SUBGROUP_SIZE;
1165       set_mode_system_value(b, mode);
1166       break;
1167    case SpvBuiltInSubgroupId:
1168       *location = SYSTEM_VALUE_SUBGROUP_ID;
1169       set_mode_system_value(b, mode);
1170       break;
1171    case SpvBuiltInSubgroupLocalInvocationId:
1172       *location = SYSTEM_VALUE_SUBGROUP_INVOCATION;
1173       set_mode_system_value(b, mode);
1174       break;
1175    case SpvBuiltInNumSubgroups:
1176       *location = SYSTEM_VALUE_NUM_SUBGROUPS;
1177       set_mode_system_value(b, mode);
1178       break;
1179    case SpvBuiltInDeviceIndex:
1180       *location = SYSTEM_VALUE_DEVICE_INDEX;
1181       set_mode_system_value(b, mode);
1182       break;
1183    case SpvBuiltInViewIndex:
1184       *location = SYSTEM_VALUE_VIEW_INDEX;
1185       set_mode_system_value(b, mode);
1186       break;
1187    case SpvBuiltInSubgroupEqMask:
1188       *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK,
1189       set_mode_system_value(b, mode);
1190       break;
1191    case SpvBuiltInSubgroupGeMask:
1192       *location = SYSTEM_VALUE_SUBGROUP_GE_MASK,
1193       set_mode_system_value(b, mode);
1194       break;
1195    case SpvBuiltInSubgroupGtMask:
1196       *location = SYSTEM_VALUE_SUBGROUP_GT_MASK,
1197       set_mode_system_value(b, mode);
1198       break;
1199    case SpvBuiltInSubgroupLeMask:
1200       *location = SYSTEM_VALUE_SUBGROUP_LE_MASK,
1201       set_mode_system_value(b, mode);
1202       break;
1203    case SpvBuiltInSubgroupLtMask:
1204       *location = SYSTEM_VALUE_SUBGROUP_LT_MASK,
1205       set_mode_system_value(b, mode);
1206       break;
1207    case SpvBuiltInFragStencilRefEXT:
1208       *location = FRAG_RESULT_STENCIL;
1209       vtn_assert(*mode == nir_var_shader_out);
1210       break;
1211    case SpvBuiltInWorkDim:
1212       *location = SYSTEM_VALUE_WORK_DIM;
1213       set_mode_system_value(b, mode);
1214       break;
1215    case SpvBuiltInGlobalSize:
1216       *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE;
1217       set_mode_system_value(b, mode);
1218       break;
1219    default:
1220       vtn_fail("unsupported builtin: %u", builtin);
1221    }
1222 }
1223
1224 static void
1225 apply_var_decoration(struct vtn_builder *b,
1226                      struct nir_variable_data *var_data,
1227                      const struct vtn_decoration *dec)
1228 {
1229    switch (dec->decoration) {
1230    case SpvDecorationRelaxedPrecision:
1231       break; /* FIXME: Do nothing with this for now. */
1232    case SpvDecorationNoPerspective:
1233       var_data->interpolation = INTERP_MODE_NOPERSPECTIVE;
1234       break;
1235    case SpvDecorationFlat:
1236       var_data->interpolation = INTERP_MODE_FLAT;
1237       break;
1238    case SpvDecorationCentroid:
1239       var_data->centroid = true;
1240       break;
1241    case SpvDecorationSample:
1242       var_data->sample = true;
1243       break;
1244    case SpvDecorationInvariant:
1245       var_data->invariant = true;
1246       break;
1247    case SpvDecorationConstant:
1248       var_data->read_only = true;
1249       break;
1250    case SpvDecorationNonReadable:
1251       var_data->image.write_only = true;
1252       break;
1253    case SpvDecorationNonWritable:
1254       var_data->read_only = true;
1255       var_data->image.read_only = true;
1256       break;
1257    case SpvDecorationRestrict:
1258       var_data->image.restrict_flag = true;
1259       break;
1260    case SpvDecorationVolatile:
1261       var_data->image._volatile = true;
1262       break;
1263    case SpvDecorationCoherent:
1264       var_data->image.coherent = true;
1265       break;
1266    case SpvDecorationComponent:
1267       var_data->location_frac = dec->literals[0];
1268       break;
1269    case SpvDecorationIndex:
1270       var_data->index = dec->literals[0];
1271       break;
1272    case SpvDecorationBuiltIn: {
1273       SpvBuiltIn builtin = dec->literals[0];
1274
1275       nir_variable_mode mode = var_data->mode;
1276       vtn_get_builtin_location(b, builtin, &var_data->location, &mode);
1277       var_data->mode = mode;
1278
1279       switch (builtin) {
1280       case SpvBuiltInTessLevelOuter:
1281       case SpvBuiltInTessLevelInner:
1282          var_data->compact = true;
1283          break;
1284       case SpvBuiltInFragCoord:
1285          var_data->pixel_center_integer = b->pixel_center_integer;
1286          /* fallthrough */
1287       case SpvBuiltInSamplePosition:
1288          var_data->origin_upper_left = b->origin_upper_left;
1289          break;
1290       default:
1291          break;
1292       }
1293    }
1294
1295    case SpvDecorationSpecId:
1296    case SpvDecorationRowMajor:
1297    case SpvDecorationColMajor:
1298    case SpvDecorationMatrixStride:
1299    case SpvDecorationAliased:
1300    case SpvDecorationUniform:
1301    case SpvDecorationStream:
1302    case SpvDecorationLinkageAttributes:
1303       break; /* Do nothing with these here */
1304
1305    case SpvDecorationPatch:
1306       var_data->patch = true;
1307       break;
1308
1309    case SpvDecorationLocation:
1310       vtn_fail("Handled above");
1311
1312    case SpvDecorationBlock:
1313    case SpvDecorationBufferBlock:
1314    case SpvDecorationArrayStride:
1315    case SpvDecorationGLSLShared:
1316    case SpvDecorationGLSLPacked:
1317       break; /* These can apply to a type but we don't care about them */
1318
1319    case SpvDecorationBinding:
1320    case SpvDecorationDescriptorSet:
1321    case SpvDecorationNoContraction:
1322    case SpvDecorationInputAttachmentIndex:
1323       vtn_warn("Decoration not allowed for variable or structure member: %s",
1324                spirv_decoration_to_string(dec->decoration));
1325       break;
1326
1327    case SpvDecorationXfbBuffer:
1328       var_data->explicit_xfb_buffer = true;
1329       var_data->xfb_buffer = dec->literals[0];
1330       break;
1331    case SpvDecorationXfbStride:
1332       var_data->explicit_xfb_stride = true;
1333       var_data->xfb_stride = dec->literals[0];
1334       break;
1335    case SpvDecorationOffset:
1336       var_data->explicit_offset = true;
1337       var_data->offset = dec->literals[0];
1338       break;
1339
1340    case SpvDecorationCPacked:
1341    case SpvDecorationSaturatedConversion:
1342    case SpvDecorationFuncParamAttr:
1343    case SpvDecorationFPRoundingMode:
1344    case SpvDecorationFPFastMathMode:
1345    case SpvDecorationAlignment:
1346       vtn_warn("Decoration only allowed for CL-style kernels: %s",
1347                spirv_decoration_to_string(dec->decoration));
1348       break;
1349
1350    default:
1351       vtn_fail("Unhandled decoration");
1352    }
1353 }
1354
1355 static void
1356 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1357                 const struct vtn_decoration *dec, void *out_is_patch)
1358 {
1359    if (dec->decoration == SpvDecorationPatch) {
1360       *((bool *) out_is_patch) = true;
1361    }
1362 }
1363
1364 static void
1365 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1366                   const struct vtn_decoration *dec, void *void_var)
1367 {
1368    struct vtn_variable *vtn_var = void_var;
1369
1370    /* Handle decorations that apply to a vtn_variable as a whole */
1371    switch (dec->decoration) {
1372    case SpvDecorationBinding:
1373       vtn_var->binding = dec->literals[0];
1374       vtn_var->explicit_binding = true;
1375       return;
1376    case SpvDecorationDescriptorSet:
1377       vtn_var->descriptor_set = dec->literals[0];
1378       return;
1379    case SpvDecorationInputAttachmentIndex:
1380       vtn_var->input_attachment_index = dec->literals[0];
1381       return;
1382    case SpvDecorationPatch:
1383       vtn_var->patch = true;
1384       break;
1385    case SpvDecorationOffset:
1386       vtn_var->offset = dec->literals[0];
1387       break;
1388    default:
1389       break;
1390    }
1391
1392    if (val->value_type == vtn_value_type_pointer) {
1393       assert(val->pointer->var == void_var);
1394       assert(val->pointer->chain == NULL);
1395       assert(member == -1);
1396    } else {
1397       assert(val->value_type == vtn_value_type_type);
1398    }
1399
1400    /* Location is odd.  If applied to a split structure, we have to walk the
1401     * whole thing and accumulate the location.  It's easier to handle as a
1402     * special case.
1403     */
1404    if (dec->decoration == SpvDecorationLocation) {
1405       unsigned location = dec->literals[0];
1406       bool is_vertex_input = false;
1407       if (b->shader->info.stage == MESA_SHADER_FRAGMENT &&
1408           vtn_var->mode == vtn_variable_mode_output) {
1409          location += FRAG_RESULT_DATA0;
1410       } else if (b->shader->info.stage == MESA_SHADER_VERTEX &&
1411                  vtn_var->mode == vtn_variable_mode_input) {
1412          is_vertex_input = true;
1413          location += VERT_ATTRIB_GENERIC0;
1414       } else if (vtn_var->mode == vtn_variable_mode_input ||
1415                  vtn_var->mode == vtn_variable_mode_output) {
1416          location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1417       } else if (vtn_var->mode != vtn_variable_mode_uniform) {
1418          vtn_warn("Location must be on input, output, uniform, sampler or "
1419                   "image variable");
1420          return;
1421       }
1422
1423       if (vtn_var->var->num_members == 0) {
1424          /* This handles the member and lone variable cases */
1425          vtn_var->var->data.location = location;
1426       } else {
1427          /* This handles the structure member case */
1428          assert(vtn_var->var->members);
1429          for (unsigned i = 0; i < vtn_var->var->num_members; i++) {
1430             vtn_var->var->members[i].location = location;
1431             const struct glsl_type *member_type =
1432                glsl_get_struct_field(vtn_var->var->interface_type, i);
1433             location += glsl_count_attribute_slots(member_type,
1434                                                    is_vertex_input);
1435          }
1436       }
1437       return;
1438    } else {
1439       if (vtn_var->var) {
1440          if (vtn_var->var->num_members == 0) {
1441             assert(member == -1);
1442             apply_var_decoration(b, &vtn_var->var->data, dec);
1443          } else if (member >= 0) {
1444             /* Member decorations must come from a type */
1445             assert(val->value_type == vtn_value_type_type);
1446             apply_var_decoration(b, &vtn_var->var->members[member], dec);
1447          } else {
1448             unsigned length =
1449                glsl_get_length(glsl_without_array(vtn_var->type->type));
1450             for (unsigned i = 0; i < length; i++)
1451                apply_var_decoration(b, &vtn_var->var->members[i], dec);
1452          }
1453       } else {
1454          /* A few variables, those with external storage, have no actual
1455           * nir_variables associated with them.  Fortunately, all decorations
1456           * we care about for those variables are on the type only.
1457           */
1458          vtn_assert(vtn_var->mode == vtn_variable_mode_ubo ||
1459                     vtn_var->mode == vtn_variable_mode_ssbo ||
1460                     vtn_var->mode == vtn_variable_mode_push_constant ||
1461                     (vtn_var->mode == vtn_variable_mode_workgroup &&
1462                      b->options->lower_workgroup_access_to_offsets));
1463       }
1464    }
1465 }
1466
1467 static enum vtn_variable_mode
1468 vtn_storage_class_to_mode(struct vtn_builder *b,
1469                           SpvStorageClass class,
1470                           struct vtn_type *interface_type,
1471                           nir_variable_mode *nir_mode_out)
1472 {
1473    enum vtn_variable_mode mode;
1474    nir_variable_mode nir_mode;
1475    switch (class) {
1476    case SpvStorageClassUniform:
1477       if (interface_type->block) {
1478          mode = vtn_variable_mode_ubo;
1479          nir_mode = 0;
1480       } else if (interface_type->buffer_block) {
1481          mode = vtn_variable_mode_ssbo;
1482          nir_mode = 0;
1483       } else {
1484          /* Default-block uniforms, coming from gl_spirv */
1485          mode = vtn_variable_mode_uniform;
1486          nir_mode = nir_var_uniform;
1487       }
1488       break;
1489    case SpvStorageClassStorageBuffer:
1490       mode = vtn_variable_mode_ssbo;
1491       nir_mode = 0;
1492       break;
1493    case SpvStorageClassUniformConstant:
1494       mode = vtn_variable_mode_uniform;
1495       nir_mode = nir_var_uniform;
1496       break;
1497    case SpvStorageClassPushConstant:
1498       mode = vtn_variable_mode_push_constant;
1499       nir_mode = nir_var_uniform;
1500       break;
1501    case SpvStorageClassInput:
1502       mode = vtn_variable_mode_input;
1503       nir_mode = nir_var_shader_in;
1504       break;
1505    case SpvStorageClassOutput:
1506       mode = vtn_variable_mode_output;
1507       nir_mode = nir_var_shader_out;
1508       break;
1509    case SpvStorageClassPrivate:
1510       mode = vtn_variable_mode_global;
1511       nir_mode = nir_var_global;
1512       break;
1513    case SpvStorageClassFunction:
1514       mode = vtn_variable_mode_local;
1515       nir_mode = nir_var_local;
1516       break;
1517    case SpvStorageClassWorkgroup:
1518       mode = vtn_variable_mode_workgroup;
1519       nir_mode = nir_var_shared;
1520       break;
1521    case SpvStorageClassAtomicCounter:
1522       mode = vtn_variable_mode_uniform;
1523       nir_mode = nir_var_uniform;
1524       break;
1525    case SpvStorageClassCrossWorkgroup:
1526    case SpvStorageClassGeneric:
1527    default:
1528       vtn_fail("Unhandled variable storage class");
1529    }
1530
1531    if (nir_mode_out)
1532       *nir_mode_out = nir_mode;
1533
1534    return mode;
1535 }
1536
1537 nir_ssa_def *
1538 vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr)
1539 {
1540    if (vtn_pointer_uses_ssa_offset(b, ptr)) {
1541       /* This pointer needs to have a pointer type with actual storage */
1542       vtn_assert(ptr->ptr_type);
1543       vtn_assert(ptr->ptr_type->type);
1544
1545       if (!ptr->offset) {
1546          /* If we don't have an offset then we must be a pointer to the variable
1547           * itself.
1548           */
1549          vtn_assert(!ptr->offset && !ptr->block_index);
1550
1551          struct vtn_access_chain chain = {
1552             .length = 0,
1553          };
1554          ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
1555       }
1556
1557       vtn_assert(ptr->offset);
1558       if (ptr->block_index) {
1559          vtn_assert(ptr->mode == vtn_variable_mode_ubo ||
1560                     ptr->mode == vtn_variable_mode_ssbo);
1561          return nir_vec2(&b->nb, ptr->block_index, ptr->offset);
1562       } else {
1563          vtn_assert(ptr->mode == vtn_variable_mode_workgroup);
1564          return ptr->offset;
1565       }
1566    } else {
1567       return &vtn_pointer_to_deref(b, ptr)->dest.ssa;
1568    }
1569 }
1570
1571 struct vtn_pointer *
1572 vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa,
1573                      struct vtn_type *ptr_type)
1574 {
1575    vtn_assert(ssa->num_components <= 2 && ssa->bit_size == 32);
1576    vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
1577
1578    struct vtn_type *interface_type = ptr_type->deref;
1579    while (interface_type->base_type == vtn_base_type_array)
1580       interface_type = interface_type->array_element;
1581
1582    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
1583    nir_variable_mode nir_mode;
1584    ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class,
1585                                          interface_type, &nir_mode);
1586    ptr->type = ptr_type->deref;
1587    ptr->ptr_type = ptr_type;
1588
1589    if (ptr->mode == vtn_variable_mode_ubo ||
1590        ptr->mode == vtn_variable_mode_ssbo) {
1591       /* This pointer type needs to have actual storage */
1592       vtn_assert(ptr_type->type);
1593       vtn_assert(ssa->num_components == 2);
1594       ptr->block_index = nir_channel(&b->nb, ssa, 0);
1595       ptr->offset = nir_channel(&b->nb, ssa, 1);
1596    } else if (ptr->mode == vtn_variable_mode_workgroup ||
1597               ptr->mode == vtn_variable_mode_push_constant) {
1598       /* This pointer type needs to have actual storage */
1599       vtn_assert(ptr_type->type);
1600       vtn_assert(ssa->num_components == 1);
1601       ptr->block_index = NULL;
1602       ptr->offset = ssa;
1603    } else {
1604       ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode,
1605                                         ptr_type->deref->type);
1606    }
1607
1608    return ptr;
1609 }
1610
1611 static bool
1612 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
1613 {
1614    if (var->patch || !glsl_type_is_array(var->type->type))
1615       return false;
1616
1617    if (var->mode == vtn_variable_mode_input) {
1618       return stage == MESA_SHADER_TESS_CTRL ||
1619              stage == MESA_SHADER_TESS_EVAL ||
1620              stage == MESA_SHADER_GEOMETRY;
1621    }
1622
1623    if (var->mode == vtn_variable_mode_output)
1624       return stage == MESA_SHADER_TESS_CTRL;
1625
1626    return false;
1627 }
1628
1629 static void
1630 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val,
1631                     struct vtn_type *ptr_type, SpvStorageClass storage_class,
1632                     nir_constant *initializer)
1633 {
1634    vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
1635    struct vtn_type *type = ptr_type->deref;
1636
1637    struct vtn_type *without_array = type;
1638    while(glsl_type_is_array(without_array->type))
1639       without_array = without_array->array_element;
1640
1641    enum vtn_variable_mode mode;
1642    nir_variable_mode nir_mode;
1643    mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode);
1644
1645    switch (mode) {
1646    case vtn_variable_mode_ubo:
1647       b->shader->info.num_ubos++;
1648       break;
1649    case vtn_variable_mode_ssbo:
1650       b->shader->info.num_ssbos++;
1651       break;
1652    case vtn_variable_mode_uniform:
1653       if (glsl_type_is_image(without_array->type))
1654          b->shader->info.num_images++;
1655       else if (glsl_type_is_sampler(without_array->type))
1656          b->shader->info.num_textures++;
1657       break;
1658    case vtn_variable_mode_push_constant:
1659       b->shader->num_uniforms = vtn_type_block_size(b, type);
1660       break;
1661    default:
1662       /* No tallying is needed */
1663       break;
1664    }
1665
1666    struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1667    var->type = type;
1668    var->mode = mode;
1669
1670    vtn_assert(val->value_type == vtn_value_type_pointer);
1671    val->pointer = vtn_pointer_for_variable(b, var, ptr_type);
1672
1673    switch (var->mode) {
1674    case vtn_variable_mode_local:
1675    case vtn_variable_mode_global:
1676    case vtn_variable_mode_uniform:
1677       /* For these, we create the variable normally */
1678       var->var = rzalloc(b->shader, nir_variable);
1679       var->var->name = ralloc_strdup(var->var, val->name);
1680
1681       /* Need to tweak the nir type here as at vtn_handle_type we don't have
1682        * the access to storage_class, that is the one that points us that is
1683        * an atomic uint.
1684        */
1685       if (storage_class == SpvStorageClassAtomicCounter) {
1686          var->var->type = repair_atomic_type(var->type->type);
1687       } else {
1688          var->var->type = var->type->type;
1689       }
1690       var->var->data.mode = nir_mode;
1691       var->var->data.location = -1;
1692       var->var->interface_type = NULL;
1693       break;
1694
1695    case vtn_variable_mode_workgroup:
1696       if (b->options->lower_workgroup_access_to_offsets) {
1697          var->shared_location = -1;
1698       } else {
1699          /* Create the variable normally */
1700          var->var = rzalloc(b->shader, nir_variable);
1701          var->var->name = ralloc_strdup(var->var, val->name);
1702          var->var->type = var->type->type;
1703          var->var->data.mode = nir_var_shared;
1704       }
1705       break;
1706
1707    case vtn_variable_mode_input:
1708    case vtn_variable_mode_output: {
1709       /* In order to know whether or not we're a per-vertex inout, we need
1710        * the patch qualifier.  This means walking the variable decorations
1711        * early before we actually create any variables.  Not a big deal.
1712        *
1713        * GLSLang really likes to place decorations in the most interior
1714        * thing it possibly can.  In particular, if you have a struct, it
1715        * will place the patch decorations on the struct members.  This
1716        * should be handled by the variable splitting below just fine.
1717        *
1718        * If you have an array-of-struct, things get even more weird as it
1719        * will place the patch decorations on the struct even though it's
1720        * inside an array and some of the members being patch and others not
1721        * makes no sense whatsoever.  Since the only sensible thing is for
1722        * it to be all or nothing, we'll call it patch if any of the members
1723        * are declared patch.
1724        */
1725       var->patch = false;
1726       vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
1727       if (glsl_type_is_array(var->type->type) &&
1728           glsl_type_is_struct(without_array->type)) {
1729          vtn_foreach_decoration(b, vtn_value(b, without_array->id,
1730                                              vtn_value_type_type),
1731                                 var_is_patch_cb, &var->patch);
1732       }
1733
1734       /* For inputs and outputs, we immediately split structures.  This
1735        * is for a couple of reasons.  For one, builtins may all come in
1736        * a struct and we really want those split out into separate
1737        * variables.  For another, interpolation qualifiers can be
1738        * applied to members of the top-level struct ane we need to be
1739        * able to preserve that information.
1740        */
1741
1742       struct vtn_type *interface_type = var->type;
1743       if (is_per_vertex_inout(var, b->shader->info.stage)) {
1744          /* In Geometry shaders (and some tessellation), inputs come
1745           * in per-vertex arrays.  However, some builtins come in
1746           * non-per-vertex, hence the need for the is_array check.  In
1747           * any case, there are no non-builtin arrays allowed so this
1748           * check should be sufficient.
1749           */
1750          interface_type = var->type->array_element;
1751       }
1752
1753       var->var = rzalloc(b->shader, nir_variable);
1754       var->var->name = ralloc_strdup(var->var, val->name);
1755       var->var->type = var->type->type;
1756       var->var->interface_type = interface_type->type;
1757       var->var->data.mode = nir_mode;
1758       var->var->data.patch = var->patch;
1759
1760       if (glsl_type_is_struct(interface_type->type)) {
1761          /* It's a struct.  Set it up as per-member. */
1762          var->var->num_members = glsl_get_length(interface_type->type);
1763          var->var->members = rzalloc_array(var->var, struct nir_variable_data,
1764                                            var->var->num_members);
1765
1766          for (unsigned i = 0; i < var->var->num_members; i++) {
1767             var->var->members[i].mode = nir_mode;
1768             var->var->members[i].patch = var->patch;
1769          }
1770       }
1771
1772       /* For inputs and outputs, we need to grab locations and builtin
1773        * information from the interface type.
1774        */
1775       vtn_foreach_decoration(b, vtn_value(b, interface_type->id,
1776                                           vtn_value_type_type),
1777                              var_decoration_cb, var);
1778       break;
1779    }
1780
1781    case vtn_variable_mode_ubo:
1782    case vtn_variable_mode_ssbo:
1783    case vtn_variable_mode_push_constant:
1784       /* These don't need actual variables. */
1785       break;
1786    }
1787
1788    if (initializer) {
1789       var->var->constant_initializer =
1790          nir_constant_clone(initializer, var->var);
1791    }
1792
1793    vtn_foreach_decoration(b, val, var_decoration_cb, var);
1794
1795    if (var->mode == vtn_variable_mode_uniform) {
1796       /* XXX: We still need the binding information in the nir_variable
1797        * for these. We should fix that.
1798        */
1799       var->var->data.binding = var->binding;
1800       var->var->data.explicit_binding = var->explicit_binding;
1801       var->var->data.descriptor_set = var->descriptor_set;
1802       var->var->data.index = var->input_attachment_index;
1803       var->var->data.offset = var->offset;
1804
1805       if (glsl_type_is_image(without_array->type))
1806          var->var->data.image.format = without_array->image_format;
1807    }
1808
1809    if (var->mode == vtn_variable_mode_local) {
1810       vtn_assert(var->var != NULL && var->var->members == NULL);
1811       nir_function_impl_add_variable(b->nb.impl, var->var);
1812    } else if (var->var) {
1813       nir_shader_add_variable(b->shader, var->var);
1814    } else {
1815       vtn_assert(vtn_pointer_is_external_block(b, val->pointer));
1816    }
1817 }
1818
1819 static void
1820 vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode,
1821                        struct vtn_type *dst_type,
1822                        struct vtn_type *src_type)
1823 {
1824    if (dst_type->id == src_type->id)
1825       return;
1826
1827    if (vtn_types_compatible(b, dst_type, src_type)) {
1828       /* Early versions of GLSLang would re-emit types unnecessarily and you
1829        * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have
1830        * mismatched source and destination types.
1831        *
1832        * https://github.com/KhronosGroup/glslang/issues/304
1833        * https://github.com/KhronosGroup/glslang/issues/307
1834        * https://bugs.freedesktop.org/show_bug.cgi?id=104338
1835        * https://bugs.freedesktop.org/show_bug.cgi?id=104424
1836        */
1837       vtn_warn("Source and destination types of %s do not have the same "
1838                "ID (but are compatible): %u vs %u",
1839                 spirv_op_to_string(opcode), dst_type->id, src_type->id);
1840       return;
1841    }
1842
1843    vtn_fail("Source and destination types of %s do not match: %s vs. %s",
1844             spirv_op_to_string(opcode),
1845             glsl_get_type_name(dst_type->type),
1846             glsl_get_type_name(src_type->type));
1847 }
1848
1849 void
1850 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1851                      const uint32_t *w, unsigned count)
1852 {
1853    switch (opcode) {
1854    case SpvOpUndef: {
1855       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef);
1856       val->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1857       break;
1858    }
1859
1860    case SpvOpVariable: {
1861       struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type;
1862
1863       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer);
1864
1865       SpvStorageClass storage_class = w[3];
1866       nir_constant *initializer = NULL;
1867       if (count > 4)
1868          initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant;
1869
1870       vtn_create_variable(b, val, ptr_type, storage_class, initializer);
1871       break;
1872    }
1873
1874    case SpvOpAccessChain:
1875    case SpvOpPtrAccessChain:
1876    case SpvOpInBoundsAccessChain: {
1877       struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4);
1878       chain->ptr_as_array = (opcode == SpvOpPtrAccessChain);
1879
1880       unsigned idx = 0;
1881       for (int i = 4; i < count; i++) {
1882          struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1883          if (link_val->value_type == vtn_value_type_constant) {
1884             chain->link[idx].mode = vtn_access_mode_literal;
1885             chain->link[idx].id = link_val->constant->values[0].u32[0];
1886          } else {
1887             chain->link[idx].mode = vtn_access_mode_id;
1888             chain->link[idx].id = w[i];
1889
1890          }
1891          idx++;
1892       }
1893
1894       struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type;
1895       struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1896       if (base_val->value_type == vtn_value_type_sampled_image) {
1897          /* This is rather insane.  SPIR-V allows you to use OpSampledImage
1898           * to combine an array of images with a single sampler to get an
1899           * array of sampled images that all share the same sampler.
1900           * Fortunately, this means that we can more-or-less ignore the
1901           * sampler when crawling the access chain, but it does leave us
1902           * with this rather awkward little special-case.
1903           */
1904          struct vtn_value *val =
1905             vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1906          val->sampled_image = ralloc(b, struct vtn_sampled_image);
1907          val->sampled_image->type = base_val->sampled_image->type;
1908          val->sampled_image->image =
1909             vtn_pointer_dereference(b, base_val->sampled_image->image, chain);
1910          val->sampled_image->sampler = base_val->sampled_image->sampler;
1911       } else {
1912          vtn_assert(base_val->value_type == vtn_value_type_pointer);
1913          struct vtn_value *val =
1914             vtn_push_value(b, w[2], vtn_value_type_pointer);
1915          val->pointer = vtn_pointer_dereference(b, base_val->pointer, chain);
1916          val->pointer->ptr_type = ptr_type;
1917       }
1918       break;
1919    }
1920
1921    case SpvOpCopyMemory: {
1922       struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer);
1923       struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer);
1924
1925       vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref);
1926
1927       vtn_variable_copy(b, dest->pointer, src->pointer);
1928       break;
1929    }
1930
1931    case SpvOpLoad: {
1932       struct vtn_type *res_type =
1933          vtn_value(b, w[1], vtn_value_type_type)->type;
1934       struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer);
1935       struct vtn_pointer *src = src_val->pointer;
1936
1937       vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref);
1938
1939       if (glsl_type_is_image(res_type->type) ||
1940           glsl_type_is_sampler(res_type->type)) {
1941          vtn_push_value(b, w[2], vtn_value_type_pointer)->pointer = src;
1942          return;
1943       }
1944
1945       vtn_push_ssa(b, w[2], res_type, vtn_variable_load(b, src));
1946       break;
1947    }
1948
1949    case SpvOpStore: {
1950       struct vtn_value *dest_val = vtn_value(b, w[1], vtn_value_type_pointer);
1951       struct vtn_pointer *dest = dest_val->pointer;
1952       struct vtn_value *src_val = vtn_untyped_value(b, w[2]);
1953
1954       /* OpStore requires us to actually have a storage type */
1955       vtn_fail_if(dest->type->type == NULL,
1956                   "Invalid destination type for OpStore");
1957
1958       if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL &&
1959           glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) {
1960          /* Early versions of GLSLang would use uint types for UBOs/SSBOs but
1961           * would then store them to a local variable as bool.  Work around
1962           * the issue by doing an implicit conversion.
1963           *
1964           * https://github.com/KhronosGroup/glslang/issues/170
1965           * https://bugs.freedesktop.org/show_bug.cgi?id=104424
1966           */
1967          vtn_warn("OpStore of value of type OpTypeInt to a pointer to type "
1968                   "OpTypeBool.  Doing an implicit conversion to work around "
1969                   "the problem.");
1970          struct vtn_ssa_value *bool_ssa =
1971             vtn_create_ssa_value(b, dest->type->type);
1972          bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def);
1973          vtn_variable_store(b, bool_ssa, dest);
1974          break;
1975       }
1976
1977       vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type);
1978
1979       if (glsl_type_is_sampler(dest->type->type)) {
1980          vtn_warn("OpStore of a sampler detected.  Doing on-the-fly copy "
1981                   "propagation to workaround the problem.");
1982          vtn_assert(dest->var->copy_prop_sampler == NULL);
1983          dest->var->copy_prop_sampler =
1984             vtn_value(b, w[2], vtn_value_type_pointer)->pointer;
1985          break;
1986       }
1987
1988       struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1989       vtn_variable_store(b, src, dest);
1990       break;
1991    }
1992
1993    case SpvOpArrayLength: {
1994       struct vtn_pointer *ptr =
1995          vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
1996
1997       const uint32_t offset = ptr->var->type->offsets[w[4]];
1998       const uint32_t stride = ptr->var->type->members[w[4]]->stride;
1999
2000       if (!ptr->block_index) {
2001          struct vtn_access_chain chain = {
2002             .length = 0,
2003          };
2004          ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
2005          vtn_assert(ptr->block_index);
2006       }
2007
2008       nir_intrinsic_instr *instr =
2009          nir_intrinsic_instr_create(b->nb.shader,
2010                                     nir_intrinsic_get_buffer_size);
2011       instr->src[0] = nir_src_for_ssa(ptr->block_index);
2012       nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
2013       nir_builder_instr_insert(&b->nb, &instr->instr);
2014       nir_ssa_def *buf_size = &instr->dest.ssa;
2015
2016       /* array_length = max(buffer_size - offset, 0) / stride */
2017       nir_ssa_def *array_length =
2018          nir_idiv(&b->nb,
2019                   nir_imax(&b->nb,
2020                            nir_isub(&b->nb,
2021                                     buf_size,
2022                                     nir_imm_int(&b->nb, offset)),
2023                            nir_imm_int(&b->nb, 0u)),
2024                   nir_imm_int(&b->nb, stride));
2025
2026       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
2027       val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
2028       val->ssa->def = array_length;
2029       break;
2030    }
2031
2032    case SpvOpCopyMemorySized:
2033    default:
2034       vtn_fail("Unhandled opcode");
2035    }
2036 }