src/compiler/spirv/vtn_variables.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 #include "vtn_private.h"
  29 #include "spirv_info.h"
  30
  31 static struct vtn_access_chain *
  32 vtn_access_chain_create(struct vtn_builder *b, unsigned length)
  33 {
  34    struct vtn_access_chain *chain;
  35
  36    /* Subtract 1 from the length since there's already one built in */
  37    size_t size = sizeof(*chain) +
  38                  (MAX2(length, 1) - 1) * sizeof(chain->link[0]);
  39    chain = rzalloc_size(b, size);
  40    chain->length = length;
  41
  42    return chain;
  43 }
  44
  45 static struct vtn_access_chain *
  46 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
  47                         unsigned new_ids)
  48 {
  49    struct vtn_access_chain *chain;
  50
  51    unsigned old_len = old ? old->length : 0;
  52    chain = vtn_access_chain_create(b, old_len + new_ids);
  53
  54    for (unsigned i = 0; i < old_len; i++)
  55       chain->link[i] = old->link[i];
  56
  57    return chain;
  58 }
  59
  60 /* Dereference the given base pointer by the access chain */
  61 static struct vtn_pointer *
  62 vtn_access_chain_pointer_dereference(struct vtn_builder *b,
  63                                      struct vtn_pointer *base,
  64                                      struct vtn_access_chain *deref_chain)
  65 {
  66    struct vtn_access_chain *chain =
  67       vtn_access_chain_extend(b, base->chain, deref_chain->length);
  68    struct vtn_type *type = base->type;
  69
  70    /* OpPtrAccessChain is only allowed on things which support variable
  71     * pointers.  For everything else, the client is expected to just pass us
  72     * the right access chain.
  73     */
  74    assert(!deref_chain->ptr_as_array);
  75
  76    unsigned start = base->chain ? base->chain->length : 0;
  77    for (unsigned i = 0; i < deref_chain->length; i++) {
  78       chain->link[start + i] = deref_chain->link[i];
  79
  80       if (glsl_type_is_struct(type->type)) {
  81          assert(deref_chain->link[i].mode == vtn_access_mode_literal);
  82          type = type->members[deref_chain->link[i].id];
  83       } else {
  84          type = type->array_element;
  85       }
  86    }
  87
  88    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
  89    ptr->mode = base->mode;
  90    ptr->type = type;
  91    ptr->var = base->var;
  92    ptr->chain = chain;
  93
  94    return ptr;
  95 }
  96
  97 static nir_ssa_def *
  98 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
  99                        unsigned stride)
 100 {
 101    assert(stride > 0);
 102    if (link.mode == vtn_access_mode_literal) {
 103       return nir_imm_int(&b->nb, link.id * stride);
 104    } else if (stride == 1) {
 105       return vtn_ssa_value(b, link.id)->def;
 106    } else {
 107       return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
 108                               nir_imm_int(&b->nb, stride));
 109    }
 110 }
 111
 112 static nir_ssa_def *
 113 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var,
 114                             nir_ssa_def *desc_array_index)
 115 {
 116    if (!desc_array_index) {
 117       assert(glsl_type_is_struct(var->type->type));
 118       desc_array_index = nir_imm_int(&b->nb, 0);
 119    }
 120
 121    nir_intrinsic_instr *instr =
 122       nir_intrinsic_instr_create(b->nb.shader,
 123                                  nir_intrinsic_vulkan_resource_index);
 124    instr->src[0] = nir_src_for_ssa(desc_array_index);
 125    nir_intrinsic_set_desc_set(instr, var->descriptor_set);
 126    nir_intrinsic_set_binding(instr, var->binding);
 127
 128    nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 129    nir_builder_instr_insert(&b->nb, &instr->instr);
 130
 131    return &instr->dest.ssa;
 132 }
 133
 134 static struct vtn_pointer *
 135 vtn_ssa_offset_pointer_dereference(struct vtn_builder *b,
 136                                    struct vtn_pointer *base,
 137                                    struct vtn_access_chain *deref_chain)
 138 {
 139    nir_ssa_def *block_index = base->block_index;
 140    nir_ssa_def *offset = base->offset;
 141    struct vtn_type *type = base->type;
 142
 143    unsigned idx = 0;
 144    if (deref_chain->ptr_as_array) {
 145       /* We need ptr_type for the stride */
 146       assert(base->ptr_type);
 147       /* This must be a pointer to an actual element somewhere */
 148       assert(block_index && offset);
 149       /* We need at least one element in the chain */
 150       assert(deref_chain->length >= 1);
 151
 152       nir_ssa_def *elem_offset =
 153          vtn_access_link_as_ssa(b, deref_chain->link[idx],
 154                                 base->ptr_type->stride);
 155       offset = nir_iadd(&b->nb, offset, elem_offset);
 156       idx++;
 157    }
 158
 159    if (!block_index) {
 160       assert(base->var);
 161       if (glsl_type_is_array(type->type)) {
 162          /* We need at least one element in the chain */
 163          assert(deref_chain->length >= 1);
 164
 165          nir_ssa_def *desc_arr_idx =
 166             vtn_access_link_as_ssa(b, deref_chain->link[0], 1);
 167          block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
 168          type = type->array_element;
 169          idx++;
 170       } else {
 171          block_index = vtn_variable_resource_index(b, base->var, NULL);
 172       }
 173
 174       /* This is the first access chain so we also need an offset */
 175       assert(!offset);
 176       offset = nir_imm_int(&b->nb, 0);
 177    }
 178    assert(offset);
 179
 180    for (; idx < deref_chain->length; idx++) {
 181       switch (glsl_get_base_type(type->type)) {
 182       case GLSL_TYPE_UINT:
 183       case GLSL_TYPE_INT:
 184       case GLSL_TYPE_UINT64:
 185       case GLSL_TYPE_INT64:
 186       case GLSL_TYPE_FLOAT:
 187       case GLSL_TYPE_DOUBLE:
 188       case GLSL_TYPE_BOOL:
 189       case GLSL_TYPE_ARRAY: {
 190          nir_ssa_def *elem_offset =
 191             vtn_access_link_as_ssa(b, deref_chain->link[idx], type->stride);
 192          offset = nir_iadd(&b->nb, offset, elem_offset);
 193          type = type->array_element;
 194          break;
 195       }
 196
 197       case GLSL_TYPE_STRUCT: {
 198          assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
 199          unsigned member = deref_chain->link[idx].id;
 200          nir_ssa_def *mem_offset = nir_imm_int(&b->nb, type->offsets[member]);
 201          offset = nir_iadd(&b->nb, offset, mem_offset);
 202          type = type->members[member];
 203          break;
 204       }
 205
 206       default:
 207          unreachable("Invalid type for deref");
 208       }
 209    }
 210
 211    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
 212    ptr->mode = base->mode;
 213    ptr->type = type;
 214    ptr->block_index = block_index;
 215    ptr->offset = offset;
 216
 217    return ptr;
 218 }
 219
 220 /* Dereference the given base pointer by the access chain */
 221 static struct vtn_pointer *
 222 vtn_pointer_dereference(struct vtn_builder *b,
 223                         struct vtn_pointer *base,
 224                         struct vtn_access_chain *deref_chain)
 225 {
 226    if (vtn_pointer_uses_ssa_offset(base)) {
 227       return vtn_ssa_offset_pointer_dereference(b, base, deref_chain);
 228    } else {
 229       return vtn_access_chain_pointer_dereference(b, base, deref_chain);
 230    }
 231 }
 232
 233 /* Crawls a chain of array derefs and rewrites the types so that the
 234  * lengths stay the same but the terminal type is the one given by
 235  * tail_type.  This is useful for split structures.
 236  */
 237 static void
 238 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type)
 239 {
 240    deref->type = type;
 241    if (deref->child) {
 242       assert(deref->child->deref_type == nir_deref_type_array);
 243       assert(glsl_type_is_array(deref->type));
 244       rewrite_deref_types(deref->child, glsl_get_array_element(type));
 245    }
 246 }
 247
 248 struct vtn_pointer *
 249 vtn_pointer_for_variable(struct vtn_builder *b,
 250                          struct vtn_variable *var, struct vtn_type *ptr_type)
 251 {
 252    struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer);
 253
 254    pointer->mode = var->mode;
 255    pointer->type = var->type;
 256    assert(ptr_type->base_type == vtn_base_type_pointer);
 257    assert(ptr_type->deref->type == var->type->type);
 258    pointer->ptr_type = ptr_type;
 259    pointer->var = var;
 260
 261    return pointer;
 262 }
 263
 264 nir_deref_var *
 265 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr)
 266 {
 267    /* Do on-the-fly copy propagation for samplers. */
 268    if (ptr->var->copy_prop_sampler)
 269       return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler);
 270
 271    nir_deref_var *deref_var;
 272    if (ptr->var->var) {
 273       deref_var = nir_deref_var_create(b, ptr->var->var);
 274       /* Raw variable access */
 275       if (!ptr->chain)
 276          return deref_var;
 277    } else {
 278       assert(ptr->var->members);
 279       /* Create the deref_var manually.  It will get filled out later. */
 280       deref_var = rzalloc(b, nir_deref_var);
 281       deref_var->deref.deref_type = nir_deref_type_var;
 282    }
 283
 284    struct vtn_access_chain *chain = ptr->chain;
 285    assert(chain);
 286
 287    struct vtn_type *deref_type = ptr->var->type;
 288    nir_deref *tail = &deref_var->deref;
 289    nir_variable **members = ptr->var->members;
 290
 291    for (unsigned i = 0; i < chain->length; i++) {
 292       enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
 293       switch (base_type) {
 294       case GLSL_TYPE_UINT:
 295       case GLSL_TYPE_INT:
 296       case GLSL_TYPE_UINT64:
 297       case GLSL_TYPE_INT64:
 298       case GLSL_TYPE_FLOAT:
 299       case GLSL_TYPE_DOUBLE:
 300       case GLSL_TYPE_BOOL:
 301       case GLSL_TYPE_ARRAY: {
 302          deref_type = deref_type->array_element;
 303
 304          nir_deref_array *deref_arr = nir_deref_array_create(b);
 305          deref_arr->deref.type = deref_type->type;
 306
 307          if (chain->link[i].mode == vtn_access_mode_literal) {
 308             deref_arr->deref_array_type = nir_deref_array_type_direct;
 309             deref_arr->base_offset = chain->link[i].id;
 310          } else {
 311             assert(chain->link[i].mode == vtn_access_mode_id);
 312             deref_arr->deref_array_type = nir_deref_array_type_indirect;
 313             deref_arr->base_offset = 0;
 314             deref_arr->indirect =
 315                nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
 316          }
 317          tail->child = &deref_arr->deref;
 318          tail = tail->child;
 319          break;
 320       }
 321
 322       case GLSL_TYPE_STRUCT: {
 323          assert(chain->link[i].mode == vtn_access_mode_literal);
 324          unsigned idx = chain->link[i].id;
 325          deref_type = deref_type->members[idx];
 326          if (members) {
 327             /* This is a pre-split structure. */
 328             deref_var->var = members[idx];
 329             rewrite_deref_types(&deref_var->deref, members[idx]->type);
 330             assert(tail->type == deref_type->type);
 331             members = NULL;
 332          } else {
 333             nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
 334             deref_struct->deref.type = deref_type->type;
 335             tail->child = &deref_struct->deref;
 336             tail = tail->child;
 337          }
 338          break;
 339       }
 340       default:
 341          unreachable("Invalid type for deref");
 342       }
 343    }
 344
 345    assert(members == NULL);
 346    return deref_var;
 347 }
 348
 349 static void
 350 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref,
 351                       nir_deref *tail, struct vtn_ssa_value *inout)
 352 {
 353    /* The deref tail may contain a deref to select a component of a vector (in
 354     * other words, it might not be an actual tail) so we have to save it away
 355     * here since we overwrite it later.
 356     */
 357    nir_deref *old_child = tail->child;
 358
 359    if (glsl_type_is_vector_or_scalar(tail->type)) {
 360       /* Terminate the deref chain in case there is one more link to pick
 361        * off a component of the vector.
 362        */
 363       tail->child = NULL;
 364
 365       nir_intrinsic_op op = load ? nir_intrinsic_load_var :
 366                                    nir_intrinsic_store_var;
 367
 368       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op);
 369       intrin->variables[0] = nir_deref_var_clone(deref, intrin);
 370       intrin->num_components = glsl_get_vector_elements(tail->type);
 371
 372       if (load) {
 373          nir_ssa_dest_init(&intrin->instr, &intrin->dest,
 374                            intrin->num_components,
 375                            glsl_get_bit_size(tail->type),
 376                            NULL);
 377          inout->def = &intrin->dest.ssa;
 378       } else {
 379          nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1);
 380          intrin->src[0] = nir_src_for_ssa(inout->def);
 381       }
 382
 383       nir_builder_instr_insert(&b->nb, &intrin->instr);
 384    } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY ||
 385               glsl_type_is_matrix(tail->type)) {
 386       unsigned elems = glsl_get_length(tail->type);
 387       nir_deref_array *deref_arr = nir_deref_array_create(b);
 388       deref_arr->deref_array_type = nir_deref_array_type_direct;
 389       deref_arr->deref.type = glsl_get_array_element(tail->type);
 390       tail->child = &deref_arr->deref;
 391       for (unsigned i = 0; i < elems; i++) {
 392          deref_arr->base_offset = i;
 393          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 394       }
 395    } else {
 396       assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT);
 397       unsigned elems = glsl_get_length(tail->type);
 398       nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0);
 399       tail->child = &deref_struct->deref;
 400       for (unsigned i = 0; i < elems; i++) {
 401          deref_struct->index = i;
 402          deref_struct->deref.type = glsl_get_struct_field(tail->type, i);
 403          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 404       }
 405    }
 406
 407    tail->child = old_child;
 408 }
 409
 410 nir_deref_var *
 411 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
 412 {
 413    struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer;
 414    return vtn_pointer_to_deref(b, ptr);
 415 }
 416
 417 /*
 418  * Gets the NIR-level deref tail, which may have as a child an array deref
 419  * selecting which component due to OpAccessChain supporting per-component
 420  * indexing in SPIR-V.
 421  */
 422 static nir_deref *
 423 get_deref_tail(nir_deref_var *deref)
 424 {
 425    nir_deref *cur = &deref->deref;
 426    while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child)
 427       cur = cur->child;
 428
 429    return cur;
 430 }
 431
 432 struct vtn_ssa_value *
 433 vtn_local_load(struct vtn_builder *b, nir_deref_var *src)
 434 {
 435    nir_deref *src_tail = get_deref_tail(src);
 436    struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
 437    _vtn_local_load_store(b, true, src, src_tail, val);
 438
 439    if (src_tail->child) {
 440       nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child);
 441       assert(vec_deref->deref.child == NULL);
 442       val->type = vec_deref->deref.type;
 443       if (vec_deref->deref_array_type == nir_deref_array_type_direct)
 444          val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset);
 445       else
 446          val->def = vtn_vector_extract_dynamic(b, val->def,
 447                                                vec_deref->indirect.ssa);
 448    }
 449
 450    return val;
 451 }
 452
 453 void
 454 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 455                 nir_deref_var *dest)
 456 {
 457    nir_deref *dest_tail = get_deref_tail(dest);
 458
 459    if (dest_tail->child) {
 460       struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
 461       _vtn_local_load_store(b, true, dest, dest_tail, val);
 462       nir_deref_array *deref = nir_deref_as_array(dest_tail->child);
 463       assert(deref->deref.child == NULL);
 464       if (deref->deref_array_type == nir_deref_array_type_direct)
 465          val->def = vtn_vector_insert(b, val->def, src->def,
 466                                       deref->base_offset);
 467       else
 468          val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
 469                                               deref->indirect.ssa);
 470       _vtn_local_load_store(b, false, dest, dest_tail, val);
 471    } else {
 472       _vtn_local_load_store(b, false, dest, dest_tail, src);
 473    }
 474 }
 475
 476 static nir_ssa_def *
 477 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_pointer *ptr,
 478                           struct vtn_type **type, unsigned *chain_idx)
 479 {
 480    /* Push constants have no explicit binding */
 481    if (ptr->mode == vtn_variable_mode_push_constant) {
 482       *chain_idx = 0;
 483       *type = ptr->var->type;
 484       return NULL;
 485    }
 486
 487    if (glsl_type_is_array(ptr->var->type->type)) {
 488       assert(ptr->chain->length > 0);
 489       nir_ssa_def *desc_array_index =
 490          vtn_access_link_as_ssa(b, ptr->chain->link[0], 1);
 491       *chain_idx = 1;
 492       *type = ptr->var->type->array_element;
 493       return vtn_variable_resource_index(b, ptr->var, desc_array_index);
 494    } else {
 495       *chain_idx = 0;
 496       *type = ptr->var->type;
 497       return vtn_variable_resource_index(b, ptr->var, NULL);
 498    }
 499 }
 500
 501 nir_ssa_def *
 502 vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr,
 503                       nir_ssa_def **index_out, unsigned *end_idx_out)
 504 {
 505    if (ptr->offset) {
 506       assert(ptr->block_index);
 507       *index_out = ptr->block_index;
 508       return ptr->offset;
 509    }
 510
 511    unsigned idx = 0;
 512    struct vtn_type *type;
 513    *index_out = get_vulkan_resource_index(b, ptr, &type, &idx);
 514
 515    nir_ssa_def *offset = nir_imm_int(&b->nb, 0);
 516    for (; idx < ptr->chain->length; idx++) {
 517       enum glsl_base_type base_type = glsl_get_base_type(type->type);
 518       switch (base_type) {
 519       case GLSL_TYPE_UINT:
 520       case GLSL_TYPE_INT:
 521       case GLSL_TYPE_UINT64:
 522       case GLSL_TYPE_INT64:
 523       case GLSL_TYPE_FLOAT:
 524       case GLSL_TYPE_DOUBLE:
 525       case GLSL_TYPE_BOOL:
 526       case GLSL_TYPE_ARRAY:
 527          offset = nir_iadd(&b->nb, offset,
 528                            vtn_access_link_as_ssa(b, ptr->chain->link[idx],
 529                                                   type->stride));
 530
 531          type = type->array_element;
 532          break;
 533
 534       case GLSL_TYPE_STRUCT: {
 535          assert(ptr->chain->link[idx].mode == vtn_access_mode_literal);
 536          unsigned member = ptr->chain->link[idx].id;
 537          offset = nir_iadd(&b->nb, offset,
 538                            nir_imm_int(&b->nb, type->offsets[member]));
 539          type = type->members[member];
 540          break;
 541       }
 542
 543       default:
 544          unreachable("Invalid type for deref");
 545       }
 546    }
 547
 548    assert(type == ptr->type);
 549    if (end_idx_out)
 550       *end_idx_out = idx;
 551
 552    return offset;
 553 }
 554
 555 /* Tries to compute the size of an interface block based on the strides and
 556  * offsets that are provided to us in the SPIR-V source.
 557  */
 558 static unsigned
 559 vtn_type_block_size(struct vtn_type *type)
 560 {
 561    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 562    switch (base_type) {
 563    case GLSL_TYPE_UINT:
 564    case GLSL_TYPE_INT:
 565    case GLSL_TYPE_UINT64:
 566    case GLSL_TYPE_INT64:
 567    case GLSL_TYPE_FLOAT:
 568    case GLSL_TYPE_BOOL:
 569    case GLSL_TYPE_DOUBLE: {
 570       unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
 571                                         glsl_get_matrix_columns(type->type);
 572       if (cols > 1) {
 573          assert(type->stride > 0);
 574          return type->stride * cols;
 575       } else if (base_type == GLSL_TYPE_DOUBLE ||
 576                  base_type == GLSL_TYPE_UINT64 ||
 577                  base_type == GLSL_TYPE_INT64) {
 578          return glsl_get_vector_elements(type->type) * 8;
 579       } else {
 580          return glsl_get_vector_elements(type->type) * 4;
 581       }
 582    }
 583
 584    case GLSL_TYPE_STRUCT:
 585    case GLSL_TYPE_INTERFACE: {
 586       unsigned size = 0;
 587       unsigned num_fields = glsl_get_length(type->type);
 588       for (unsigned f = 0; f < num_fields; f++) {
 589          unsigned field_end = type->offsets[f] +
 590                               vtn_type_block_size(type->members[f]);
 591          size = MAX2(size, field_end);
 592       }
 593       return size;
 594    }
 595
 596    case GLSL_TYPE_ARRAY:
 597       assert(type->stride > 0);
 598       assert(glsl_get_length(type->type) > 0);
 599       return type->stride * glsl_get_length(type->type);
 600
 601    default:
 602       unreachable("Invalid block type");
 603       return 0;
 604    }
 605 }
 606
 607 static void
 608 vtn_access_chain_get_offset_size(struct vtn_access_chain *chain,
 609                                  struct vtn_type *type,
 610                                  unsigned *access_offset,
 611                                  unsigned *access_size)
 612 {
 613    *access_offset = 0;
 614
 615    for (unsigned i = 0; i < chain->length; i++) {
 616       if (chain->link[i].mode != vtn_access_mode_literal)
 617          break;
 618
 619       if (glsl_type_is_struct(type->type)) {
 620          *access_offset += type->offsets[chain->link[i].id];
 621          type = type->members[chain->link[i].id];
 622       } else {
 623          *access_offset += type->stride * chain->link[i].id;
 624          type = type->array_element;
 625       }
 626    }
 627
 628    *access_size = vtn_type_block_size(type);
 629 }
 630
 631 static void
 632 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 633                      nir_ssa_def *index, nir_ssa_def *offset,
 634                      unsigned access_offset, unsigned access_size,
 635                      struct vtn_ssa_value **inout, const struct glsl_type *type)
 636 {
 637    nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
 638    instr->num_components = glsl_get_vector_elements(type);
 639
 640    int src = 0;
 641    if (!load) {
 642       nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
 643       instr->src[src++] = nir_src_for_ssa((*inout)->def);
 644    }
 645
 646    if (op == nir_intrinsic_load_push_constant) {
 647       assert(access_offset % 4 == 0);
 648
 649       nir_intrinsic_set_base(instr, access_offset);
 650       nir_intrinsic_set_range(instr, access_size);
 651    }
 652
 653    if (index)
 654       instr->src[src++] = nir_src_for_ssa(index);
 655
 656    if (op == nir_intrinsic_load_push_constant) {
 657       /* We need to subtract the offset from where the intrinsic will load the
 658        * data. */
 659       instr->src[src++] =
 660          nir_src_for_ssa(nir_isub(&b->nb, offset,
 661                                   nir_imm_int(&b->nb, access_offset)));
 662    } else {
 663       instr->src[src++] = nir_src_for_ssa(offset);
 664    }
 665
 666    if (load) {
 667       nir_ssa_dest_init(&instr->instr, &instr->dest,
 668                         instr->num_components,
 669                         glsl_get_bit_size(type), NULL);
 670       (*inout)->def = &instr->dest.ssa;
 671    }
 672
 673    nir_builder_instr_insert(&b->nb, &instr->instr);
 674
 675    if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
 676       (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
 677 }
 678
 679 static void
 680 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 681                       nir_ssa_def *index, nir_ssa_def *offset,
 682                       unsigned access_offset, unsigned access_size,
 683                       struct vtn_access_chain *chain, unsigned chain_idx,
 684                       struct vtn_type *type, struct vtn_ssa_value **inout)
 685 {
 686    if (chain && chain_idx >= chain->length)
 687       chain = NULL;
 688
 689    if (load && chain == NULL && *inout == NULL)
 690       *inout = vtn_create_ssa_value(b, type->type);
 691
 692    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 693    switch (base_type) {
 694    case GLSL_TYPE_UINT:
 695    case GLSL_TYPE_INT:
 696    case GLSL_TYPE_UINT64:
 697    case GLSL_TYPE_INT64:
 698    case GLSL_TYPE_FLOAT:
 699    case GLSL_TYPE_DOUBLE:
 700    case GLSL_TYPE_BOOL:
 701       /* This is where things get interesting.  At this point, we've hit
 702        * a vector, a scalar, or a matrix.
 703        */
 704       if (glsl_type_is_matrix(type->type)) {
 705          /* Loading the whole matrix */
 706          struct vtn_ssa_value *transpose;
 707          unsigned num_ops, vec_width, col_stride;
 708          if (type->row_major) {
 709             num_ops = glsl_get_vector_elements(type->type);
 710             vec_width = glsl_get_matrix_columns(type->type);
 711             col_stride = type->array_element->stride;
 712             if (load) {
 713                const struct glsl_type *transpose_type =
 714                   glsl_matrix_type(base_type, vec_width, num_ops);
 715                *inout = vtn_create_ssa_value(b, transpose_type);
 716             } else {
 717                transpose = vtn_ssa_transpose(b, *inout);
 718                inout = &transpose;
 719             }
 720          } else {
 721             num_ops = glsl_get_matrix_columns(type->type);
 722             vec_width = glsl_get_vector_elements(type->type);
 723             col_stride = type->stride;
 724          }
 725
 726          for (unsigned i = 0; i < num_ops; i++) {
 727             nir_ssa_def *elem_offset =
 728                nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * col_stride));
 729             _vtn_load_store_tail(b, op, load, index, elem_offset,
 730                                  access_offset, access_size,
 731                                  &(*inout)->elems[i],
 732                                  glsl_vector_type(base_type, vec_width));
 733          }
 734
 735          if (load && type->row_major)
 736             *inout = vtn_ssa_transpose(b, *inout);
 737       } else {
 738          unsigned elems = glsl_get_vector_elements(type->type);
 739          unsigned type_size = glsl_get_bit_size(type->type) / 8;
 740          if (elems == 1 || type->stride == type_size) {
 741             /* This is a tightly-packed normal scalar or vector load */
 742             assert(glsl_type_is_vector_or_scalar(type->type));
 743             _vtn_load_store_tail(b, op, load, index, offset,
 744                                  access_offset, access_size,
 745                                  inout, type->type);
 746          } else {
 747             /* This is a strided load.  We have to load N things separately.
 748              * This is the single column of a row-major matrix case.
 749              */
 750             assert(type->stride > type_size);
 751             assert(type->stride % type_size == 0);
 752
 753             nir_ssa_def *per_comp[4];
 754             for (unsigned i = 0; i < elems; i++) {
 755                nir_ssa_def *elem_offset =
 756                   nir_iadd(&b->nb, offset,
 757                                    nir_imm_int(&b->nb, i * type->stride));
 758                struct vtn_ssa_value *comp, temp_val;
 759                if (!load) {
 760                   temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
 761                   temp_val.type = glsl_scalar_type(base_type);
 762                }
 763                comp = &temp_val;
 764                _vtn_load_store_tail(b, op, load, index, elem_offset,
 765                                     access_offset, access_size,
 766                                     &comp, glsl_scalar_type(base_type));
 767                per_comp[i] = comp->def;
 768             }
 769
 770             if (load) {
 771                if (*inout == NULL)
 772                   *inout = vtn_create_ssa_value(b, type->type);
 773                (*inout)->def = nir_vec(&b->nb, per_comp, elems);
 774             }
 775          }
 776       }
 777       return;
 778
 779    case GLSL_TYPE_ARRAY: {
 780       unsigned elems = glsl_get_length(type->type);
 781       for (unsigned i = 0; i < elems; i++) {
 782          nir_ssa_def *elem_off =
 783             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
 784          _vtn_block_load_store(b, op, load, index, elem_off,
 785                                access_offset, access_size,
 786                                NULL, 0,
 787                                type->array_element, &(*inout)->elems[i]);
 788       }
 789       return;
 790    }
 791
 792    case GLSL_TYPE_STRUCT: {
 793       unsigned elems = glsl_get_length(type->type);
 794       for (unsigned i = 0; i < elems; i++) {
 795          nir_ssa_def *elem_off =
 796             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
 797          _vtn_block_load_store(b, op, load, index, elem_off,
 798                                access_offset, access_size,
 799                                NULL, 0,
 800                                type->members[i], &(*inout)->elems[i]);
 801       }
 802       return;
 803    }
 804
 805    default:
 806       unreachable("Invalid block member type");
 807    }
 808 }
 809
 810 static struct vtn_ssa_value *
 811 vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src)
 812 {
 813    nir_intrinsic_op op;
 814    unsigned access_offset = 0, access_size = 0;
 815    switch (src->mode) {
 816    case vtn_variable_mode_ubo:
 817       op = nir_intrinsic_load_ubo;
 818       break;
 819    case vtn_variable_mode_ssbo:
 820       op = nir_intrinsic_load_ssbo;
 821       break;
 822    case vtn_variable_mode_push_constant:
 823       op = nir_intrinsic_load_push_constant;
 824       vtn_access_chain_get_offset_size(src->chain, src->var->type,
 825                                        &access_offset, &access_size);
 826       break;
 827    default:
 828       unreachable("Invalid block variable mode");
 829    }
 830
 831    nir_ssa_def *offset, *index = NULL;
 832    unsigned chain_idx;
 833    offset = vtn_pointer_to_offset(b, src, &index, &chain_idx);
 834
 835    struct vtn_ssa_value *value = NULL;
 836    _vtn_block_load_store(b, op, true, index, offset,
 837                          access_offset, access_size,
 838                          src->chain, chain_idx, src->type, &value);
 839    return value;
 840 }
 841
 842 static void
 843 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 844                 struct vtn_pointer *dst)
 845 {
 846    nir_ssa_def *offset, *index = NULL;
 847    unsigned chain_idx;
 848    offset = vtn_pointer_to_offset(b, dst, &index, &chain_idx);
 849
 850    _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset,
 851                          0, 0, dst->chain, chain_idx, dst->type, &src);
 852 }
 853
 854 static bool
 855 vtn_pointer_is_external_block(struct vtn_pointer *ptr)
 856 {
 857    return ptr->mode == vtn_variable_mode_ssbo ||
 858           ptr->mode == vtn_variable_mode_ubo ||
 859           ptr->mode == vtn_variable_mode_push_constant;
 860 }
 861
 862 static void
 863 _vtn_variable_load_store(struct vtn_builder *b, bool load,
 864                          struct vtn_pointer *ptr,
 865                          struct vtn_ssa_value **inout)
 866 {
 867    enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type);
 868    switch (base_type) {
 869    case GLSL_TYPE_UINT:
 870    case GLSL_TYPE_INT:
 871    case GLSL_TYPE_UINT64:
 872    case GLSL_TYPE_INT64:
 873    case GLSL_TYPE_FLOAT:
 874    case GLSL_TYPE_BOOL:
 875    case GLSL_TYPE_DOUBLE:
 876       /* At this point, we have a scalar, vector, or matrix so we know that
 877        * there cannot be any structure splitting still in the way.  By
 878        * stopping at the matrix level rather than the vector level, we
 879        * ensure that matrices get loaded in the optimal way even if they
 880        * are storred row-major in a UBO.
 881        */
 882       if (load) {
 883          *inout = vtn_local_load(b, vtn_pointer_to_deref(b, ptr));
 884       } else {
 885          vtn_local_store(b, *inout, vtn_pointer_to_deref(b, ptr));
 886       }
 887       return;
 888
 889    case GLSL_TYPE_ARRAY:
 890    case GLSL_TYPE_STRUCT: {
 891       unsigned elems = glsl_get_length(ptr->type->type);
 892       if (load) {
 893          assert(*inout == NULL);
 894          *inout = rzalloc(b, struct vtn_ssa_value);
 895          (*inout)->type = ptr->type->type;
 896          (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
 897       }
 898
 899       struct vtn_access_chain chain = {
 900          .length = 1,
 901          .link = {
 902             { .mode = vtn_access_mode_literal, },
 903          }
 904       };
 905       for (unsigned i = 0; i < elems; i++) {
 906          chain.link[0].id = i;
 907          struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain);
 908          _vtn_variable_load_store(b, load, elem, &(*inout)->elems[i]);
 909       }
 910       return;
 911    }
 912
 913    default:
 914       unreachable("Invalid access chain type");
 915    }
 916 }
 917
 918 struct vtn_ssa_value *
 919 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src)
 920 {
 921    if (vtn_pointer_is_external_block(src)) {
 922       return vtn_block_load(b, src);
 923    } else {
 924       struct vtn_ssa_value *val = NULL;
 925       _vtn_variable_load_store(b, true, src, &val);
 926       return val;
 927    }
 928 }
 929
 930 void
 931 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 932                    struct vtn_pointer *dest)
 933 {
 934    if (vtn_pointer_is_external_block(dest)) {
 935       assert(dest->mode == vtn_variable_mode_ssbo);
 936       vtn_block_store(b, src, dest);
 937    } else {
 938       _vtn_variable_load_store(b, false, dest, &src);
 939    }
 940 }
 941
 942 static void
 943 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
 944                    struct vtn_pointer *src)
 945 {
 946    assert(src->type->type == dest->type->type);
 947    enum glsl_base_type base_type = glsl_get_base_type(src->type->type);
 948    switch (base_type) {
 949    case GLSL_TYPE_UINT:
 950    case GLSL_TYPE_INT:
 951    case GLSL_TYPE_UINT64:
 952    case GLSL_TYPE_INT64:
 953    case GLSL_TYPE_FLOAT:
 954    case GLSL_TYPE_DOUBLE:
 955    case GLSL_TYPE_BOOL:
 956       /* At this point, we have a scalar, vector, or matrix so we know that
 957        * there cannot be any structure splitting still in the way.  By
 958        * stopping at the matrix level rather than the vector level, we
 959        * ensure that matrices get loaded in the optimal way even if they
 960        * are storred row-major in a UBO.
 961        */
 962       vtn_variable_store(b, vtn_variable_load(b, src), dest);
 963       return;
 964
 965    case GLSL_TYPE_ARRAY:
 966    case GLSL_TYPE_STRUCT: {
 967       struct vtn_access_chain chain = {
 968          .length = 1,
 969          .link = {
 970             { .mode = vtn_access_mode_literal, },
 971          }
 972       };
 973       unsigned elems = glsl_get_length(src->type->type);
 974       for (unsigned i = 0; i < elems; i++) {
 975          chain.link[0].id = i;
 976          struct vtn_pointer *src_elem =
 977             vtn_pointer_dereference(b, src, &chain);
 978          struct vtn_pointer *dest_elem =
 979             vtn_pointer_dereference(b, dest, &chain);
 980
 981          _vtn_variable_copy(b, dest_elem, src_elem);
 982       }
 983       return;
 984    }
 985
 986    default:
 987       unreachable("Invalid access chain type");
 988    }
 989 }
 990
 991 static void
 992 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
 993                   struct vtn_pointer *src)
 994 {
 995    /* TODO: At some point, we should add a special-case for when we can
 996     * just emit a copy_var intrinsic.
 997     */
 998    _vtn_variable_copy(b, dest, src);
 999 }
1000
1001 static void
1002 set_mode_system_value(nir_variable_mode *mode)
1003 {
1004    assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
1005    *mode = nir_var_system_value;
1006 }
1007
1008 static void
1009 vtn_get_builtin_location(struct vtn_builder *b,
1010                          SpvBuiltIn builtin, int *location,
1011                          nir_variable_mode *mode)
1012 {
1013    switch (builtin) {
1014    case SpvBuiltInPosition:
1015       *location = VARYING_SLOT_POS;
1016       break;
1017    case SpvBuiltInPointSize:
1018       *location = VARYING_SLOT_PSIZ;
1019       break;
1020    case SpvBuiltInClipDistance:
1021       *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
1022       break;
1023    case SpvBuiltInCullDistance:
1024       *location = VARYING_SLOT_CULL_DIST0;
1025       break;
1026    case SpvBuiltInVertexIndex:
1027       *location = SYSTEM_VALUE_VERTEX_ID;
1028       set_mode_system_value(mode);
1029       break;
1030    case SpvBuiltInVertexId:
1031       /* Vulkan defines VertexID to be zero-based and reserves the new
1032        * builtin keyword VertexIndex to indicate the non-zero-based value.
1033        */
1034       *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
1035       set_mode_system_value(mode);
1036       break;
1037    case SpvBuiltInInstanceIndex:
1038       *location = SYSTEM_VALUE_INSTANCE_INDEX;
1039       set_mode_system_value(mode);
1040       break;
1041    case SpvBuiltInInstanceId:
1042       *location = SYSTEM_VALUE_INSTANCE_ID;
1043       set_mode_system_value(mode);
1044       break;
1045    case SpvBuiltInPrimitiveId:
1046       if (b->shader->stage == MESA_SHADER_FRAGMENT) {
1047          assert(*mode == nir_var_shader_in);
1048          *location = VARYING_SLOT_PRIMITIVE_ID;
1049       } else if (*mode == nir_var_shader_out) {
1050          *location = VARYING_SLOT_PRIMITIVE_ID;
1051       } else {
1052          *location = SYSTEM_VALUE_PRIMITIVE_ID;
1053          set_mode_system_value(mode);
1054       }
1055       break;
1056    case SpvBuiltInInvocationId:
1057       *location = SYSTEM_VALUE_INVOCATION_ID;
1058       set_mode_system_value(mode);
1059       break;
1060    case SpvBuiltInLayer:
1061       *location = VARYING_SLOT_LAYER;
1062       if (b->shader->stage == MESA_SHADER_FRAGMENT)
1063          *mode = nir_var_shader_in;
1064       else if (b->shader->stage == MESA_SHADER_GEOMETRY)
1065          *mode = nir_var_shader_out;
1066       else
1067          unreachable("invalid stage for SpvBuiltInLayer");
1068       break;
1069    case SpvBuiltInViewportIndex:
1070       *location = VARYING_SLOT_VIEWPORT;
1071       if (b->shader->stage == MESA_SHADER_GEOMETRY)
1072          *mode = nir_var_shader_out;
1073       else if (b->shader->stage == MESA_SHADER_FRAGMENT)
1074          *mode = nir_var_shader_in;
1075       else
1076          unreachable("invalid stage for SpvBuiltInViewportIndex");
1077       break;
1078    case SpvBuiltInTessLevelOuter:
1079       *location = VARYING_SLOT_TESS_LEVEL_OUTER;
1080       break;
1081    case SpvBuiltInTessLevelInner:
1082       *location = VARYING_SLOT_TESS_LEVEL_INNER;
1083       break;
1084    case SpvBuiltInTessCoord:
1085       *location = SYSTEM_VALUE_TESS_COORD;
1086       set_mode_system_value(mode);
1087       break;
1088    case SpvBuiltInPatchVertices:
1089       *location = SYSTEM_VALUE_VERTICES_IN;
1090       set_mode_system_value(mode);
1091       break;
1092    case SpvBuiltInFragCoord:
1093       *location = VARYING_SLOT_POS;
1094       assert(*mode == nir_var_shader_in);
1095       break;
1096    case SpvBuiltInPointCoord:
1097       *location = VARYING_SLOT_PNTC;
1098       assert(*mode == nir_var_shader_in);
1099       break;
1100    case SpvBuiltInFrontFacing:
1101       *location = SYSTEM_VALUE_FRONT_FACE;
1102       set_mode_system_value(mode);
1103       break;
1104    case SpvBuiltInSampleId:
1105       *location = SYSTEM_VALUE_SAMPLE_ID;
1106       set_mode_system_value(mode);
1107       break;
1108    case SpvBuiltInSamplePosition:
1109       *location = SYSTEM_VALUE_SAMPLE_POS;
1110       set_mode_system_value(mode);
1111       break;
1112    case SpvBuiltInSampleMask:
1113       if (*mode == nir_var_shader_out) {
1114          *location = FRAG_RESULT_SAMPLE_MASK;
1115       } else {
1116          *location = SYSTEM_VALUE_SAMPLE_MASK_IN;
1117          set_mode_system_value(mode);
1118       }
1119       break;
1120    case SpvBuiltInFragDepth:
1121       *location = FRAG_RESULT_DEPTH;
1122       assert(*mode == nir_var_shader_out);
1123       break;
1124    case SpvBuiltInHelperInvocation:
1125       *location = SYSTEM_VALUE_HELPER_INVOCATION;
1126       set_mode_system_value(mode);
1127       break;
1128    case SpvBuiltInNumWorkgroups:
1129       *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
1130       set_mode_system_value(mode);
1131       break;
1132    case SpvBuiltInWorkgroupSize:
1133       /* This should already be handled */
1134       unreachable("unsupported builtin");
1135       break;
1136    case SpvBuiltInWorkgroupId:
1137       *location = SYSTEM_VALUE_WORK_GROUP_ID;
1138       set_mode_system_value(mode);
1139       break;
1140    case SpvBuiltInLocalInvocationId:
1141       *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
1142       set_mode_system_value(mode);
1143       break;
1144    case SpvBuiltInLocalInvocationIndex:
1145       *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1146       set_mode_system_value(mode);
1147       break;
1148    case SpvBuiltInGlobalInvocationId:
1149       *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1150       set_mode_system_value(mode);
1151       break;
1152    case SpvBuiltInBaseVertex:
1153       *location = SYSTEM_VALUE_BASE_VERTEX;
1154       set_mode_system_value(mode);
1155       break;
1156    case SpvBuiltInBaseInstance:
1157       *location = SYSTEM_VALUE_BASE_INSTANCE;
1158       set_mode_system_value(mode);
1159       break;
1160    case SpvBuiltInDrawIndex:
1161       *location = SYSTEM_VALUE_DRAW_ID;
1162       set_mode_system_value(mode);
1163       break;
1164    case SpvBuiltInViewIndex:
1165       *location = SYSTEM_VALUE_VIEW_INDEX;
1166       set_mode_system_value(mode);
1167       break;
1168    default:
1169       unreachable("unsupported builtin");
1170    }
1171 }
1172
1173 static void
1174 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var,
1175                      const struct vtn_decoration *dec)
1176 {
1177    switch (dec->decoration) {
1178    case SpvDecorationRelaxedPrecision:
1179       break; /* FIXME: Do nothing with this for now. */
1180    case SpvDecorationNoPerspective:
1181       nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1182       break;
1183    case SpvDecorationFlat:
1184       nir_var->data.interpolation = INTERP_MODE_FLAT;
1185       break;
1186    case SpvDecorationCentroid:
1187       nir_var->data.centroid = true;
1188       break;
1189    case SpvDecorationSample:
1190       nir_var->data.sample = true;
1191       break;
1192    case SpvDecorationInvariant:
1193       nir_var->data.invariant = true;
1194       break;
1195    case SpvDecorationConstant:
1196       assert(nir_var->constant_initializer != NULL);
1197       nir_var->data.read_only = true;
1198       break;
1199    case SpvDecorationNonReadable:
1200       nir_var->data.image.write_only = true;
1201       break;
1202    case SpvDecorationNonWritable:
1203       nir_var->data.read_only = true;
1204       nir_var->data.image.read_only = true;
1205       break;
1206    case SpvDecorationComponent:
1207       nir_var->data.location_frac = dec->literals[0];
1208       break;
1209    case SpvDecorationIndex:
1210       nir_var->data.index = dec->literals[0];
1211       break;
1212    case SpvDecorationBuiltIn: {
1213       SpvBuiltIn builtin = dec->literals[0];
1214
1215       if (builtin == SpvBuiltInWorkgroupSize) {
1216          /* This shouldn't be a builtin.  It's actually a constant. */
1217          nir_var->data.mode = nir_var_global;
1218          nir_var->data.read_only = true;
1219
1220          nir_constant *c = rzalloc(nir_var, nir_constant);
1221          c->values[0].u32[0] = b->shader->info.cs.local_size[0];
1222          c->values[0].u32[1] = b->shader->info.cs.local_size[1];
1223          c->values[0].u32[2] = b->shader->info.cs.local_size[2];
1224          nir_var->constant_initializer = c;
1225          break;
1226       }
1227
1228       nir_variable_mode mode = nir_var->data.mode;
1229       vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode);
1230       nir_var->data.mode = mode;
1231
1232       switch (builtin) {
1233       case SpvBuiltInTessLevelOuter:
1234       case SpvBuiltInTessLevelInner:
1235          nir_var->data.compact = true;
1236          break;
1237       case SpvBuiltInSamplePosition:
1238          nir_var->data.origin_upper_left = b->origin_upper_left;
1239          /* fallthrough */
1240       case SpvBuiltInFragCoord:
1241          nir_var->data.pixel_center_integer = b->pixel_center_integer;
1242          break;
1243       default:
1244          break;
1245       }
1246    }
1247
1248    case SpvDecorationSpecId:
1249    case SpvDecorationRowMajor:
1250    case SpvDecorationColMajor:
1251    case SpvDecorationMatrixStride:
1252    case SpvDecorationRestrict:
1253    case SpvDecorationAliased:
1254    case SpvDecorationVolatile:
1255    case SpvDecorationCoherent:
1256    case SpvDecorationUniform:
1257    case SpvDecorationStream:
1258    case SpvDecorationOffset:
1259    case SpvDecorationLinkageAttributes:
1260       break; /* Do nothing with these here */
1261
1262    case SpvDecorationPatch:
1263       nir_var->data.patch = true;
1264       break;
1265
1266    case SpvDecorationLocation:
1267       unreachable("Handled above");
1268
1269    case SpvDecorationBlock:
1270    case SpvDecorationBufferBlock:
1271    case SpvDecorationArrayStride:
1272    case SpvDecorationGLSLShared:
1273    case SpvDecorationGLSLPacked:
1274       break; /* These can apply to a type but we don't care about them */
1275
1276    case SpvDecorationBinding:
1277    case SpvDecorationDescriptorSet:
1278    case SpvDecorationNoContraction:
1279    case SpvDecorationInputAttachmentIndex:
1280       vtn_warn("Decoration not allowed for variable or structure member: %s",
1281                spirv_decoration_to_string(dec->decoration));
1282       break;
1283
1284    case SpvDecorationXfbBuffer:
1285    case SpvDecorationXfbStride:
1286       vtn_warn("Vulkan does not have transform feedback: %s",
1287                spirv_decoration_to_string(dec->decoration));
1288       break;
1289
1290    case SpvDecorationCPacked:
1291    case SpvDecorationSaturatedConversion:
1292    case SpvDecorationFuncParamAttr:
1293    case SpvDecorationFPRoundingMode:
1294    case SpvDecorationFPFastMathMode:
1295    case SpvDecorationAlignment:
1296       vtn_warn("Decoration only allowed for CL-style kernels: %s",
1297                spirv_decoration_to_string(dec->decoration));
1298       break;
1299
1300    default:
1301       unreachable("Unhandled decoration");
1302    }
1303 }
1304
1305 static void
1306 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1307                 const struct vtn_decoration *dec, void *out_is_patch)
1308 {
1309    if (dec->decoration == SpvDecorationPatch) {
1310       *((bool *) out_is_patch) = true;
1311    }
1312 }
1313
1314 static void
1315 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1316                   const struct vtn_decoration *dec, void *void_var)
1317 {
1318    struct vtn_variable *vtn_var = void_var;
1319
1320    /* Handle decorations that apply to a vtn_variable as a whole */
1321    switch (dec->decoration) {
1322    case SpvDecorationBinding:
1323       vtn_var->binding = dec->literals[0];
1324       return;
1325    case SpvDecorationDescriptorSet:
1326       vtn_var->descriptor_set = dec->literals[0];
1327       return;
1328    case SpvDecorationInputAttachmentIndex:
1329       vtn_var->input_attachment_index = dec->literals[0];
1330       return;
1331    case SpvDecorationPatch:
1332       vtn_var->patch = true;
1333       break;
1334    default:
1335       break;
1336    }
1337
1338    if (val->value_type == vtn_value_type_pointer) {
1339       assert(val->pointer->var == void_var);
1340       assert(val->pointer->chain == NULL);
1341       assert(member == -1);
1342    } else {
1343       assert(val->value_type == vtn_value_type_type);
1344    }
1345
1346    /* Location is odd.  If applied to a split structure, we have to walk the
1347     * whole thing and accumulate the location.  It's easier to handle as a
1348     * special case.
1349     */
1350    if (dec->decoration == SpvDecorationLocation) {
1351       unsigned location = dec->literals[0];
1352       bool is_vertex_input;
1353       if (b->shader->stage == MESA_SHADER_FRAGMENT &&
1354           vtn_var->mode == vtn_variable_mode_output) {
1355          is_vertex_input = false;
1356          location += FRAG_RESULT_DATA0;
1357       } else if (b->shader->stage == MESA_SHADER_VERTEX &&
1358                  vtn_var->mode == vtn_variable_mode_input) {
1359          is_vertex_input = true;
1360          location += VERT_ATTRIB_GENERIC0;
1361       } else if (vtn_var->mode == vtn_variable_mode_input ||
1362                  vtn_var->mode == vtn_variable_mode_output) {
1363          is_vertex_input = false;
1364          location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1365       } else {
1366          vtn_warn("Location must be on input or output variable");
1367          return;
1368       }
1369
1370       if (vtn_var->var) {
1371          /* This handles the member and lone variable cases */
1372          vtn_var->var->data.location = location;
1373       } else {
1374          /* This handles the structure member case */
1375          assert(vtn_var->members);
1376          unsigned length =
1377             glsl_get_length(glsl_without_array(vtn_var->type->type));
1378          for (unsigned i = 0; i < length; i++) {
1379             vtn_var->members[i]->data.location = location;
1380             location +=
1381                glsl_count_attribute_slots(vtn_var->members[i]->interface_type,
1382                                           is_vertex_input);
1383          }
1384       }
1385       return;
1386    } else {
1387       if (vtn_var->var) {
1388          assert(member <= 0);
1389          apply_var_decoration(b, vtn_var->var, dec);
1390       } else if (vtn_var->members) {
1391          if (member >= 0) {
1392             assert(vtn_var->members);
1393             apply_var_decoration(b, vtn_var->members[member], dec);
1394          } else {
1395             unsigned length =
1396                glsl_get_length(glsl_without_array(vtn_var->type->type));
1397             for (unsigned i = 0; i < length; i++)
1398                apply_var_decoration(b, vtn_var->members[i], dec);
1399          }
1400       } else {
1401          /* A few variables, those with external storage, have no actual
1402           * nir_variables associated with them.  Fortunately, all decorations
1403           * we care about for those variables are on the type only.
1404           */
1405          assert(vtn_var->mode == vtn_variable_mode_ubo ||
1406                 vtn_var->mode == vtn_variable_mode_ssbo ||
1407                 vtn_var->mode == vtn_variable_mode_push_constant);
1408       }
1409    }
1410 }
1411
1412 static enum vtn_variable_mode
1413 vtn_storage_class_to_mode(SpvStorageClass class,
1414                           struct vtn_type *interface_type,
1415                           nir_variable_mode *nir_mode_out)
1416 {
1417    enum vtn_variable_mode mode;
1418    nir_variable_mode nir_mode;
1419    switch (class) {
1420    case SpvStorageClassUniform:
1421       if (interface_type->block) {
1422          mode = vtn_variable_mode_ubo;
1423          nir_mode = 0;
1424       } else if (interface_type->buffer_block) {
1425          mode = vtn_variable_mode_ssbo;
1426          nir_mode = 0;
1427       } else {
1428          unreachable("Invalid uniform variable type");
1429       }
1430       break;
1431    case SpvStorageClassStorageBuffer:
1432       mode = vtn_variable_mode_ssbo;
1433       nir_mode = 0;
1434       break;
1435    case SpvStorageClassUniformConstant:
1436       if (glsl_type_is_image(interface_type->type)) {
1437          mode = vtn_variable_mode_image;
1438          nir_mode = nir_var_uniform;
1439       } else if (glsl_type_is_sampler(interface_type->type)) {
1440          mode = vtn_variable_mode_sampler;
1441          nir_mode = nir_var_uniform;
1442       } else {
1443          unreachable("Invalid uniform constant variable type");
1444       }
1445       break;
1446    case SpvStorageClassPushConstant:
1447       mode = vtn_variable_mode_push_constant;
1448       nir_mode = nir_var_uniform;
1449       break;
1450    case SpvStorageClassInput:
1451       mode = vtn_variable_mode_input;
1452       nir_mode = nir_var_shader_in;
1453       break;
1454    case SpvStorageClassOutput:
1455       mode = vtn_variable_mode_output;
1456       nir_mode = nir_var_shader_out;
1457       break;
1458    case SpvStorageClassPrivate:
1459       mode = vtn_variable_mode_global;
1460       nir_mode = nir_var_global;
1461       break;
1462    case SpvStorageClassFunction:
1463       mode = vtn_variable_mode_local;
1464       nir_mode = nir_var_local;
1465       break;
1466    case SpvStorageClassWorkgroup:
1467       mode = vtn_variable_mode_workgroup;
1468       nir_mode = nir_var_shared;
1469       break;
1470    case SpvStorageClassCrossWorkgroup:
1471    case SpvStorageClassGeneric:
1472    case SpvStorageClassAtomicCounter:
1473    default:
1474       unreachable("Unhandled variable storage class");
1475    }
1476
1477    if (nir_mode_out)
1478       *nir_mode_out = nir_mode;
1479
1480    return mode;
1481 }
1482
1483 nir_ssa_def *
1484 vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr)
1485 {
1486    /* This pointer needs to have a pointer type with actual storage */
1487    assert(ptr->ptr_type);
1488    assert(ptr->ptr_type->type);
1489
1490    if (ptr->offset && ptr->block_index) {
1491       return nir_vec2(&b->nb, ptr->block_index, ptr->offset);
1492    } else {
1493       /* If we don't have an offset or block index, then we must be a pointer
1494        * to the variable itself.
1495        */
1496       assert(!ptr->offset && !ptr->block_index);
1497
1498       /* We can't handle a pointer to an array of descriptors because we have
1499        * no way of knowing later on that we need to add to update the block
1500        * index when dereferencing.
1501        */
1502       assert(ptr->var && ptr->var->type->base_type == vtn_base_type_struct);
1503
1504       return nir_vec2(&b->nb, vtn_variable_resource_index(b, ptr->var, NULL),
1505                               nir_imm_int(&b->nb, 0));
1506    }
1507 }
1508
1509 struct vtn_pointer *
1510 vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa,
1511                      struct vtn_type *ptr_type)
1512 {
1513    assert(ssa->num_components == 2 && ssa->bit_size == 32);
1514    assert(ptr_type->base_type == vtn_base_type_pointer);
1515    assert(ptr_type->deref->base_type != vtn_base_type_pointer);
1516    /* This pointer type needs to have actual storage */
1517    assert(ptr_type->type);
1518
1519    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
1520    ptr->mode = vtn_storage_class_to_mode(ptr_type->storage_class,
1521                                          ptr_type, NULL);
1522    ptr->type = ptr_type->deref;
1523    ptr->ptr_type = ptr_type;
1524    ptr->block_index = nir_channel(&b->nb, ssa, 0);
1525    ptr->offset = nir_channel(&b->nb, ssa, 1);
1526
1527    return ptr;
1528 }
1529
1530 static bool
1531 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
1532 {
1533    if (var->patch || !glsl_type_is_array(var->type->type))
1534       return false;
1535
1536    if (var->mode == vtn_variable_mode_input) {
1537       return stage == MESA_SHADER_TESS_CTRL ||
1538              stage == MESA_SHADER_TESS_EVAL ||
1539              stage == MESA_SHADER_GEOMETRY;
1540    }
1541
1542    if (var->mode == vtn_variable_mode_output)
1543       return stage == MESA_SHADER_TESS_CTRL;
1544
1545    return false;
1546 }
1547
1548 static void
1549 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val,
1550                     struct vtn_type *ptr_type, SpvStorageClass storage_class,
1551                     nir_constant *initializer)
1552 {
1553    assert(ptr_type->base_type == vtn_base_type_pointer);
1554    struct vtn_type *type = ptr_type->deref;
1555
1556    struct vtn_type *without_array = type;
1557    while(glsl_type_is_array(without_array->type))
1558       without_array = without_array->array_element;
1559
1560    enum vtn_variable_mode mode;
1561    nir_variable_mode nir_mode;
1562    mode = vtn_storage_class_to_mode(storage_class, without_array, &nir_mode);
1563
1564    switch (mode) {
1565    case vtn_variable_mode_ubo:
1566       b->shader->info.num_ubos++;
1567       break;
1568    case vtn_variable_mode_ssbo:
1569       b->shader->info.num_ssbos++;
1570       break;
1571    case vtn_variable_mode_image:
1572       b->shader->info.num_images++;
1573       break;
1574    case vtn_variable_mode_sampler:
1575       b->shader->info.num_textures++;
1576       break;
1577    case vtn_variable_mode_push_constant:
1578       b->shader->num_uniforms = vtn_type_block_size(type);
1579       break;
1580    default:
1581       /* No tallying is needed */
1582       break;
1583    }
1584
1585    struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1586    var->type = type;
1587    var->mode = mode;
1588
1589    assert(val->value_type == vtn_value_type_pointer);
1590    val->pointer = vtn_pointer_for_variable(b, var, ptr_type);
1591
1592    switch (var->mode) {
1593    case vtn_variable_mode_local:
1594    case vtn_variable_mode_global:
1595    case vtn_variable_mode_image:
1596    case vtn_variable_mode_sampler:
1597    case vtn_variable_mode_workgroup:
1598       /* For these, we create the variable normally */
1599       var->var = rzalloc(b->shader, nir_variable);
1600       var->var->name = ralloc_strdup(var->var, val->name);
1601       var->var->type = var->type->type;
1602       var->var->data.mode = nir_mode;
1603
1604       switch (var->mode) {
1605       case vtn_variable_mode_image:
1606       case vtn_variable_mode_sampler:
1607          var->var->interface_type = without_array->type;
1608          break;
1609       default:
1610          var->var->interface_type = NULL;
1611          break;
1612       }
1613       break;
1614
1615    case vtn_variable_mode_input:
1616    case vtn_variable_mode_output: {
1617       /* In order to know whether or not we're a per-vertex inout, we need
1618        * the patch qualifier.  This means walking the variable decorations
1619        * early before we actually create any variables.  Not a big deal.
1620        *
1621        * GLSLang really likes to place decorations in the most interior
1622        * thing it possibly can.  In particular, if you have a struct, it
1623        * will place the patch decorations on the struct members.  This
1624        * should be handled by the variable splitting below just fine.
1625        *
1626        * If you have an array-of-struct, things get even more weird as it
1627        * will place the patch decorations on the struct even though it's
1628        * inside an array and some of the members being patch and others not
1629        * makes no sense whatsoever.  Since the only sensible thing is for
1630        * it to be all or nothing, we'll call it patch if any of the members
1631        * are declared patch.
1632        */
1633       var->patch = false;
1634       vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
1635       if (glsl_type_is_array(var->type->type) &&
1636           glsl_type_is_struct(without_array->type)) {
1637          vtn_foreach_decoration(b, without_array->val,
1638                                 var_is_patch_cb, &var->patch);
1639       }
1640
1641       /* For inputs and outputs, we immediately split structures.  This
1642        * is for a couple of reasons.  For one, builtins may all come in
1643        * a struct and we really want those split out into separate
1644        * variables.  For another, interpolation qualifiers can be
1645        * applied to members of the top-level struct ane we need to be
1646        * able to preserve that information.
1647        */
1648
1649       int array_length = -1;
1650       struct vtn_type *interface_type = var->type;
1651       if (is_per_vertex_inout(var, b->shader->stage)) {
1652          /* In Geometry shaders (and some tessellation), inputs come
1653           * in per-vertex arrays.  However, some builtins come in
1654           * non-per-vertex, hence the need for the is_array check.  In
1655           * any case, there are no non-builtin arrays allowed so this
1656           * check should be sufficient.
1657           */
1658          interface_type = var->type->array_element;
1659          array_length = glsl_get_length(var->type->type);
1660       }
1661
1662       if (glsl_type_is_struct(interface_type->type)) {
1663          /* It's a struct.  Split it. */
1664          unsigned num_members = glsl_get_length(interface_type->type);
1665          var->members = ralloc_array(b, nir_variable *, num_members);
1666
1667          for (unsigned i = 0; i < num_members; i++) {
1668             const struct glsl_type *mtype = interface_type->members[i]->type;
1669             if (array_length >= 0)
1670                mtype = glsl_array_type(mtype, array_length);
1671
1672             var->members[i] = rzalloc(b->shader, nir_variable);
1673             var->members[i]->name =
1674                ralloc_asprintf(var->members[i], "%s.%d", val->name, i);
1675             var->members[i]->type = mtype;
1676             var->members[i]->interface_type =
1677                interface_type->members[i]->type;
1678             var->members[i]->data.mode = nir_mode;
1679             var->members[i]->data.patch = var->patch;
1680          }
1681       } else {
1682          var->var = rzalloc(b->shader, nir_variable);
1683          var->var->name = ralloc_strdup(var->var, val->name);
1684          var->var->type = var->type->type;
1685          var->var->interface_type = interface_type->type;
1686          var->var->data.mode = nir_mode;
1687          var->var->data.patch = var->patch;
1688       }
1689
1690       /* For inputs and outputs, we need to grab locations and builtin
1691        * information from the interface type.
1692        */
1693       vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var);
1694       break;
1695    }
1696
1697    case vtn_variable_mode_param:
1698       unreachable("Not created through OpVariable");
1699
1700    case vtn_variable_mode_ubo:
1701    case vtn_variable_mode_ssbo:
1702    case vtn_variable_mode_push_constant:
1703       /* These don't need actual variables. */
1704       break;
1705    }
1706
1707    if (initializer) {
1708       var->var->constant_initializer =
1709          nir_constant_clone(initializer, var->var);
1710    }
1711
1712    vtn_foreach_decoration(b, val, var_decoration_cb, var);
1713
1714    if (var->mode == vtn_variable_mode_image ||
1715        var->mode == vtn_variable_mode_sampler) {
1716       /* XXX: We still need the binding information in the nir_variable
1717        * for these. We should fix that.
1718        */
1719       var->var->data.binding = var->binding;
1720       var->var->data.descriptor_set = var->descriptor_set;
1721       var->var->data.index = var->input_attachment_index;
1722
1723       if (var->mode == vtn_variable_mode_image)
1724          var->var->data.image.format = without_array->image_format;
1725    }
1726
1727    if (var->mode == vtn_variable_mode_local) {
1728       assert(var->members == NULL && var->var != NULL);
1729       nir_function_impl_add_variable(b->impl, var->var);
1730    } else if (var->var) {
1731       nir_shader_add_variable(b->shader, var->var);
1732    } else if (var->members) {
1733       unsigned count = glsl_get_length(without_array->type);
1734       for (unsigned i = 0; i < count; i++) {
1735          assert(var->members[i]->data.mode != nir_var_local);
1736          nir_shader_add_variable(b->shader, var->members[i]);
1737       }
1738    } else {
1739       assert(var->mode == vtn_variable_mode_ubo ||
1740              var->mode == vtn_variable_mode_ssbo ||
1741              var->mode == vtn_variable_mode_push_constant);
1742    }
1743 }
1744
1745 void
1746 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1747                      const uint32_t *w, unsigned count)
1748 {
1749    switch (opcode) {
1750    case SpvOpUndef: {
1751       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef);
1752       val->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1753       break;
1754    }
1755
1756    case SpvOpVariable: {
1757       struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type;
1758
1759       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer);
1760
1761       SpvStorageClass storage_class = w[3];
1762       nir_constant *initializer = NULL;
1763       if (count > 4)
1764          initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant;
1765
1766       vtn_create_variable(b, val, ptr_type, storage_class, initializer);
1767       break;
1768    }
1769
1770    case SpvOpAccessChain:
1771    case SpvOpPtrAccessChain:
1772    case SpvOpInBoundsAccessChain: {
1773       struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4);
1774       chain->ptr_as_array = (opcode == SpvOpPtrAccessChain);
1775
1776       unsigned idx = 0;
1777       for (int i = 4; i < count; i++) {
1778          struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1779          if (link_val->value_type == vtn_value_type_constant) {
1780             chain->link[idx].mode = vtn_access_mode_literal;
1781             chain->link[idx].id = link_val->constant->values[0].u32[0];
1782          } else {
1783             chain->link[idx].mode = vtn_access_mode_id;
1784             chain->link[idx].id = w[i];
1785
1786          }
1787          idx++;
1788       }
1789
1790       struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type;
1791       struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1792       if (base_val->value_type == vtn_value_type_sampled_image) {
1793          /* This is rather insane.  SPIR-V allows you to use OpSampledImage
1794           * to combine an array of images with a single sampler to get an
1795           * array of sampled images that all share the same sampler.
1796           * Fortunately, this means that we can more-or-less ignore the
1797           * sampler when crawling the access chain, but it does leave us
1798           * with this rather awkward little special-case.
1799           */
1800          struct vtn_value *val =
1801             vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1802          val->sampled_image = ralloc(b, struct vtn_sampled_image);
1803          val->sampled_image->image =
1804             vtn_pointer_dereference(b, base_val->sampled_image->image, chain);
1805          val->sampled_image->sampler = base_val->sampled_image->sampler;
1806       } else {
1807          assert(base_val->value_type == vtn_value_type_pointer);
1808          struct vtn_value *val =
1809             vtn_push_value(b, w[2], vtn_value_type_pointer);
1810          val->pointer = vtn_pointer_dereference(b, base_val->pointer, chain);
1811          val->pointer->ptr_type = ptr_type;
1812       }
1813       break;
1814    }
1815
1816    case SpvOpCopyMemory: {
1817       struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer);
1818       struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer);
1819
1820       vtn_variable_copy(b, dest->pointer, src->pointer);
1821       break;
1822    }
1823
1824    case SpvOpLoad: {
1825       struct vtn_type *res_type =
1826          vtn_value(b, w[1], vtn_value_type_type)->type;
1827       struct vtn_pointer *src =
1828          vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
1829
1830       if (src->mode == vtn_variable_mode_image ||
1831           src->mode == vtn_variable_mode_sampler) {
1832          vtn_push_value(b, w[2], vtn_value_type_pointer)->pointer = src;
1833          return;
1834       }
1835
1836       vtn_push_ssa(b, w[2], res_type, vtn_variable_load(b, src));
1837       break;
1838    }
1839
1840    case SpvOpStore: {
1841       struct vtn_pointer *dest =
1842          vtn_value(b, w[1], vtn_value_type_pointer)->pointer;
1843
1844       if (glsl_type_is_sampler(dest->type->type)) {
1845          vtn_warn("OpStore of a sampler detected.  Doing on-the-fly copy "
1846                   "propagation to workaround the problem.");
1847          assert(dest->var->copy_prop_sampler == NULL);
1848          dest->var->copy_prop_sampler =
1849             vtn_value(b, w[2], vtn_value_type_pointer)->pointer;
1850          break;
1851       }
1852
1853       struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1854       vtn_variable_store(b, src, dest);
1855       break;
1856    }
1857
1858    case SpvOpArrayLength: {
1859       struct vtn_pointer *ptr =
1860          vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
1861
1862       const uint32_t offset = ptr->var->type->offsets[w[4]];
1863       const uint32_t stride = ptr->var->type->members[w[4]]->stride;
1864
1865       unsigned chain_idx;
1866       struct vtn_type *type;
1867       nir_ssa_def *index =
1868          get_vulkan_resource_index(b, ptr, &type, &chain_idx);
1869
1870       nir_intrinsic_instr *instr =
1871          nir_intrinsic_instr_create(b->nb.shader,
1872                                     nir_intrinsic_get_buffer_size);
1873       instr->src[0] = nir_src_for_ssa(index);
1874       nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1875       nir_builder_instr_insert(&b->nb, &instr->instr);
1876       nir_ssa_def *buf_size = &instr->dest.ssa;
1877
1878       /* array_length = max(buffer_size - offset, 0) / stride */
1879       nir_ssa_def *array_length =
1880          nir_idiv(&b->nb,
1881                   nir_imax(&b->nb,
1882                            nir_isub(&b->nb,
1883                                     buf_size,
1884                                     nir_imm_int(&b->nb, offset)),
1885                            nir_imm_int(&b->nb, 0u)),
1886                   nir_imm_int(&b->nb, stride));
1887
1888       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1889       val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
1890       val->ssa->def = array_length;
1891       break;
1892    }
1893
1894    case SpvOpCopyMemorySized:
1895    default:
1896       unreachable("Unhandled opcode");
1897    }
1898 }