src/compiler/spirv/vtn_variables.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 #include "vtn_private.h"
  29 #include "spirv_info.h"
  30
  31 static struct vtn_access_chain *
  32 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
  33                         unsigned new_ids)
  34 {
  35    struct vtn_access_chain *chain;
  36
  37    unsigned new_len = old->length + new_ids;
  38    /* TODO: don't use rzalloc */
  39    chain = rzalloc_size(b, sizeof(*chain) + new_len * sizeof(chain->link[0]));
  40
  41    chain->var = old->var;
  42    chain->length = new_len;
  43
  44    for (unsigned i = 0; i < old->length; i++)
  45       chain->link[i] = old->link[i];
  46
  47    return chain;
  48 }
  49
  50 static nir_ssa_def *
  51 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
  52                        unsigned stride)
  53 {
  54    assert(stride > 0);
  55    if (link.mode == vtn_access_mode_literal) {
  56       return nir_imm_int(&b->nb, link.id * stride);
  57    } else if (stride == 1) {
  58       return vtn_ssa_value(b, link.id)->def;
  59    } else {
  60       return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
  61                               nir_imm_int(&b->nb, stride));
  62    }
  63 }
  64
  65 static struct vtn_type *
  66 vtn_access_chain_tail_type(struct vtn_builder *b,
  67                            struct vtn_access_chain *chain)
  68 {
  69    struct vtn_type *type = chain->var->type;
  70    for (unsigned i = 0; i < chain->length; i++) {
  71       if (glsl_type_is_struct(type->type)) {
  72          assert(chain->link[i].mode == vtn_access_mode_literal);
  73          type = type->members[chain->link[i].id];
  74       } else {
  75          type = type->array_element;
  76       }
  77    }
  78    return type;
  79 }
  80
  81 /* Crawls a chain of array derefs and rewrites the types so that the
  82  * lengths stay the same but the terminal type is the one given by
  83  * tail_type.  This is useful for split structures.
  84  */
  85 static void
  86 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type)
  87 {
  88    deref->type = type;
  89    if (deref->child) {
  90       assert(deref->child->deref_type == nir_deref_type_array);
  91       assert(glsl_type_is_array(deref->type));
  92       rewrite_deref_types(deref->child, glsl_get_array_element(type));
  93    }
  94 }
  95
  96 nir_deref_var *
  97 vtn_access_chain_to_deref(struct vtn_builder *b, struct vtn_access_chain *chain)
  98 {
  99    nir_deref_var *deref_var;
 100    if (chain->var->var) {
 101       deref_var = nir_deref_var_create(b, chain->var->var);
 102    } else {
 103       assert(chain->var->members);
 104       /* Create the deref_var manually.  It will get filled out later. */
 105       deref_var = rzalloc(b, nir_deref_var);
 106       deref_var->deref.deref_type = nir_deref_type_var;
 107    }
 108
 109    struct vtn_type *deref_type = chain->var->type;
 110    nir_deref *tail = &deref_var->deref;
 111    nir_variable **members = chain->var->members;
 112
 113    for (unsigned i = 0; i < chain->length; i++) {
 114       enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
 115       switch (base_type) {
 116       case GLSL_TYPE_UINT:
 117       case GLSL_TYPE_INT:
 118       case GLSL_TYPE_FLOAT:
 119       case GLSL_TYPE_DOUBLE:
 120       case GLSL_TYPE_BOOL:
 121       case GLSL_TYPE_ARRAY: {
 122          deref_type = deref_type->array_element;
 123
 124          nir_deref_array *deref_arr = nir_deref_array_create(b);
 125          deref_arr->deref.type = deref_type->type;
 126
 127          if (chain->link[i].mode == vtn_access_mode_literal) {
 128             deref_arr->deref_array_type = nir_deref_array_type_direct;
 129             deref_arr->base_offset = chain->link[i].id;
 130          } else {
 131             assert(chain->link[i].mode == vtn_access_mode_id);
 132             deref_arr->deref_array_type = nir_deref_array_type_indirect;
 133             deref_arr->base_offset = 0;
 134             deref_arr->indirect =
 135                nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
 136          }
 137          tail->child = &deref_arr->deref;
 138          tail = tail->child;
 139          break;
 140       }
 141
 142       case GLSL_TYPE_STRUCT: {
 143          assert(chain->link[i].mode == vtn_access_mode_literal);
 144          unsigned idx = chain->link[i].id;
 145          deref_type = deref_type->members[idx];
 146          if (members) {
 147             /* This is a pre-split structure. */
 148             deref_var->var = members[idx];
 149             rewrite_deref_types(&deref_var->deref, members[idx]->type);
 150             assert(tail->type == deref_type->type);
 151             members = NULL;
 152          } else {
 153             nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
 154             deref_struct->deref.type = deref_type->type;
 155             tail->child = &deref_struct->deref;
 156             tail = tail->child;
 157          }
 158          break;
 159       }
 160       default:
 161          unreachable("Invalid type for deref");
 162       }
 163    }
 164
 165    assert(members == NULL);
 166    return deref_var;
 167 }
 168
 169 static void
 170 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref,
 171                       nir_deref *tail, struct vtn_ssa_value *inout)
 172 {
 173    /* The deref tail may contain a deref to select a component of a vector (in
 174     * other words, it might not be an actual tail) so we have to save it away
 175     * here since we overwrite it later.
 176     */
 177    nir_deref *old_child = tail->child;
 178
 179    if (glsl_type_is_vector_or_scalar(tail->type)) {
 180       /* Terminate the deref chain in case there is one more link to pick
 181        * off a component of the vector.
 182        */
 183       tail->child = NULL;
 184
 185       nir_intrinsic_op op = load ? nir_intrinsic_load_var :
 186                                    nir_intrinsic_store_var;
 187
 188       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op);
 189       intrin->variables[0] = nir_deref_var_clone(deref, intrin);
 190       intrin->num_components = glsl_get_vector_elements(tail->type);
 191
 192       if (load) {
 193          nir_ssa_dest_init(&intrin->instr, &intrin->dest,
 194                            intrin->num_components,
 195                            glsl_get_bit_size(tail->type),
 196                            NULL);
 197          inout->def = &intrin->dest.ssa;
 198       } else {
 199          nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1);
 200          intrin->src[0] = nir_src_for_ssa(inout->def);
 201       }
 202
 203       nir_builder_instr_insert(&b->nb, &intrin->instr);
 204    } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY ||
 205               glsl_type_is_matrix(tail->type)) {
 206       unsigned elems = glsl_get_length(tail->type);
 207       nir_deref_array *deref_arr = nir_deref_array_create(b);
 208       deref_arr->deref_array_type = nir_deref_array_type_direct;
 209       deref_arr->deref.type = glsl_get_array_element(tail->type);
 210       tail->child = &deref_arr->deref;
 211       for (unsigned i = 0; i < elems; i++) {
 212          deref_arr->base_offset = i;
 213          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 214       }
 215    } else {
 216       assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT);
 217       unsigned elems = glsl_get_length(tail->type);
 218       nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0);
 219       tail->child = &deref_struct->deref;
 220       for (unsigned i = 0; i < elems; i++) {
 221          deref_struct->index = i;
 222          deref_struct->deref.type = glsl_get_struct_field(tail->type, i);
 223          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 224       }
 225    }
 226
 227    tail->child = old_child;
 228 }
 229
 230 nir_deref_var *
 231 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
 232 {
 233    struct vtn_access_chain *chain =
 234       vtn_value(b, id, vtn_value_type_access_chain)->access_chain;
 235
 236    return vtn_access_chain_to_deref(b, chain);
 237 }
 238
 239 /*
 240  * Gets the NIR-level deref tail, which may have as a child an array deref
 241  * selecting which component due to OpAccessChain supporting per-component
 242  * indexing in SPIR-V.
 243  */
 244 static nir_deref *
 245 get_deref_tail(nir_deref_var *deref)
 246 {
 247    nir_deref *cur = &deref->deref;
 248    while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child)
 249       cur = cur->child;
 250
 251    return cur;
 252 }
 253
 254 struct vtn_ssa_value *
 255 vtn_local_load(struct vtn_builder *b, nir_deref_var *src)
 256 {
 257    nir_deref *src_tail = get_deref_tail(src);
 258    struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
 259    _vtn_local_load_store(b, true, src, src_tail, val);
 260
 261    if (src_tail->child) {
 262       nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child);
 263       assert(vec_deref->deref.child == NULL);
 264       val->type = vec_deref->deref.type;
 265       if (vec_deref->deref_array_type == nir_deref_array_type_direct)
 266          val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset);
 267       else
 268          val->def = vtn_vector_extract_dynamic(b, val->def,
 269                                                vec_deref->indirect.ssa);
 270    }
 271
 272    return val;
 273 }
 274
 275 void
 276 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 277                 nir_deref_var *dest)
 278 {
 279    nir_deref *dest_tail = get_deref_tail(dest);
 280
 281    if (dest_tail->child) {
 282       struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
 283       _vtn_local_load_store(b, true, dest, dest_tail, val);
 284       nir_deref_array *deref = nir_deref_as_array(dest_tail->child);
 285       assert(deref->deref.child == NULL);
 286       if (deref->deref_array_type == nir_deref_array_type_direct)
 287          val->def = vtn_vector_insert(b, val->def, src->def,
 288                                       deref->base_offset);
 289       else
 290          val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
 291                                               deref->indirect.ssa);
 292       _vtn_local_load_store(b, false, dest, dest_tail, val);
 293    } else {
 294       _vtn_local_load_store(b, false, dest, dest_tail, src);
 295    }
 296 }
 297
 298 static nir_ssa_def *
 299 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain,
 300                           struct vtn_type **type, unsigned *chain_idx)
 301 {
 302    /* Push constants have no explicit binding */
 303    if (chain->var->mode == vtn_variable_mode_push_constant) {
 304       *chain_idx = 0;
 305       *type = chain->var->type;
 306       return NULL;
 307    }
 308
 309    nir_ssa_def *array_index;
 310    if (glsl_type_is_array(chain->var->type->type)) {
 311       assert(chain->length > 0);
 312       array_index = vtn_access_link_as_ssa(b, chain->link[0], 1);
 313       *chain_idx = 1;
 314       *type = chain->var->type->array_element;
 315    } else {
 316       array_index = nir_imm_int(&b->nb, 0);
 317       *chain_idx = 0;
 318       *type = chain->var->type;
 319    }
 320
 321    nir_intrinsic_instr *instr =
 322       nir_intrinsic_instr_create(b->nb.shader,
 323                                  nir_intrinsic_vulkan_resource_index);
 324    instr->src[0] = nir_src_for_ssa(array_index);
 325    nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set);
 326    nir_intrinsic_set_binding(instr, chain->var->binding);
 327
 328    nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 329    nir_builder_instr_insert(&b->nb, &instr->instr);
 330
 331    return &instr->dest.ssa;
 332 }
 333
 334 nir_ssa_def *
 335 vtn_access_chain_to_offset(struct vtn_builder *b,
 336                            struct vtn_access_chain *chain,
 337                            nir_ssa_def **index_out, struct vtn_type **type_out,
 338                            unsigned *end_idx_out, bool stop_at_matrix)
 339 {
 340    unsigned idx = 0;
 341    struct vtn_type *type;
 342    *index_out = get_vulkan_resource_index(b, chain, &type, &idx);
 343
 344    nir_ssa_def *offset = nir_imm_int(&b->nb, 0);
 345    for (; idx < chain->length; idx++) {
 346       enum glsl_base_type base_type = glsl_get_base_type(type->type);
 347       switch (base_type) {
 348       case GLSL_TYPE_UINT:
 349       case GLSL_TYPE_INT:
 350       case GLSL_TYPE_FLOAT:
 351       case GLSL_TYPE_DOUBLE:
 352       case GLSL_TYPE_BOOL:
 353          /* Some users may not want matrix or vector derefs */
 354          if (stop_at_matrix)
 355             goto end;
 356          /* Fall through */
 357
 358       case GLSL_TYPE_ARRAY:
 359          offset = nir_iadd(&b->nb, offset,
 360                            vtn_access_link_as_ssa(b, chain->link[idx],
 361                                                   type->stride));
 362
 363          type = type->array_element;
 364          break;
 365
 366       case GLSL_TYPE_STRUCT: {
 367          assert(chain->link[idx].mode == vtn_access_mode_literal);
 368          unsigned member = chain->link[idx].id;
 369          offset = nir_iadd(&b->nb, offset,
 370                            nir_imm_int(&b->nb, type->offsets[member]));
 371          type = type->members[member];
 372          break;
 373       }
 374
 375       default:
 376          unreachable("Invalid type for deref");
 377       }
 378    }
 379
 380 end:
 381    *type_out = type;
 382    if (end_idx_out)
 383       *end_idx_out = idx;
 384
 385    return offset;
 386 }
 387
 388 /* Tries to compute the size of an interface block based on the strides and
 389  * offsets that are provided to us in the SPIR-V source.
 390  */
 391 static unsigned
 392 vtn_type_block_size(struct vtn_type *type)
 393 {
 394    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 395    switch (base_type) {
 396    case GLSL_TYPE_UINT:
 397    case GLSL_TYPE_INT:
 398    case GLSL_TYPE_FLOAT:
 399    case GLSL_TYPE_BOOL:
 400    case GLSL_TYPE_DOUBLE: {
 401       unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
 402                                         glsl_get_matrix_columns(type->type);
 403       if (cols > 1) {
 404          assert(type->stride > 0);
 405          return type->stride * cols;
 406       } else if (base_type == GLSL_TYPE_DOUBLE) {
 407          return glsl_get_vector_elements(type->type) * 8;
 408       } else {
 409          return glsl_get_vector_elements(type->type) * 4;
 410       }
 411    }
 412
 413    case GLSL_TYPE_STRUCT:
 414    case GLSL_TYPE_INTERFACE: {
 415       unsigned size = 0;
 416       unsigned num_fields = glsl_get_length(type->type);
 417       for (unsigned f = 0; f < num_fields; f++) {
 418          unsigned field_end = type->offsets[f] +
 419                               vtn_type_block_size(type->members[f]);
 420          size = MAX2(size, field_end);
 421       }
 422       return size;
 423    }
 424
 425    case GLSL_TYPE_ARRAY:
 426       assert(type->stride > 0);
 427       assert(glsl_get_length(type->type) > 0);
 428       return type->stride * glsl_get_length(type->type);
 429
 430    default:
 431       assert(!"Invalid block type");
 432       return 0;
 433    }
 434 }
 435
 436 static void
 437 vtn_access_chain_get_offset_size(struct vtn_access_chain *chain,
 438                                  unsigned *access_offset,
 439                                  unsigned *access_size)
 440 {
 441    /* Only valid for push constants accesses now. */
 442    assert(chain->var->mode == vtn_variable_mode_push_constant);
 443
 444    struct vtn_type *type = chain->var->type;
 445
 446    *access_offset = 0;
 447
 448    for (unsigned i = 0; i < chain->length; i++) {
 449       if (chain->link[i].mode != vtn_access_mode_literal)
 450          break;
 451
 452       if (glsl_type_is_struct(type->type)) {
 453          *access_offset += type->offsets[chain->link[i].id];
 454          type = type->members[chain->link[i].id];
 455       } else {
 456          *access_offset += type->stride * chain->link[i].id;
 457          type = type->array_element;
 458       }
 459    }
 460
 461    *access_size = vtn_type_block_size(type);
 462 }
 463
 464 static void
 465 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 466                      nir_ssa_def *index, nir_ssa_def *offset,
 467                      unsigned access_offset, unsigned access_size,
 468                      struct vtn_ssa_value **inout, const struct glsl_type *type)
 469 {
 470    nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
 471    instr->num_components = glsl_get_vector_elements(type);
 472
 473    int src = 0;
 474    if (!load) {
 475       nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
 476       instr->src[src++] = nir_src_for_ssa((*inout)->def);
 477    }
 478
 479    if (op == nir_intrinsic_load_push_constant) {
 480       assert(access_offset % 4 == 0);
 481
 482       nir_intrinsic_set_base(instr, access_offset);
 483       nir_intrinsic_set_range(instr, access_size);
 484    }
 485
 486    if (index)
 487       instr->src[src++] = nir_src_for_ssa(index);
 488
 489    if (op == nir_intrinsic_load_push_constant) {
 490       /* We need to subtract the offset from where the intrinsic will load the
 491        * data. */
 492       instr->src[src++] =
 493          nir_src_for_ssa(nir_isub(&b->nb, offset,
 494                                   nir_imm_int(&b->nb, access_offset)));
 495    } else {
 496       instr->src[src++] = nir_src_for_ssa(offset);
 497    }
 498
 499    if (load) {
 500       nir_ssa_dest_init(&instr->instr, &instr->dest,
 501                         instr->num_components,
 502                         glsl_get_bit_size(type), NULL);
 503       (*inout)->def = &instr->dest.ssa;
 504    }
 505
 506    nir_builder_instr_insert(&b->nb, &instr->instr);
 507
 508    if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
 509       (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
 510 }
 511
 512 static void
 513 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 514                       nir_ssa_def *index, nir_ssa_def *offset,
 515                       unsigned access_offset, unsigned access_size,
 516                       struct vtn_access_chain *chain, unsigned chain_idx,
 517                       struct vtn_type *type, struct vtn_ssa_value **inout)
 518 {
 519    if (chain && chain_idx >= chain->length)
 520       chain = NULL;
 521
 522    if (load && chain == NULL && *inout == NULL)
 523       *inout = vtn_create_ssa_value(b, type->type);
 524
 525    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 526    switch (base_type) {
 527    case GLSL_TYPE_UINT:
 528    case GLSL_TYPE_INT:
 529    case GLSL_TYPE_FLOAT:
 530    case GLSL_TYPE_DOUBLE:
 531    case GLSL_TYPE_BOOL:
 532       /* This is where things get interesting.  At this point, we've hit
 533        * a vector, a scalar, or a matrix.
 534        */
 535       if (glsl_type_is_matrix(type->type)) {
 536          if (chain == NULL) {
 537             /* Loading the whole matrix */
 538             struct vtn_ssa_value *transpose;
 539             unsigned num_ops, vec_width;
 540             if (type->row_major) {
 541                num_ops = glsl_get_vector_elements(type->type);
 542                vec_width = glsl_get_matrix_columns(type->type);
 543                if (load) {
 544                   const struct glsl_type *transpose_type =
 545                      glsl_matrix_type(base_type, vec_width, num_ops);
 546                   *inout = vtn_create_ssa_value(b, transpose_type);
 547                } else {
 548                   transpose = vtn_ssa_transpose(b, *inout);
 549                   inout = &transpose;
 550                }
 551             } else {
 552                num_ops = glsl_get_matrix_columns(type->type);
 553                vec_width = glsl_get_vector_elements(type->type);
 554             }
 555
 556             for (unsigned i = 0; i < num_ops; i++) {
 557                nir_ssa_def *elem_offset =
 558                   nir_iadd(&b->nb, offset,
 559                            nir_imm_int(&b->nb, i * type->stride));
 560                _vtn_load_store_tail(b, op, load, index, elem_offset,
 561                                     access_offset, access_size,
 562                                     &(*inout)->elems[i],
 563                                     glsl_vector_type(base_type, vec_width));
 564             }
 565
 566             if (load && type->row_major)
 567                *inout = vtn_ssa_transpose(b, *inout);
 568          } else if (type->row_major) {
 569             /* Row-major but with an access chiain. */
 570             nir_ssa_def *col_offset =
 571                vtn_access_link_as_ssa(b, chain->link[chain_idx],
 572                                       type->array_element->stride);
 573             offset = nir_iadd(&b->nb, offset, col_offset);
 574
 575             if (chain_idx + 1 < chain->length) {
 576                /* Picking off a single element */
 577                nir_ssa_def *row_offset =
 578                   vtn_access_link_as_ssa(b, chain->link[chain_idx + 1],
 579                                          type->stride);
 580                offset = nir_iadd(&b->nb, offset, row_offset);
 581                if (load)
 582                   *inout = vtn_create_ssa_value(b, glsl_scalar_type(base_type));
 583                _vtn_load_store_tail(b, op, load, index, offset,
 584                                     access_offset, access_size,
 585                                     inout, glsl_scalar_type(base_type));
 586             } else {
 587                /* Grabbing a column; picking one element off each row */
 588                unsigned num_comps = glsl_get_vector_elements(type->type);
 589                const struct glsl_type *column_type =
 590                   glsl_get_column_type(type->type);
 591
 592                nir_ssa_def *comps[4];
 593                for (unsigned i = 0; i < num_comps; i++) {
 594                   nir_ssa_def *elem_offset =
 595                      nir_iadd(&b->nb, offset,
 596                               nir_imm_int(&b->nb, i * type->stride));
 597
 598                   struct vtn_ssa_value *comp, temp_val;
 599                   if (!load) {
 600                      temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
 601                      temp_val.type = glsl_scalar_type(base_type);
 602                   }
 603                   comp = &temp_val;
 604                   _vtn_load_store_tail(b, op, load, index, elem_offset,
 605                                        access_offset, access_size,
 606                                        &comp, glsl_scalar_type(base_type));
 607                   comps[i] = comp->def;
 608                }
 609
 610                if (load) {
 611                   if (*inout == NULL)
 612                      *inout = vtn_create_ssa_value(b, column_type);
 613
 614                   (*inout)->def = nir_vec(&b->nb, comps, num_comps);
 615                }
 616             }
 617          } else {
 618             /* Column-major with a deref. Fall through to array case. */
 619             nir_ssa_def *col_offset =
 620                vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
 621             offset = nir_iadd(&b->nb, offset, col_offset);
 622
 623             _vtn_block_load_store(b, op, load, index, offset,
 624                                   access_offset, access_size,
 625                                   chain, chain_idx + 1,
 626                                   type->array_element, inout);
 627          }
 628       } else if (chain == NULL) {
 629          /* Single whole vector */
 630          assert(glsl_type_is_vector_or_scalar(type->type));
 631          _vtn_load_store_tail(b, op, load, index, offset,
 632                               access_offset, access_size,
 633                               inout, type->type);
 634       } else {
 635          /* Single component of a vector. Fall through to array case. */
 636          nir_ssa_def *elem_offset =
 637             vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
 638          offset = nir_iadd(&b->nb, offset, elem_offset);
 639
 640          _vtn_block_load_store(b, op, load, index, offset,
 641                                access_offset, access_size,
 642                                NULL, 0,
 643                                type->array_element, inout);
 644       }
 645       return;
 646
 647    case GLSL_TYPE_ARRAY: {
 648       unsigned elems = glsl_get_length(type->type);
 649       for (unsigned i = 0; i < elems; i++) {
 650          nir_ssa_def *elem_off =
 651             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
 652          _vtn_block_load_store(b, op, load, index, elem_off,
 653                                access_offset, access_size,
 654                                NULL, 0,
 655                                type->array_element, &(*inout)->elems[i]);
 656       }
 657       return;
 658    }
 659
 660    case GLSL_TYPE_STRUCT: {
 661       unsigned elems = glsl_get_length(type->type);
 662       for (unsigned i = 0; i < elems; i++) {
 663          nir_ssa_def *elem_off =
 664             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
 665          _vtn_block_load_store(b, op, load, index, elem_off,
 666                                access_offset, access_size,
 667                                NULL, 0,
 668                                type->members[i], &(*inout)->elems[i]);
 669       }
 670       return;
 671    }
 672
 673    default:
 674       unreachable("Invalid block member type");
 675    }
 676 }
 677
 678 static struct vtn_ssa_value *
 679 vtn_block_load(struct vtn_builder *b, struct vtn_access_chain *src)
 680 {
 681    nir_intrinsic_op op;
 682    unsigned access_offset = 0, access_size = 0;
 683    switch (src->var->mode) {
 684    case vtn_variable_mode_ubo:
 685       op = nir_intrinsic_load_ubo;
 686       break;
 687    case vtn_variable_mode_ssbo:
 688       op = nir_intrinsic_load_ssbo;
 689       break;
 690    case vtn_variable_mode_push_constant:
 691       op = nir_intrinsic_load_push_constant;
 692       vtn_access_chain_get_offset_size(src, &access_offset, &access_size);
 693       break;
 694    default:
 695       assert(!"Invalid block variable mode");
 696    }
 697
 698    nir_ssa_def *offset, *index = NULL;
 699    struct vtn_type *type;
 700    unsigned chain_idx;
 701    offset = vtn_access_chain_to_offset(b, src, &index, &type, &chain_idx, true);
 702
 703    struct vtn_ssa_value *value = NULL;
 704    _vtn_block_load_store(b, op, true, index, offset,
 705                          access_offset, access_size,
 706                          src, chain_idx, type, &value);
 707    return value;
 708 }
 709
 710 static void
 711 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 712                 struct vtn_access_chain *dst)
 713 {
 714    nir_ssa_def *offset, *index = NULL;
 715    struct vtn_type *type;
 716    unsigned chain_idx;
 717    offset = vtn_access_chain_to_offset(b, dst, &index, &type, &chain_idx, true);
 718
 719    _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset,
 720                          0, 0, dst, chain_idx, type, &src);
 721 }
 722
 723 static bool
 724 vtn_variable_is_external_block(struct vtn_variable *var)
 725 {
 726    return var->mode == vtn_variable_mode_ssbo ||
 727           var->mode == vtn_variable_mode_ubo ||
 728           var->mode == vtn_variable_mode_push_constant;
 729 }
 730
 731 static void
 732 _vtn_variable_load_store(struct vtn_builder *b, bool load,
 733                          struct vtn_access_chain *chain,
 734                          struct vtn_type *tail_type,
 735                          struct vtn_ssa_value **inout)
 736 {
 737    enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
 738    switch (base_type) {
 739    case GLSL_TYPE_UINT:
 740    case GLSL_TYPE_INT:
 741    case GLSL_TYPE_FLOAT:
 742    case GLSL_TYPE_BOOL:
 743    case GLSL_TYPE_DOUBLE:
 744       /* At this point, we have a scalar, vector, or matrix so we know that
 745        * there cannot be any structure splitting still in the way.  By
 746        * stopping at the matrix level rather than the vector level, we
 747        * ensure that matrices get loaded in the optimal way even if they
 748        * are storred row-major in a UBO.
 749        */
 750       if (load) {
 751          *inout = vtn_local_load(b, vtn_access_chain_to_deref(b, chain));
 752       } else {
 753          vtn_local_store(b, *inout, vtn_access_chain_to_deref(b, chain));
 754       }
 755       return;
 756
 757    case GLSL_TYPE_ARRAY:
 758    case GLSL_TYPE_STRUCT: {
 759       struct vtn_access_chain *new_chain =
 760          vtn_access_chain_extend(b, chain, 1);
 761       new_chain->link[chain->length].mode = vtn_access_mode_literal;
 762       unsigned elems = glsl_get_length(tail_type->type);
 763       if (load) {
 764          assert(*inout == NULL);
 765          *inout = rzalloc(b, struct vtn_ssa_value);
 766          (*inout)->type = tail_type->type;
 767          (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
 768       }
 769       for (unsigned i = 0; i < elems; i++) {
 770          new_chain->link[chain->length].id = i;
 771          struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
 772             tail_type->array_element : tail_type->members[i];
 773          _vtn_variable_load_store(b, load, new_chain, elem_type,
 774                                   &(*inout)->elems[i]);
 775       }
 776       return;
 777    }
 778
 779    default:
 780       unreachable("Invalid access chain type");
 781    }
 782 }
 783
 784 struct vtn_ssa_value *
 785 vtn_variable_load(struct vtn_builder *b, struct vtn_access_chain *src)
 786 {
 787    if (vtn_variable_is_external_block(src->var)) {
 788       return vtn_block_load(b, src);
 789    } else {
 790       struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
 791       struct vtn_ssa_value *val = NULL;
 792       _vtn_variable_load_store(b, true, src, tail_type, &val);
 793       return val;
 794    }
 795 }
 796
 797 void
 798 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 799                    struct vtn_access_chain *dest)
 800 {
 801    if (vtn_variable_is_external_block(dest->var)) {
 802       assert(dest->var->mode == vtn_variable_mode_ssbo);
 803       vtn_block_store(b, src, dest);
 804    } else {
 805       struct vtn_type *tail_type = vtn_access_chain_tail_type(b, dest);
 806       _vtn_variable_load_store(b, false, dest, tail_type, &src);
 807    }
 808 }
 809
 810 static void
 811 _vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
 812                    struct vtn_access_chain *src, struct vtn_type *tail_type)
 813 {
 814    enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
 815    switch (base_type) {
 816    case GLSL_TYPE_UINT:
 817    case GLSL_TYPE_INT:
 818    case GLSL_TYPE_FLOAT:
 819    case GLSL_TYPE_DOUBLE:
 820    case GLSL_TYPE_BOOL:
 821       /* At this point, we have a scalar, vector, or matrix so we know that
 822        * there cannot be any structure splitting still in the way.  By
 823        * stopping at the matrix level rather than the vector level, we
 824        * ensure that matrices get loaded in the optimal way even if they
 825        * are storred row-major in a UBO.
 826        */
 827       vtn_variable_store(b, vtn_variable_load(b, src), dest);
 828       return;
 829
 830    case GLSL_TYPE_ARRAY:
 831    case GLSL_TYPE_STRUCT: {
 832       struct vtn_access_chain *new_src, *new_dest;
 833       new_src = vtn_access_chain_extend(b, src, 1);
 834       new_dest = vtn_access_chain_extend(b, dest, 1);
 835       new_src->link[src->length].mode = vtn_access_mode_literal;
 836       new_dest->link[dest->length].mode = vtn_access_mode_literal;
 837       unsigned elems = glsl_get_length(tail_type->type);
 838       for (unsigned i = 0; i < elems; i++) {
 839          new_src->link[src->length].id = i;
 840          new_dest->link[dest->length].id = i;
 841          struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
 842             tail_type->array_element : tail_type->members[i];
 843          _vtn_variable_copy(b, new_dest, new_src, elem_type);
 844       }
 845       return;
 846    }
 847
 848    default:
 849       unreachable("Invalid access chain type");
 850    }
 851 }
 852
 853 static void
 854 vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
 855                   struct vtn_access_chain *src)
 856 {
 857    struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
 858    assert(vtn_access_chain_tail_type(b, dest)->type == tail_type->type);
 859
 860    /* TODO: At some point, we should add a special-case for when we can
 861     * just emit a copy_var intrinsic.
 862     */
 863    _vtn_variable_copy(b, dest, src, tail_type);
 864 }
 865
 866 static void
 867 set_mode_system_value(nir_variable_mode *mode)
 868 {
 869    assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
 870    *mode = nir_var_system_value;
 871 }
 872
 873 static void
 874 vtn_get_builtin_location(struct vtn_builder *b,
 875                          SpvBuiltIn builtin, int *location,
 876                          nir_variable_mode *mode)
 877 {
 878    switch (builtin) {
 879    case SpvBuiltInPosition:
 880       *location = VARYING_SLOT_POS;
 881       break;
 882    case SpvBuiltInPointSize:
 883       *location = VARYING_SLOT_PSIZ;
 884       break;
 885    case SpvBuiltInClipDistance:
 886       *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
 887       break;
 888    case SpvBuiltInCullDistance:
 889       *location = VARYING_SLOT_CULL_DIST0;
 890       break;
 891    case SpvBuiltInVertexIndex:
 892       *location = SYSTEM_VALUE_VERTEX_ID;
 893       set_mode_system_value(mode);
 894       break;
 895    case SpvBuiltInVertexId:
 896       /* Vulkan defines VertexID to be zero-based and reserves the new
 897        * builtin keyword VertexIndex to indicate the non-zero-based value.
 898        */
 899       *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
 900       set_mode_system_value(mode);
 901       break;
 902    case SpvBuiltInInstanceIndex:
 903       *location = SYSTEM_VALUE_INSTANCE_INDEX;
 904       set_mode_system_value(mode);
 905       break;
 906    case SpvBuiltInInstanceId:
 907       *location = SYSTEM_VALUE_INSTANCE_ID;
 908       set_mode_system_value(mode);
 909       break;
 910    case SpvBuiltInPrimitiveId:
 911       if (*mode == nir_var_shader_out) {
 912          *location = VARYING_SLOT_PRIMITIVE_ID;
 913       } else {
 914          *location = SYSTEM_VALUE_PRIMITIVE_ID;
 915          set_mode_system_value(mode);
 916       }
 917       break;
 918    case SpvBuiltInInvocationId:
 919       *location = SYSTEM_VALUE_INVOCATION_ID;
 920       set_mode_system_value(mode);
 921       break;
 922    case SpvBuiltInLayer:
 923       *location = VARYING_SLOT_LAYER;
 924       if (b->shader->stage == MESA_SHADER_FRAGMENT)
 925          *mode = nir_var_shader_in;
 926       else if (b->shader->stage == MESA_SHADER_GEOMETRY)
 927          *mode = nir_var_shader_out;
 928       else
 929          unreachable("invalid stage for SpvBuiltInLayer");
 930       break;
 931    case SpvBuiltInViewportIndex:
 932       *location = VARYING_SLOT_VIEWPORT;
 933       if (b->shader->stage == MESA_SHADER_GEOMETRY)
 934          *mode = nir_var_shader_out;
 935       else if (b->shader->stage == MESA_SHADER_FRAGMENT)
 936          *mode = nir_var_shader_in;
 937       else
 938          unreachable("invalid stage for SpvBuiltInViewportIndex");
 939       break;
 940    case SpvBuiltInTessLevelOuter:
 941       *location = VARYING_SLOT_TESS_LEVEL_OUTER;
 942       break;
 943    case SpvBuiltInTessLevelInner:
 944       *location = VARYING_SLOT_TESS_LEVEL_INNER;
 945       break;
 946    case SpvBuiltInTessCoord:
 947       *location = SYSTEM_VALUE_TESS_COORD;
 948       set_mode_system_value(mode);
 949       break;
 950    case SpvBuiltInPatchVertices:
 951       *location = SYSTEM_VALUE_VERTICES_IN;
 952       set_mode_system_value(mode);
 953       break;
 954    case SpvBuiltInFragCoord:
 955       *location = VARYING_SLOT_POS;
 956       assert(*mode == nir_var_shader_in);
 957       break;
 958    case SpvBuiltInPointCoord:
 959       *location = VARYING_SLOT_PNTC;
 960       assert(*mode == nir_var_shader_in);
 961       break;
 962    case SpvBuiltInFrontFacing:
 963       *location = SYSTEM_VALUE_FRONT_FACE;
 964       set_mode_system_value(mode);
 965       break;
 966    case SpvBuiltInSampleId:
 967       *location = SYSTEM_VALUE_SAMPLE_ID;
 968       set_mode_system_value(mode);
 969       break;
 970    case SpvBuiltInSamplePosition:
 971       *location = SYSTEM_VALUE_SAMPLE_POS;
 972       set_mode_system_value(mode);
 973       break;
 974    case SpvBuiltInSampleMask:
 975       *location = SYSTEM_VALUE_SAMPLE_MASK_IN; /* XXX out? */
 976       set_mode_system_value(mode);
 977       break;
 978    case SpvBuiltInFragDepth:
 979       *location = FRAG_RESULT_DEPTH;
 980       assert(*mode == nir_var_shader_out);
 981       break;
 982    case SpvBuiltInNumWorkgroups:
 983       *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
 984       set_mode_system_value(mode);
 985       break;
 986    case SpvBuiltInWorkgroupSize:
 987       /* This should already be handled */
 988       unreachable("unsupported builtin");
 989       break;
 990    case SpvBuiltInWorkgroupId:
 991       *location = SYSTEM_VALUE_WORK_GROUP_ID;
 992       set_mode_system_value(mode);
 993       break;
 994    case SpvBuiltInLocalInvocationId:
 995       *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
 996       set_mode_system_value(mode);
 997       break;
 998    case SpvBuiltInLocalInvocationIndex:
 999       *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1000       set_mode_system_value(mode);
1001       break;
1002    case SpvBuiltInGlobalInvocationId:
1003       *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1004       set_mode_system_value(mode);
1005       break;
1006    case SpvBuiltInHelperInvocation:
1007    default:
1008       unreachable("unsupported builtin");
1009    }
1010 }
1011
1012 static void
1013 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var,
1014                      const struct vtn_decoration *dec)
1015 {
1016    switch (dec->decoration) {
1017    case SpvDecorationRelaxedPrecision:
1018       break; /* FIXME: Do nothing with this for now. */
1019    case SpvDecorationNoPerspective:
1020       nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1021       break;
1022    case SpvDecorationFlat:
1023       nir_var->data.interpolation = INTERP_MODE_FLAT;
1024       break;
1025    case SpvDecorationCentroid:
1026       nir_var->data.centroid = true;
1027       break;
1028    case SpvDecorationSample:
1029       nir_var->data.sample = true;
1030       break;
1031    case SpvDecorationInvariant:
1032       nir_var->data.invariant = true;
1033       break;
1034    case SpvDecorationConstant:
1035       assert(nir_var->constant_initializer != NULL);
1036       nir_var->data.read_only = true;
1037       break;
1038    case SpvDecorationNonWritable:
1039       nir_var->data.read_only = true;
1040       break;
1041    case SpvDecorationComponent:
1042       nir_var->data.location_frac = dec->literals[0];
1043       break;
1044    case SpvDecorationIndex:
1045       nir_var->data.index = dec->literals[0];
1046       break;
1047    case SpvDecorationBuiltIn: {
1048       SpvBuiltIn builtin = dec->literals[0];
1049
1050       if (builtin == SpvBuiltInWorkgroupSize) {
1051          /* This shouldn't be a builtin.  It's actually a constant. */
1052          nir_var->data.mode = nir_var_global;
1053          nir_var->data.read_only = true;
1054
1055          nir_constant *c = rzalloc(nir_var, nir_constant);
1056          c->values[0].u32[0] = b->shader->info->cs.local_size[0];
1057          c->values[0].u32[1] = b->shader->info->cs.local_size[1];
1058          c->values[0].u32[2] = b->shader->info->cs.local_size[2];
1059          nir_var->constant_initializer = c;
1060          break;
1061       }
1062
1063       nir_variable_mode mode = nir_var->data.mode;
1064       vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode);
1065       nir_var->data.mode = mode;
1066
1067       if (builtin == SpvBuiltInTessLevelOuter ||
1068           builtin == SpvBuiltInTessLevelInner) {
1069          nir_var->data.compact = true;
1070       }
1071
1072       if (builtin == SpvBuiltInFragCoord || builtin == SpvBuiltInSamplePosition)
1073          nir_var->data.origin_upper_left = b->origin_upper_left;
1074
1075       if (builtin == SpvBuiltInFragCoord)
1076          nir_var->data.pixel_center_integer = b->pixel_center_integer;
1077       break;
1078    }
1079
1080    case SpvDecorationSpecId:
1081    case SpvDecorationRowMajor:
1082    case SpvDecorationColMajor:
1083    case SpvDecorationMatrixStride:
1084    case SpvDecorationRestrict:
1085    case SpvDecorationAliased:
1086    case SpvDecorationVolatile:
1087    case SpvDecorationCoherent:
1088    case SpvDecorationNonReadable:
1089    case SpvDecorationUniform:
1090    case SpvDecorationStream:
1091    case SpvDecorationOffset:
1092    case SpvDecorationLinkageAttributes:
1093       break; /* Do nothing with these here */
1094
1095    case SpvDecorationPatch:
1096       nir_var->data.patch = true;
1097       break;
1098
1099    case SpvDecorationLocation:
1100       unreachable("Handled above");
1101
1102    case SpvDecorationBlock:
1103    case SpvDecorationBufferBlock:
1104    case SpvDecorationArrayStride:
1105    case SpvDecorationGLSLShared:
1106    case SpvDecorationGLSLPacked:
1107       break; /* These can apply to a type but we don't care about them */
1108
1109    case SpvDecorationBinding:
1110    case SpvDecorationDescriptorSet:
1111    case SpvDecorationNoContraction:
1112    case SpvDecorationInputAttachmentIndex:
1113       vtn_warn("Decoration not allowed for variable or structure member: %s",
1114                spirv_decoration_to_string(dec->decoration));
1115       break;
1116
1117    case SpvDecorationXfbBuffer:
1118    case SpvDecorationXfbStride:
1119       vtn_warn("Vulkan does not have transform feedback: %s",
1120                spirv_decoration_to_string(dec->decoration));
1121       break;
1122
1123    case SpvDecorationCPacked:
1124    case SpvDecorationSaturatedConversion:
1125    case SpvDecorationFuncParamAttr:
1126    case SpvDecorationFPRoundingMode:
1127    case SpvDecorationFPFastMathMode:
1128    case SpvDecorationAlignment:
1129       vtn_warn("Decoraiton only allowed for CL-style kernels: %s",
1130                spirv_decoration_to_string(dec->decoration));
1131       break;
1132    }
1133 }
1134
1135 static void
1136 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1137                 const struct vtn_decoration *dec, void *out_is_patch)
1138 {
1139    if (dec->decoration == SpvDecorationPatch) {
1140       *((bool *) out_is_patch) = true;
1141    }
1142 }
1143
1144 static void
1145 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1146                   const struct vtn_decoration *dec, void *void_var)
1147 {
1148    struct vtn_variable *vtn_var = void_var;
1149
1150    /* Handle decorations that apply to a vtn_variable as a whole */
1151    switch (dec->decoration) {
1152    case SpvDecorationBinding:
1153       vtn_var->binding = dec->literals[0];
1154       return;
1155    case SpvDecorationDescriptorSet:
1156       vtn_var->descriptor_set = dec->literals[0];
1157       return;
1158    case SpvDecorationInputAttachmentIndex:
1159       vtn_var->input_attachment_index = dec->literals[0];
1160       return;
1161    case SpvDecorationPatch:
1162       vtn_var->patch = true;
1163       break;
1164    default:
1165       break;
1166    }
1167
1168    if (val->value_type == vtn_value_type_access_chain) {
1169       assert(val->access_chain->length == 0);
1170       assert(val->access_chain->var == void_var);
1171       assert(member == -1);
1172    } else {
1173       assert(val->value_type == vtn_value_type_type);
1174    }
1175
1176    /* Location is odd.  If applied to a split structure, we have to walk the
1177     * whole thing and accumulate the location.  It's easier to handle as a
1178     * special case.
1179     */
1180    if (dec->decoration == SpvDecorationLocation) {
1181       unsigned location = dec->literals[0];
1182       bool is_vertex_input;
1183       if (b->shader->stage == MESA_SHADER_FRAGMENT &&
1184           vtn_var->mode == vtn_variable_mode_output) {
1185          is_vertex_input = false;
1186          location += FRAG_RESULT_DATA0;
1187       } else if (b->shader->stage == MESA_SHADER_VERTEX &&
1188                  vtn_var->mode == vtn_variable_mode_input) {
1189          is_vertex_input = true;
1190          location += VERT_ATTRIB_GENERIC0;
1191       } else if (vtn_var->mode == vtn_variable_mode_input ||
1192                  vtn_var->mode == vtn_variable_mode_output) {
1193          is_vertex_input = false;
1194          location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1195       } else {
1196          unreachable("Location must be on input or output variable");
1197       }
1198
1199       if (vtn_var->var) {
1200          /* This handles the member and lone variable cases */
1201          vtn_var->var->data.location = location;
1202       } else {
1203          /* This handles the structure member case */
1204          assert(vtn_var->members);
1205          unsigned length =
1206             glsl_get_length(glsl_without_array(vtn_var->type->type));
1207          for (unsigned i = 0; i < length; i++) {
1208             vtn_var->members[i]->data.location = location;
1209             location +=
1210                glsl_count_attribute_slots(vtn_var->members[i]->interface_type,
1211                                           is_vertex_input);
1212          }
1213       }
1214       return;
1215    } else {
1216       if (vtn_var->var) {
1217          assert(member <= 0);
1218          apply_var_decoration(b, vtn_var->var, dec);
1219       } else if (vtn_var->members) {
1220          if (member >= 0) {
1221             assert(vtn_var->members);
1222             apply_var_decoration(b, vtn_var->members[member], dec);
1223          } else {
1224             unsigned length =
1225                glsl_get_length(glsl_without_array(vtn_var->type->type));
1226             for (unsigned i = 0; i < length; i++)
1227                apply_var_decoration(b, vtn_var->members[i], dec);
1228          }
1229       } else {
1230          /* A few variables, those with external storage, have no actual
1231           * nir_variables associated with them.  Fortunately, all decorations
1232           * we care about for those variables are on the type only.
1233           */
1234          assert(vtn_var->mode == vtn_variable_mode_ubo ||
1235                 vtn_var->mode == vtn_variable_mode_ssbo ||
1236                 vtn_var->mode == vtn_variable_mode_push_constant);
1237       }
1238    }
1239 }
1240
1241 static bool
1242 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
1243 {
1244    if (var->patch || !glsl_type_is_array(var->type->type))
1245       return false;
1246
1247    if (var->mode == vtn_variable_mode_input) {
1248       return stage == MESA_SHADER_TESS_CTRL ||
1249              stage == MESA_SHADER_TESS_EVAL ||
1250              stage == MESA_SHADER_GEOMETRY;
1251    }
1252
1253    if (var->mode == vtn_variable_mode_output)
1254       return stage == MESA_SHADER_TESS_CTRL;
1255
1256    return false;
1257 }
1258
1259 void
1260 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1261                      const uint32_t *w, unsigned count)
1262 {
1263    switch (opcode) {
1264    case SpvOpVariable: {
1265       struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1266       var->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1267
1268       var->chain.var = var;
1269       var->chain.length = 0;
1270
1271       struct vtn_value *val =
1272          vtn_push_value(b, w[2], vtn_value_type_access_chain);
1273       val->access_chain = &var->chain;
1274
1275       struct vtn_type *without_array = var->type;
1276       while(glsl_type_is_array(without_array->type))
1277          without_array = without_array->array_element;
1278
1279       nir_variable_mode nir_mode;
1280       switch ((SpvStorageClass)w[3]) {
1281       case SpvStorageClassUniform:
1282       case SpvStorageClassUniformConstant:
1283          if (without_array->block) {
1284             var->mode = vtn_variable_mode_ubo;
1285             b->shader->info->num_ubos++;
1286          } else if (without_array->buffer_block) {
1287             var->mode = vtn_variable_mode_ssbo;
1288             b->shader->info->num_ssbos++;
1289          } else if (glsl_type_is_image(without_array->type)) {
1290             var->mode = vtn_variable_mode_image;
1291             nir_mode = nir_var_uniform;
1292             b->shader->info->num_images++;
1293          } else if (glsl_type_is_sampler(without_array->type)) {
1294             var->mode = vtn_variable_mode_sampler;
1295             nir_mode = nir_var_uniform;
1296             b->shader->info->num_textures++;
1297          } else {
1298             assert(!"Invalid uniform variable type");
1299          }
1300          break;
1301       case SpvStorageClassPushConstant:
1302          var->mode = vtn_variable_mode_push_constant;
1303          assert(b->shader->num_uniforms == 0);
1304          b->shader->num_uniforms = vtn_type_block_size(var->type);
1305          break;
1306       case SpvStorageClassInput:
1307          var->mode = vtn_variable_mode_input;
1308          nir_mode = nir_var_shader_in;
1309          break;
1310       case SpvStorageClassOutput:
1311          var->mode = vtn_variable_mode_output;
1312          nir_mode = nir_var_shader_out;
1313          break;
1314       case SpvStorageClassPrivate:
1315          var->mode = vtn_variable_mode_global;
1316          nir_mode = nir_var_global;
1317          break;
1318       case SpvStorageClassFunction:
1319          var->mode = vtn_variable_mode_local;
1320          nir_mode = nir_var_local;
1321          break;
1322       case SpvStorageClassWorkgroup:
1323          var->mode = vtn_variable_mode_workgroup;
1324          nir_mode = nir_var_shared;
1325          break;
1326       case SpvStorageClassCrossWorkgroup:
1327       case SpvStorageClassGeneric:
1328       case SpvStorageClassAtomicCounter:
1329       default:
1330          unreachable("Unhandled variable storage class");
1331       }
1332
1333       switch (var->mode) {
1334       case vtn_variable_mode_local:
1335       case vtn_variable_mode_global:
1336       case vtn_variable_mode_image:
1337       case vtn_variable_mode_sampler:
1338       case vtn_variable_mode_workgroup:
1339          /* For these, we create the variable normally */
1340          var->var = rzalloc(b->shader, nir_variable);
1341          var->var->name = ralloc_strdup(var->var, val->name);
1342          var->var->type = var->type->type;
1343          var->var->data.mode = nir_mode;
1344
1345          switch (var->mode) {
1346          case vtn_variable_mode_image:
1347          case vtn_variable_mode_sampler:
1348             var->var->interface_type = without_array->type;
1349             break;
1350          default:
1351             var->var->interface_type = NULL;
1352             break;
1353          }
1354          break;
1355
1356       case vtn_variable_mode_input:
1357       case vtn_variable_mode_output: {
1358          var->patch = false;
1359          vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
1360
1361          /* For inputs and outputs, we immediately split structures.  This
1362           * is for a couple of reasons.  For one, builtins may all come in
1363           * a struct and we really want those split out into separate
1364           * variables.  For another, interpolation qualifiers can be
1365           * applied to members of the top-level struct ane we need to be
1366           * able to preserve that information.
1367           */
1368
1369          int array_length = -1;
1370          struct vtn_type *interface_type = var->type;
1371          if (is_per_vertex_inout(var, b->shader->stage)) {
1372             /* In Geometry shaders (and some tessellation), inputs come
1373              * in per-vertex arrays.  However, some builtins come in
1374              * non-per-vertex, hence the need for the is_array check.  In
1375              * any case, there are no non-builtin arrays allowed so this
1376              * check should be sufficient.
1377              */
1378             interface_type = var->type->array_element;
1379             array_length = glsl_get_length(var->type->type);
1380          }
1381
1382          if (glsl_type_is_struct(interface_type->type)) {
1383             /* It's a struct.  Split it. */
1384             unsigned num_members = glsl_get_length(interface_type->type);
1385             var->members = ralloc_array(b, nir_variable *, num_members);
1386
1387             for (unsigned i = 0; i < num_members; i++) {
1388                const struct glsl_type *mtype = interface_type->members[i]->type;
1389                if (array_length >= 0)
1390                   mtype = glsl_array_type(mtype, array_length);
1391
1392                var->members[i] = rzalloc(b->shader, nir_variable);
1393                var->members[i]->name =
1394                   ralloc_asprintf(var->members[i], "%s.%d", val->name, i);
1395                var->members[i]->type = mtype;
1396                var->members[i]->interface_type =
1397                   interface_type->members[i]->type;
1398                var->members[i]->data.mode = nir_mode;
1399             }
1400          } else {
1401             var->var = rzalloc(b->shader, nir_variable);
1402             var->var->name = ralloc_strdup(var->var, val->name);
1403             var->var->type = var->type->type;
1404             var->var->interface_type = interface_type->type;
1405             var->var->data.mode = nir_mode;
1406          }
1407
1408          /* For inputs and outputs, we need to grab locations and builtin
1409           * information from the interface type.
1410           */
1411          vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var);
1412          break;
1413
1414       case vtn_variable_mode_param:
1415          unreachable("Not created through OpVariable");
1416       }
1417
1418       case vtn_variable_mode_ubo:
1419       case vtn_variable_mode_ssbo:
1420       case vtn_variable_mode_push_constant:
1421          /* These don't need actual variables. */
1422          break;
1423       }
1424
1425       if (count > 4) {
1426          assert(count == 5);
1427          nir_constant *constant =
1428             vtn_value(b, w[4], vtn_value_type_constant)->constant;
1429          var->var->constant_initializer =
1430             nir_constant_clone(constant, var->var);
1431       }
1432
1433       vtn_foreach_decoration(b, val, var_decoration_cb, var);
1434
1435       if (var->mode == vtn_variable_mode_image ||
1436           var->mode == vtn_variable_mode_sampler) {
1437          /* XXX: We still need the binding information in the nir_variable
1438           * for these. We should fix that.
1439           */
1440          var->var->data.binding = var->binding;
1441          var->var->data.descriptor_set = var->descriptor_set;
1442          var->var->data.index = var->input_attachment_index;
1443
1444          if (var->mode == vtn_variable_mode_image)
1445             var->var->data.image.format = without_array->image_format;
1446       }
1447
1448       if (var->mode == vtn_variable_mode_local) {
1449          assert(var->members == NULL && var->var != NULL);
1450          nir_function_impl_add_variable(b->impl, var->var);
1451       } else if (var->var) {
1452          nir_shader_add_variable(b->shader, var->var);
1453       } else if (var->members) {
1454          unsigned count = glsl_get_length(without_array->type);
1455          for (unsigned i = 0; i < count; i++) {
1456             assert(var->members[i]->data.mode != nir_var_local);
1457             nir_shader_add_variable(b->shader, var->members[i]);
1458          }
1459       } else {
1460          assert(var->mode == vtn_variable_mode_ubo ||
1461                 var->mode == vtn_variable_mode_ssbo ||
1462                 var->mode == vtn_variable_mode_push_constant);
1463       }
1464       break;
1465    }
1466
1467    case SpvOpAccessChain:
1468    case SpvOpInBoundsAccessChain: {
1469       struct vtn_access_chain *base, *chain;
1470       struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1471       if (base_val->value_type == vtn_value_type_sampled_image) {
1472          /* This is rather insane.  SPIR-V allows you to use OpSampledImage
1473           * to combine an array of images with a single sampler to get an
1474           * array of sampled images that all share the same sampler.
1475           * Fortunately, this means that we can more-or-less ignore the
1476           * sampler when crawling the access chain, but it does leave us
1477           * with this rather awkward little special-case.
1478           */
1479          base = base_val->sampled_image->image;
1480       } else {
1481          assert(base_val->value_type == vtn_value_type_access_chain);
1482          base = base_val->access_chain;
1483       }
1484
1485       chain = vtn_access_chain_extend(b, base, count - 4);
1486
1487       unsigned idx = base->length;
1488       for (int i = 4; i < count; i++) {
1489          struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1490          if (link_val->value_type == vtn_value_type_constant) {
1491             chain->link[idx].mode = vtn_access_mode_literal;
1492             chain->link[idx].id = link_val->constant->values[0].u32[0];
1493          } else {
1494             chain->link[idx].mode = vtn_access_mode_id;
1495             chain->link[idx].id = w[i];
1496          }
1497          idx++;
1498       }
1499
1500       if (base_val->value_type == vtn_value_type_sampled_image) {
1501          struct vtn_value *val =
1502             vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1503          val->sampled_image = ralloc(b, struct vtn_sampled_image);
1504          val->sampled_image->image = chain;
1505          val->sampled_image->sampler = base_val->sampled_image->sampler;
1506       } else {
1507          struct vtn_value *val =
1508             vtn_push_value(b, w[2], vtn_value_type_access_chain);
1509          val->access_chain = chain;
1510       }
1511       break;
1512    }
1513
1514    case SpvOpCopyMemory: {
1515       struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_access_chain);
1516       struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_access_chain);
1517
1518       vtn_variable_copy(b, dest->access_chain, src->access_chain);
1519       break;
1520    }
1521
1522    case SpvOpLoad: {
1523       struct vtn_access_chain *src =
1524          vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1525
1526       if (src->var->mode == vtn_variable_mode_image ||
1527           src->var->mode == vtn_variable_mode_sampler) {
1528          vtn_push_value(b, w[2], vtn_value_type_access_chain)->access_chain = src;
1529          return;
1530       }
1531
1532       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1533       val->ssa = vtn_variable_load(b, src);
1534       break;
1535    }
1536
1537    case SpvOpStore: {
1538       struct vtn_access_chain *dest =
1539          vtn_value(b, w[1], vtn_value_type_access_chain)->access_chain;
1540       struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1541       vtn_variable_store(b, src, dest);
1542       break;
1543    }
1544
1545    case SpvOpArrayLength: {
1546       struct vtn_access_chain *chain =
1547          vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1548
1549       const uint32_t offset = chain->var->type->offsets[w[4]];
1550       const uint32_t stride = chain->var->type->members[w[4]]->stride;
1551
1552       unsigned chain_idx;
1553       struct vtn_type *type;
1554       nir_ssa_def *index =
1555          get_vulkan_resource_index(b, chain, &type, &chain_idx);
1556
1557       nir_intrinsic_instr *instr =
1558          nir_intrinsic_instr_create(b->nb.shader,
1559                                     nir_intrinsic_get_buffer_size);
1560       instr->src[0] = nir_src_for_ssa(index);
1561       nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1562       nir_builder_instr_insert(&b->nb, &instr->instr);
1563       nir_ssa_def *buf_size = &instr->dest.ssa;
1564
1565       /* array_length = max(buffer_size - offset, 0) / stride */
1566       nir_ssa_def *array_length =
1567          nir_idiv(&b->nb,
1568                   nir_imax(&b->nb,
1569                            nir_isub(&b->nb,
1570                                     buf_size,
1571                                     nir_imm_int(&b->nb, offset)),
1572                            nir_imm_int(&b->nb, 0u)),
1573                   nir_imm_int(&b->nb, stride));
1574
1575       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1576       val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
1577       val->ssa->def = array_length;
1578       break;
1579    }
1580
1581    case SpvOpCopyMemorySized:
1582    default:
1583       unreachable("Unhandled opcode");
1584    }
1585 }