src/compiler/spirv/vtn_variables.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 #include "vtn_private.h"
  29 #include "spirv_info.h"
  30
  31 static struct vtn_access_chain *
  32 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
  33                         unsigned new_ids)
  34 {
  35    struct vtn_access_chain *chain;
  36
  37    unsigned new_len = old->length + new_ids;
  38    /* TODO: don't use rzalloc */
  39    chain = rzalloc_size(b, sizeof(*chain) + new_len * sizeof(chain->link[0]));
  40
  41    chain->var = old->var;
  42    chain->length = new_len;
  43
  44    for (unsigned i = 0; i < old->length; i++)
  45       chain->link[i] = old->link[i];
  46
  47    return chain;
  48 }
  49
  50 static nir_ssa_def *
  51 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
  52                        unsigned stride)
  53 {
  54    assert(stride > 0);
  55    if (link.mode == vtn_access_mode_literal) {
  56       return nir_imm_int(&b->nb, link.id * stride);
  57    } else if (stride == 1) {
  58       return vtn_ssa_value(b, link.id)->def;
  59    } else {
  60       return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
  61                               nir_imm_int(&b->nb, stride));
  62    }
  63 }
  64
  65 static struct vtn_type *
  66 vtn_access_chain_tail_type(struct vtn_builder *b,
  67                            struct vtn_access_chain *chain)
  68 {
  69    struct vtn_type *type = chain->var->type;
  70    for (unsigned i = 0; i < chain->length; i++) {
  71       if (glsl_type_is_struct(type->type)) {
  72          assert(chain->link[i].mode == vtn_access_mode_literal);
  73          type = type->members[chain->link[i].id];
  74       } else {
  75          type = type->array_element;
  76       }
  77    }
  78    return type;
  79 }
  80
  81 /* Crawls a chain of array derefs and rewrites the types so that the
  82  * lengths stay the same but the terminal type is the one given by
  83  * tail_type.  This is useful for split structures.
  84  */
  85 static void
  86 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type)
  87 {
  88    deref->type = type;
  89    if (deref->child) {
  90       assert(deref->child->deref_type == nir_deref_type_array);
  91       assert(glsl_type_is_array(deref->type));
  92       rewrite_deref_types(deref->child, glsl_get_array_element(type));
  93    }
  94 }
  95
  96 nir_deref_var *
  97 vtn_access_chain_to_deref(struct vtn_builder *b, struct vtn_access_chain *chain)
  98 {
  99    nir_deref_var *deref_var;
 100    if (chain->var->var) {
 101       deref_var = nir_deref_var_create(b, chain->var->var);
 102    } else {
 103       assert(chain->var->members);
 104       /* Create the deref_var manually.  It will get filled out later. */
 105       deref_var = rzalloc(b, nir_deref_var);
 106       deref_var->deref.deref_type = nir_deref_type_var;
 107    }
 108
 109    struct vtn_type *deref_type = chain->var->type;
 110    nir_deref *tail = &deref_var->deref;
 111    nir_variable **members = chain->var->members;
 112
 113    for (unsigned i = 0; i < chain->length; i++) {
 114       enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
 115       switch (base_type) {
 116       case GLSL_TYPE_UINT:
 117       case GLSL_TYPE_INT:
 118       case GLSL_TYPE_FLOAT:
 119       case GLSL_TYPE_DOUBLE:
 120       case GLSL_TYPE_BOOL:
 121       case GLSL_TYPE_ARRAY: {
 122          deref_type = deref_type->array_element;
 123
 124          nir_deref_array *deref_arr = nir_deref_array_create(b);
 125          deref_arr->deref.type = deref_type->type;
 126
 127          if (chain->link[i].mode == vtn_access_mode_literal) {
 128             deref_arr->deref_array_type = nir_deref_array_type_direct;
 129             deref_arr->base_offset = chain->link[i].id;
 130          } else {
 131             assert(chain->link[i].mode == vtn_access_mode_id);
 132             deref_arr->deref_array_type = nir_deref_array_type_indirect;
 133             deref_arr->base_offset = 0;
 134             deref_arr->indirect =
 135                nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
 136          }
 137          tail->child = &deref_arr->deref;
 138          tail = tail->child;
 139          break;
 140       }
 141
 142       case GLSL_TYPE_STRUCT: {
 143          assert(chain->link[i].mode == vtn_access_mode_literal);
 144          unsigned idx = chain->link[i].id;
 145          deref_type = deref_type->members[idx];
 146          if (members) {
 147             /* This is a pre-split structure. */
 148             deref_var->var = members[idx];
 149             rewrite_deref_types(&deref_var->deref, members[idx]->type);
 150             assert(tail->type == deref_type->type);
 151             members = NULL;
 152          } else {
 153             nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
 154             deref_struct->deref.type = deref_type->type;
 155             tail->child = &deref_struct->deref;
 156             tail = tail->child;
 157          }
 158          break;
 159       }
 160       default:
 161          unreachable("Invalid type for deref");
 162       }
 163    }
 164
 165    assert(members == NULL);
 166    return deref_var;
 167 }
 168
 169 static void
 170 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref,
 171                       nir_deref *tail, struct vtn_ssa_value *inout)
 172 {
 173    /* The deref tail may contain a deref to select a component of a vector (in
 174     * other words, it might not be an actual tail) so we have to save it away
 175     * here since we overwrite it later.
 176     */
 177    nir_deref *old_child = tail->child;
 178
 179    if (glsl_type_is_vector_or_scalar(tail->type)) {
 180       /* Terminate the deref chain in case there is one more link to pick
 181        * off a component of the vector.
 182        */
 183       tail->child = NULL;
 184
 185       nir_intrinsic_op op = load ? nir_intrinsic_load_var :
 186                                    nir_intrinsic_store_var;
 187
 188       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op);
 189       intrin->variables[0] = nir_deref_var_clone(deref, intrin);
 190       intrin->num_components = glsl_get_vector_elements(tail->type);
 191
 192       if (load) {
 193          nir_ssa_dest_init(&intrin->instr, &intrin->dest,
 194                            intrin->num_components,
 195                            glsl_get_bit_size(tail->type),
 196                            NULL);
 197          inout->def = &intrin->dest.ssa;
 198       } else {
 199          nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1);
 200          intrin->src[0] = nir_src_for_ssa(inout->def);
 201       }
 202
 203       nir_builder_instr_insert(&b->nb, &intrin->instr);
 204    } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY ||
 205               glsl_type_is_matrix(tail->type)) {
 206       unsigned elems = glsl_get_length(tail->type);
 207       nir_deref_array *deref_arr = nir_deref_array_create(b);
 208       deref_arr->deref_array_type = nir_deref_array_type_direct;
 209       deref_arr->deref.type = glsl_get_array_element(tail->type);
 210       tail->child = &deref_arr->deref;
 211       for (unsigned i = 0; i < elems; i++) {
 212          deref_arr->base_offset = i;
 213          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 214       }
 215    } else {
 216       assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT);
 217       unsigned elems = glsl_get_length(tail->type);
 218       nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0);
 219       tail->child = &deref_struct->deref;
 220       for (unsigned i = 0; i < elems; i++) {
 221          deref_struct->index = i;
 222          deref_struct->deref.type = glsl_get_struct_field(tail->type, i);
 223          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 224       }
 225    }
 226
 227    tail->child = old_child;
 228 }
 229
 230 nir_deref_var *
 231 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
 232 {
 233    struct vtn_access_chain *chain =
 234       vtn_value(b, id, vtn_value_type_access_chain)->access_chain;
 235
 236    return vtn_access_chain_to_deref(b, chain);
 237 }
 238
 239 /*
 240  * Gets the NIR-level deref tail, which may have as a child an array deref
 241  * selecting which component due to OpAccessChain supporting per-component
 242  * indexing in SPIR-V.
 243  */
 244 static nir_deref *
 245 get_deref_tail(nir_deref_var *deref)
 246 {
 247    nir_deref *cur = &deref->deref;
 248    while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child)
 249       cur = cur->child;
 250
 251    return cur;
 252 }
 253
 254 struct vtn_ssa_value *
 255 vtn_local_load(struct vtn_builder *b, nir_deref_var *src)
 256 {
 257    nir_deref *src_tail = get_deref_tail(src);
 258    struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
 259    _vtn_local_load_store(b, true, src, src_tail, val);
 260
 261    if (src_tail->child) {
 262       nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child);
 263       assert(vec_deref->deref.child == NULL);
 264       val->type = vec_deref->deref.type;
 265       if (vec_deref->deref_array_type == nir_deref_array_type_direct)
 266          val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset);
 267       else
 268          val->def = vtn_vector_extract_dynamic(b, val->def,
 269                                                vec_deref->indirect.ssa);
 270    }
 271
 272    return val;
 273 }
 274
 275 void
 276 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 277                 nir_deref_var *dest)
 278 {
 279    nir_deref *dest_tail = get_deref_tail(dest);
 280
 281    if (dest_tail->child) {
 282       struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
 283       _vtn_local_load_store(b, true, dest, dest_tail, val);
 284       nir_deref_array *deref = nir_deref_as_array(dest_tail->child);
 285       assert(deref->deref.child == NULL);
 286       if (deref->deref_array_type == nir_deref_array_type_direct)
 287          val->def = vtn_vector_insert(b, val->def, src->def,
 288                                       deref->base_offset);
 289       else
 290          val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
 291                                               deref->indirect.ssa);
 292       _vtn_local_load_store(b, false, dest, dest_tail, val);
 293    } else {
 294       _vtn_local_load_store(b, false, dest, dest_tail, src);
 295    }
 296 }
 297
 298 static nir_ssa_def *
 299 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain,
 300                           struct vtn_type **type, unsigned *chain_idx)
 301 {
 302    /* Push constants have no explicit binding */
 303    if (chain->var->mode == vtn_variable_mode_push_constant) {
 304       *chain_idx = 0;
 305       *type = chain->var->type;
 306       return NULL;
 307    }
 308
 309    nir_ssa_def *array_index;
 310    if (glsl_type_is_array(chain->var->type->type)) {
 311       assert(chain->length > 0);
 312       array_index = vtn_access_link_as_ssa(b, chain->link[0], 1);
 313       *chain_idx = 1;
 314       *type = chain->var->type->array_element;
 315    } else {
 316       array_index = nir_imm_int(&b->nb, 0);
 317       *chain_idx = 0;
 318       *type = chain->var->type;
 319    }
 320
 321    nir_intrinsic_instr *instr =
 322       nir_intrinsic_instr_create(b->nb.shader,
 323                                  nir_intrinsic_vulkan_resource_index);
 324    instr->src[0] = nir_src_for_ssa(array_index);
 325    nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set);
 326    nir_intrinsic_set_binding(instr, chain->var->binding);
 327
 328    nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 329    nir_builder_instr_insert(&b->nb, &instr->instr);
 330
 331    return &instr->dest.ssa;
 332 }
 333
 334 nir_ssa_def *
 335 vtn_access_chain_to_offset(struct vtn_builder *b,
 336                            struct vtn_access_chain *chain,
 337                            nir_ssa_def **index_out, struct vtn_type **type_out,
 338                            unsigned *end_idx_out, bool stop_at_matrix)
 339 {
 340    unsigned idx = 0;
 341    struct vtn_type *type;
 342    *index_out = get_vulkan_resource_index(b, chain, &type, &idx);
 343
 344    nir_ssa_def *offset = nir_imm_int(&b->nb, 0);
 345    for (; idx < chain->length; idx++) {
 346       enum glsl_base_type base_type = glsl_get_base_type(type->type);
 347       switch (base_type) {
 348       case GLSL_TYPE_UINT:
 349       case GLSL_TYPE_INT:
 350       case GLSL_TYPE_FLOAT:
 351       case GLSL_TYPE_DOUBLE:
 352       case GLSL_TYPE_BOOL:
 353          /* Some users may not want matrix or vector derefs */
 354          if (stop_at_matrix)
 355             goto end;
 356          /* Fall through */
 357
 358       case GLSL_TYPE_ARRAY:
 359          offset = nir_iadd(&b->nb, offset,
 360                            vtn_access_link_as_ssa(b, chain->link[idx],
 361                                                   type->stride));
 362
 363          type = type->array_element;
 364          break;
 365
 366       case GLSL_TYPE_STRUCT: {
 367          assert(chain->link[idx].mode == vtn_access_mode_literal);
 368          unsigned member = chain->link[idx].id;
 369          offset = nir_iadd(&b->nb, offset,
 370                            nir_imm_int(&b->nb, type->offsets[member]));
 371          type = type->members[member];
 372          break;
 373       }
 374
 375       default:
 376          unreachable("Invalid type for deref");
 377       }
 378    }
 379
 380 end:
 381    *type_out = type;
 382    if (end_idx_out)
 383       *end_idx_out = idx;
 384
 385    return offset;
 386 }
 387
 388 /* Tries to compute the size of an interface block based on the strides and
 389  * offsets that are provided to us in the SPIR-V source.
 390  */
 391 static unsigned
 392 vtn_type_block_size(struct vtn_type *type)
 393 {
 394    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 395    switch (base_type) {
 396    case GLSL_TYPE_UINT:
 397    case GLSL_TYPE_INT:
 398    case GLSL_TYPE_FLOAT:
 399    case GLSL_TYPE_BOOL:
 400    case GLSL_TYPE_DOUBLE: {
 401       unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
 402                                         glsl_get_matrix_columns(type->type);
 403       if (cols > 1) {
 404          assert(type->stride > 0);
 405          return type->stride * cols;
 406       } else if (base_type == GLSL_TYPE_DOUBLE) {
 407          return glsl_get_vector_elements(type->type) * 8;
 408       } else {
 409          return glsl_get_vector_elements(type->type) * 4;
 410       }
 411    }
 412
 413    case GLSL_TYPE_STRUCT:
 414    case GLSL_TYPE_INTERFACE: {
 415       unsigned size = 0;
 416       unsigned num_fields = glsl_get_length(type->type);
 417       for (unsigned f = 0; f < num_fields; f++) {
 418          unsigned field_end = type->offsets[f] +
 419                               vtn_type_block_size(type->members[f]);
 420          size = MAX2(size, field_end);
 421       }
 422       return size;
 423    }
 424
 425    case GLSL_TYPE_ARRAY:
 426       assert(type->stride > 0);
 427       assert(glsl_get_length(type->type) > 0);
 428       return type->stride * glsl_get_length(type->type);
 429
 430    default:
 431       assert(!"Invalid block type");
 432       return 0;
 433    }
 434 }
 435
 436 static void
 437 vtn_access_chain_get_offset_size(struct vtn_access_chain *chain,
 438                                  unsigned *access_offset,
 439                                  unsigned *access_size)
 440 {
 441    /* Only valid for push constants accesses now. */
 442    assert(chain->var->mode == vtn_variable_mode_push_constant);
 443
 444    struct vtn_type *type = chain->var->type;
 445
 446    *access_offset = 0;
 447
 448    for (unsigned i = 0; i < chain->length; i++) {
 449       if (chain->link[i].mode != vtn_access_mode_literal)
 450          break;
 451
 452       if (glsl_type_is_struct(type->type)) {
 453          *access_offset += type->offsets[chain->link[i].id];
 454          type = type->members[chain->link[i].id];
 455       } else {
 456          *access_offset += type->stride * chain->link[i].id;
 457          type = type->array_element;
 458       }
 459    }
 460
 461    *access_size = vtn_type_block_size(type);
 462 }
 463
 464 static void
 465 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 466                      nir_ssa_def *index, nir_ssa_def *offset,
 467                      unsigned access_offset, unsigned access_size,
 468                      struct vtn_ssa_value **inout, const struct glsl_type *type)
 469 {
 470    nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
 471    instr->num_components = glsl_get_vector_elements(type);
 472
 473    int src = 0;
 474    if (!load) {
 475       nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
 476       instr->src[src++] = nir_src_for_ssa((*inout)->def);
 477    }
 478
 479    if (op == nir_intrinsic_load_push_constant) {
 480       assert(access_offset % 4 == 0);
 481
 482       nir_intrinsic_set_base(instr, access_offset);
 483       nir_intrinsic_set_range(instr, access_size);
 484    }
 485
 486    if (index)
 487       instr->src[src++] = nir_src_for_ssa(index);
 488
 489    if (op == nir_intrinsic_load_push_constant) {
 490       /* We need to subtract the offset from where the intrinsic will load the
 491        * data. */
 492       instr->src[src++] =
 493          nir_src_for_ssa(nir_isub(&b->nb, offset,
 494                                   nir_imm_int(&b->nb, access_offset)));
 495    } else {
 496       instr->src[src++] = nir_src_for_ssa(offset);
 497    }
 498
 499    if (load) {
 500       nir_ssa_dest_init(&instr->instr, &instr->dest,
 501                         instr->num_components,
 502                         glsl_get_bit_size(type), NULL);
 503       (*inout)->def = &instr->dest.ssa;
 504    }
 505
 506    nir_builder_instr_insert(&b->nb, &instr->instr);
 507
 508    if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
 509       (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
 510 }
 511
 512 static void
 513 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 514                       nir_ssa_def *index, nir_ssa_def *offset,
 515                       unsigned access_offset, unsigned access_size,
 516                       struct vtn_access_chain *chain, unsigned chain_idx,
 517                       struct vtn_type *type, struct vtn_ssa_value **inout)
 518 {
 519    if (chain && chain_idx >= chain->length)
 520       chain = NULL;
 521
 522    if (load && chain == NULL && *inout == NULL)
 523       *inout = vtn_create_ssa_value(b, type->type);
 524
 525    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 526    switch (base_type) {
 527    case GLSL_TYPE_UINT:
 528    case GLSL_TYPE_INT:
 529    case GLSL_TYPE_FLOAT:
 530    case GLSL_TYPE_BOOL:
 531       /* This is where things get interesting.  At this point, we've hit
 532        * a vector, a scalar, or a matrix.
 533        */
 534       if (glsl_type_is_matrix(type->type)) {
 535          if (chain == NULL) {
 536             /* Loading the whole matrix */
 537             struct vtn_ssa_value *transpose;
 538             unsigned num_ops, vec_width;
 539             if (type->row_major) {
 540                num_ops = glsl_get_vector_elements(type->type);
 541                vec_width = glsl_get_matrix_columns(type->type);
 542                if (load) {
 543                   const struct glsl_type *transpose_type =
 544                      glsl_matrix_type(base_type, vec_width, num_ops);
 545                   *inout = vtn_create_ssa_value(b, transpose_type);
 546                } else {
 547                   transpose = vtn_ssa_transpose(b, *inout);
 548                   inout = &transpose;
 549                }
 550             } else {
 551                num_ops = glsl_get_matrix_columns(type->type);
 552                vec_width = glsl_get_vector_elements(type->type);
 553             }
 554
 555             for (unsigned i = 0; i < num_ops; i++) {
 556                nir_ssa_def *elem_offset =
 557                   nir_iadd(&b->nb, offset,
 558                            nir_imm_int(&b->nb, i * type->stride));
 559                _vtn_load_store_tail(b, op, load, index, elem_offset,
 560                                     access_offset, access_size,
 561                                     &(*inout)->elems[i],
 562                                     glsl_vector_type(base_type, vec_width));
 563             }
 564
 565             if (load && type->row_major)
 566                *inout = vtn_ssa_transpose(b, *inout);
 567          } else if (type->row_major) {
 568             /* Row-major but with an access chiain. */
 569             nir_ssa_def *col_offset =
 570                vtn_access_link_as_ssa(b, chain->link[chain_idx],
 571                                       type->array_element->stride);
 572             offset = nir_iadd(&b->nb, offset, col_offset);
 573
 574             if (chain_idx + 1 < chain->length) {
 575                /* Picking off a single element */
 576                nir_ssa_def *row_offset =
 577                   vtn_access_link_as_ssa(b, chain->link[chain_idx + 1],
 578                                          type->stride);
 579                offset = nir_iadd(&b->nb, offset, row_offset);
 580                if (load)
 581                   *inout = vtn_create_ssa_value(b, glsl_scalar_type(base_type));
 582                _vtn_load_store_tail(b, op, load, index, offset,
 583                                     access_offset, access_size,
 584                                     inout, glsl_scalar_type(base_type));
 585             } else {
 586                /* Grabbing a column; picking one element off each row */
 587                unsigned num_comps = glsl_get_vector_elements(type->type);
 588                const struct glsl_type *column_type =
 589                   glsl_get_column_type(type->type);
 590
 591                nir_ssa_def *comps[4];
 592                for (unsigned i = 0; i < num_comps; i++) {
 593                   nir_ssa_def *elem_offset =
 594                      nir_iadd(&b->nb, offset,
 595                               nir_imm_int(&b->nb, i * type->stride));
 596
 597                   struct vtn_ssa_value *comp, temp_val;
 598                   if (!load) {
 599                      temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
 600                      temp_val.type = glsl_scalar_type(base_type);
 601                   }
 602                   comp = &temp_val;
 603                   _vtn_load_store_tail(b, op, load, index, elem_offset,
 604                                        access_offset, access_size,
 605                                        &comp, glsl_scalar_type(base_type));
 606                   comps[i] = comp->def;
 607                }
 608
 609                if (load) {
 610                   if (*inout == NULL)
 611                      *inout = vtn_create_ssa_value(b, column_type);
 612
 613                   (*inout)->def = nir_vec(&b->nb, comps, num_comps);
 614                }
 615             }
 616          } else {
 617             /* Column-major with a deref. Fall through to array case. */
 618             nir_ssa_def *col_offset =
 619                vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
 620             offset = nir_iadd(&b->nb, offset, col_offset);
 621
 622             _vtn_block_load_store(b, op, load, index, offset,
 623                                   access_offset, access_size,
 624                                   chain, chain_idx + 1,
 625                                   type->array_element, inout);
 626          }
 627       } else if (chain == NULL) {
 628          /* Single whole vector */
 629          assert(glsl_type_is_vector_or_scalar(type->type));
 630          _vtn_load_store_tail(b, op, load, index, offset,
 631                               access_offset, access_size,
 632                               inout, type->type);
 633       } else {
 634          /* Single component of a vector. Fall through to array case. */
 635          nir_ssa_def *elem_offset =
 636             vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
 637          offset = nir_iadd(&b->nb, offset, elem_offset);
 638
 639          _vtn_block_load_store(b, op, load, index, offset,
 640                                access_offset, access_size,
 641                                NULL, 0,
 642                                type->array_element, inout);
 643       }
 644       return;
 645
 646    case GLSL_TYPE_ARRAY: {
 647       unsigned elems = glsl_get_length(type->type);
 648       for (unsigned i = 0; i < elems; i++) {
 649          nir_ssa_def *elem_off =
 650             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
 651          _vtn_block_load_store(b, op, load, index, elem_off,
 652                                access_offset, access_size,
 653                                NULL, 0,
 654                                type->array_element, &(*inout)->elems[i]);
 655       }
 656       return;
 657    }
 658
 659    case GLSL_TYPE_STRUCT: {
 660       unsigned elems = glsl_get_length(type->type);
 661       for (unsigned i = 0; i < elems; i++) {
 662          nir_ssa_def *elem_off =
 663             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
 664          _vtn_block_load_store(b, op, load, index, elem_off,
 665                                access_offset, access_size,
 666                                NULL, 0,
 667                                type->members[i], &(*inout)->elems[i]);
 668       }
 669       return;
 670    }
 671
 672    default:
 673       unreachable("Invalid block member type");
 674    }
 675 }
 676
 677 static struct vtn_ssa_value *
 678 vtn_block_load(struct vtn_builder *b, struct vtn_access_chain *src)
 679 {
 680    nir_intrinsic_op op;
 681    unsigned access_offset = 0, access_size = 0;
 682    switch (src->var->mode) {
 683    case vtn_variable_mode_ubo:
 684       op = nir_intrinsic_load_ubo;
 685       break;
 686    case vtn_variable_mode_ssbo:
 687       op = nir_intrinsic_load_ssbo;
 688       break;
 689    case vtn_variable_mode_push_constant:
 690       op = nir_intrinsic_load_push_constant;
 691       vtn_access_chain_get_offset_size(src, &access_offset, &access_size);
 692       break;
 693    default:
 694       assert(!"Invalid block variable mode");
 695    }
 696
 697    nir_ssa_def *offset, *index = NULL;
 698    struct vtn_type *type;
 699    unsigned chain_idx;
 700    offset = vtn_access_chain_to_offset(b, src, &index, &type, &chain_idx, true);
 701
 702    struct vtn_ssa_value *value = NULL;
 703    _vtn_block_load_store(b, op, true, index, offset,
 704                          access_offset, access_size,
 705                          src, chain_idx, type, &value);
 706    return value;
 707 }
 708
 709 static void
 710 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 711                 struct vtn_access_chain *dst)
 712 {
 713    nir_ssa_def *offset, *index = NULL;
 714    struct vtn_type *type;
 715    unsigned chain_idx;
 716    offset = vtn_access_chain_to_offset(b, dst, &index, &type, &chain_idx, true);
 717
 718    _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset,
 719                          0, 0, dst, chain_idx, type, &src);
 720 }
 721
 722 static bool
 723 vtn_variable_is_external_block(struct vtn_variable *var)
 724 {
 725    return var->mode == vtn_variable_mode_ssbo ||
 726           var->mode == vtn_variable_mode_ubo ||
 727           var->mode == vtn_variable_mode_push_constant;
 728 }
 729
 730 static void
 731 _vtn_variable_load_store(struct vtn_builder *b, bool load,
 732                          struct vtn_access_chain *chain,
 733                          struct vtn_type *tail_type,
 734                          struct vtn_ssa_value **inout)
 735 {
 736    enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
 737    switch (base_type) {
 738    case GLSL_TYPE_UINT:
 739    case GLSL_TYPE_INT:
 740    case GLSL_TYPE_FLOAT:
 741    case GLSL_TYPE_BOOL:
 742       /* At this point, we have a scalar, vector, or matrix so we know that
 743        * there cannot be any structure splitting still in the way.  By
 744        * stopping at the matrix level rather than the vector level, we
 745        * ensure that matrices get loaded in the optimal way even if they
 746        * are storred row-major in a UBO.
 747        */
 748       if (load) {
 749          *inout = vtn_local_load(b, vtn_access_chain_to_deref(b, chain));
 750       } else {
 751          vtn_local_store(b, *inout, vtn_access_chain_to_deref(b, chain));
 752       }
 753       return;
 754
 755    case GLSL_TYPE_ARRAY:
 756    case GLSL_TYPE_STRUCT: {
 757       struct vtn_access_chain *new_chain =
 758          vtn_access_chain_extend(b, chain, 1);
 759       new_chain->link[chain->length].mode = vtn_access_mode_literal;
 760       unsigned elems = glsl_get_length(tail_type->type);
 761       if (load) {
 762          assert(*inout == NULL);
 763          *inout = rzalloc(b, struct vtn_ssa_value);
 764          (*inout)->type = tail_type->type;
 765          (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
 766       }
 767       for (unsigned i = 0; i < elems; i++) {
 768          new_chain->link[chain->length].id = i;
 769          struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
 770             tail_type->array_element : tail_type->members[i];
 771          _vtn_variable_load_store(b, load, new_chain, elem_type,
 772                                   &(*inout)->elems[i]);
 773       }
 774       return;
 775    }
 776
 777    default:
 778       unreachable("Invalid access chain type");
 779    }
 780 }
 781
 782 struct vtn_ssa_value *
 783 vtn_variable_load(struct vtn_builder *b, struct vtn_access_chain *src)
 784 {
 785    if (vtn_variable_is_external_block(src->var)) {
 786       return vtn_block_load(b, src);
 787    } else {
 788       struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
 789       struct vtn_ssa_value *val = NULL;
 790       _vtn_variable_load_store(b, true, src, tail_type, &val);
 791       return val;
 792    }
 793 }
 794
 795 void
 796 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 797                    struct vtn_access_chain *dest)
 798 {
 799    if (vtn_variable_is_external_block(dest->var)) {
 800       assert(dest->var->mode == vtn_variable_mode_ssbo);
 801       vtn_block_store(b, src, dest);
 802    } else {
 803       struct vtn_type *tail_type = vtn_access_chain_tail_type(b, dest);
 804       _vtn_variable_load_store(b, false, dest, tail_type, &src);
 805    }
 806 }
 807
 808 static void
 809 _vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
 810                    struct vtn_access_chain *src, struct vtn_type *tail_type)
 811 {
 812    enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
 813    switch (base_type) {
 814    case GLSL_TYPE_UINT:
 815    case GLSL_TYPE_INT:
 816    case GLSL_TYPE_FLOAT:
 817    case GLSL_TYPE_BOOL:
 818       /* At this point, we have a scalar, vector, or matrix so we know that
 819        * there cannot be any structure splitting still in the way.  By
 820        * stopping at the matrix level rather than the vector level, we
 821        * ensure that matrices get loaded in the optimal way even if they
 822        * are storred row-major in a UBO.
 823        */
 824       vtn_variable_store(b, vtn_variable_load(b, src), dest);
 825       return;
 826
 827    case GLSL_TYPE_ARRAY:
 828    case GLSL_TYPE_STRUCT: {
 829       struct vtn_access_chain *new_src, *new_dest;
 830       new_src = vtn_access_chain_extend(b, src, 1);
 831       new_dest = vtn_access_chain_extend(b, dest, 1);
 832       new_src->link[src->length].mode = vtn_access_mode_literal;
 833       new_dest->link[dest->length].mode = vtn_access_mode_literal;
 834       unsigned elems = glsl_get_length(tail_type->type);
 835       for (unsigned i = 0; i < elems; i++) {
 836          new_src->link[src->length].id = i;
 837          new_dest->link[dest->length].id = i;
 838          struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
 839             tail_type->array_element : tail_type->members[i];
 840          _vtn_variable_copy(b, new_dest, new_src, elem_type);
 841       }
 842       return;
 843    }
 844
 845    default:
 846       unreachable("Invalid access chain type");
 847    }
 848 }
 849
 850 static void
 851 vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
 852                   struct vtn_access_chain *src)
 853 {
 854    struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
 855    assert(vtn_access_chain_tail_type(b, dest)->type == tail_type->type);
 856
 857    /* TODO: At some point, we should add a special-case for when we can
 858     * just emit a copy_var intrinsic.
 859     */
 860    _vtn_variable_copy(b, dest, src, tail_type);
 861 }
 862
 863 static void
 864 set_mode_system_value(nir_variable_mode *mode)
 865 {
 866    assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
 867    *mode = nir_var_system_value;
 868 }
 869
 870 static void
 871 vtn_get_builtin_location(struct vtn_builder *b,
 872                          SpvBuiltIn builtin, int *location,
 873                          nir_variable_mode *mode)
 874 {
 875    switch (builtin) {
 876    case SpvBuiltInPosition:
 877       *location = VARYING_SLOT_POS;
 878       break;
 879    case SpvBuiltInPointSize:
 880       *location = VARYING_SLOT_PSIZ;
 881       break;
 882    case SpvBuiltInClipDistance:
 883       *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
 884       break;
 885    case SpvBuiltInCullDistance:
 886       *location = VARYING_SLOT_CULL_DIST0;
 887       break;
 888    case SpvBuiltInVertexIndex:
 889       *location = SYSTEM_VALUE_VERTEX_ID;
 890       set_mode_system_value(mode);
 891       break;
 892    case SpvBuiltInVertexId:
 893       /* Vulkan defines VertexID to be zero-based and reserves the new
 894        * builtin keyword VertexIndex to indicate the non-zero-based value.
 895        */
 896       *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
 897       set_mode_system_value(mode);
 898       break;
 899    case SpvBuiltInInstanceIndex:
 900       *location = SYSTEM_VALUE_INSTANCE_INDEX;
 901       set_mode_system_value(mode);
 902       break;
 903    case SpvBuiltInInstanceId:
 904       *location = SYSTEM_VALUE_INSTANCE_ID;
 905       set_mode_system_value(mode);
 906       break;
 907    case SpvBuiltInPrimitiveId:
 908       if (*mode == nir_var_shader_out) {
 909          *location = VARYING_SLOT_PRIMITIVE_ID;
 910       } else {
 911          *location = SYSTEM_VALUE_PRIMITIVE_ID;
 912          set_mode_system_value(mode);
 913       }
 914       break;
 915    case SpvBuiltInInvocationId:
 916       *location = SYSTEM_VALUE_INVOCATION_ID;
 917       set_mode_system_value(mode);
 918       break;
 919    case SpvBuiltInLayer:
 920       *location = VARYING_SLOT_LAYER;
 921       if (b->shader->stage == MESA_SHADER_FRAGMENT)
 922          *mode = nir_var_shader_in;
 923       else if (b->shader->stage == MESA_SHADER_GEOMETRY)
 924          *mode = nir_var_shader_out;
 925       else
 926          unreachable("invalid stage for SpvBuiltInLayer");
 927       break;
 928    case SpvBuiltInViewportIndex:
 929       *location = VARYING_SLOT_VIEWPORT;
 930       if (b->shader->stage == MESA_SHADER_GEOMETRY)
 931          *mode = nir_var_shader_out;
 932       else if (b->shader->stage == MESA_SHADER_FRAGMENT)
 933          *mode = nir_var_shader_in;
 934       else
 935          unreachable("invalid stage for SpvBuiltInViewportIndex");
 936       break;
 937    case SpvBuiltInTessLevelOuter:
 938    case SpvBuiltInTessLevelInner:
 939    case SpvBuiltInTessCoord:
 940    case SpvBuiltInPatchVertices:
 941       unreachable("no tessellation support");
 942    case SpvBuiltInFragCoord:
 943       *location = VARYING_SLOT_POS;
 944       assert(*mode == nir_var_shader_in);
 945       break;
 946    case SpvBuiltInPointCoord:
 947       *location = VARYING_SLOT_PNTC;
 948       assert(*mode == nir_var_shader_in);
 949       break;
 950    case SpvBuiltInFrontFacing:
 951       *location = SYSTEM_VALUE_FRONT_FACE;
 952       set_mode_system_value(mode);
 953       break;
 954    case SpvBuiltInSampleId:
 955       *location = SYSTEM_VALUE_SAMPLE_ID;
 956       set_mode_system_value(mode);
 957       break;
 958    case SpvBuiltInSamplePosition:
 959       *location = SYSTEM_VALUE_SAMPLE_POS;
 960       set_mode_system_value(mode);
 961       break;
 962    case SpvBuiltInSampleMask:
 963       *location = SYSTEM_VALUE_SAMPLE_MASK_IN; /* XXX out? */
 964       set_mode_system_value(mode);
 965       break;
 966    case SpvBuiltInFragDepth:
 967       *location = FRAG_RESULT_DEPTH;
 968       assert(*mode == nir_var_shader_out);
 969       break;
 970    case SpvBuiltInNumWorkgroups:
 971       *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
 972       set_mode_system_value(mode);
 973       break;
 974    case SpvBuiltInWorkgroupSize:
 975       /* This should already be handled */
 976       unreachable("unsupported builtin");
 977       break;
 978    case SpvBuiltInWorkgroupId:
 979       *location = SYSTEM_VALUE_WORK_GROUP_ID;
 980       set_mode_system_value(mode);
 981       break;
 982    case SpvBuiltInLocalInvocationId:
 983       *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
 984       set_mode_system_value(mode);
 985       break;
 986    case SpvBuiltInLocalInvocationIndex:
 987       *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
 988       set_mode_system_value(mode);
 989       break;
 990    case SpvBuiltInGlobalInvocationId:
 991       *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
 992       set_mode_system_value(mode);
 993       break;
 994    case SpvBuiltInHelperInvocation:
 995    default:
 996       unreachable("unsupported builtin");
 997    }
 998 }
 999
1000 static void
1001 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var,
1002                      const struct vtn_decoration *dec)
1003 {
1004    switch (dec->decoration) {
1005    case SpvDecorationRelaxedPrecision:
1006       break; /* FIXME: Do nothing with this for now. */
1007    case SpvDecorationNoPerspective:
1008       nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1009       break;
1010    case SpvDecorationFlat:
1011       nir_var->data.interpolation = INTERP_MODE_FLAT;
1012       break;
1013    case SpvDecorationCentroid:
1014       nir_var->data.centroid = true;
1015       break;
1016    case SpvDecorationSample:
1017       nir_var->data.sample = true;
1018       break;
1019    case SpvDecorationInvariant:
1020       nir_var->data.invariant = true;
1021       break;
1022    case SpvDecorationConstant:
1023       assert(nir_var->constant_initializer != NULL);
1024       nir_var->data.read_only = true;
1025       break;
1026    case SpvDecorationNonWritable:
1027       nir_var->data.read_only = true;
1028       break;
1029    case SpvDecorationComponent:
1030       nir_var->data.location_frac = dec->literals[0];
1031       break;
1032    case SpvDecorationIndex:
1033       nir_var->data.index = dec->literals[0];
1034       break;
1035    case SpvDecorationBuiltIn: {
1036       SpvBuiltIn builtin = dec->literals[0];
1037
1038       if (builtin == SpvBuiltInWorkgroupSize) {
1039          /* This shouldn't be a builtin.  It's actually a constant. */
1040          nir_var->data.mode = nir_var_global;
1041          nir_var->data.read_only = true;
1042
1043          nir_constant *c = rzalloc(nir_var, nir_constant);
1044          c->values[0].u32[0] = b->shader->info->cs.local_size[0];
1045          c->values[0].u32[1] = b->shader->info->cs.local_size[1];
1046          c->values[0].u32[2] = b->shader->info->cs.local_size[2];
1047          nir_var->constant_initializer = c;
1048          break;
1049       }
1050
1051       nir_variable_mode mode = nir_var->data.mode;
1052       vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode);
1053       nir_var->data.mode = mode;
1054
1055       if (builtin == SpvBuiltInFragCoord || builtin == SpvBuiltInSamplePosition)
1056          nir_var->data.origin_upper_left = b->origin_upper_left;
1057
1058       if (builtin == SpvBuiltInFragCoord)
1059          nir_var->data.pixel_center_integer = b->pixel_center_integer;
1060       break;
1061    }
1062
1063    case SpvDecorationSpecId:
1064    case SpvDecorationRowMajor:
1065    case SpvDecorationColMajor:
1066    case SpvDecorationMatrixStride:
1067    case SpvDecorationRestrict:
1068    case SpvDecorationAliased:
1069    case SpvDecorationVolatile:
1070    case SpvDecorationCoherent:
1071    case SpvDecorationNonReadable:
1072    case SpvDecorationUniform:
1073    case SpvDecorationStream:
1074    case SpvDecorationOffset:
1075    case SpvDecorationLinkageAttributes:
1076       break; /* Do nothing with these here */
1077
1078    case SpvDecorationPatch:
1079       vtn_warn("Tessellation not yet supported");
1080       break;
1081
1082    case SpvDecorationLocation:
1083       unreachable("Handled above");
1084
1085    case SpvDecorationBlock:
1086    case SpvDecorationBufferBlock:
1087    case SpvDecorationArrayStride:
1088    case SpvDecorationGLSLShared:
1089    case SpvDecorationGLSLPacked:
1090       break; /* These can apply to a type but we don't care about them */
1091
1092    case SpvDecorationBinding:
1093    case SpvDecorationDescriptorSet:
1094    case SpvDecorationNoContraction:
1095    case SpvDecorationInputAttachmentIndex:
1096       vtn_warn("Decoration not allowed for variable or structure member: %s",
1097                spirv_decoration_to_string(dec->decoration));
1098       break;
1099
1100    case SpvDecorationXfbBuffer:
1101    case SpvDecorationXfbStride:
1102       vtn_warn("Vulkan does not have transform feedback: %s",
1103                spirv_decoration_to_string(dec->decoration));
1104       break;
1105
1106    case SpvDecorationCPacked:
1107    case SpvDecorationSaturatedConversion:
1108    case SpvDecorationFuncParamAttr:
1109    case SpvDecorationFPRoundingMode:
1110    case SpvDecorationFPFastMathMode:
1111    case SpvDecorationAlignment:
1112       vtn_warn("Decoraiton only allowed for CL-style kernels: %s",
1113                spirv_decoration_to_string(dec->decoration));
1114       break;
1115    }
1116 }
1117
1118 static void
1119 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1120                   const struct vtn_decoration *dec, void *void_var)
1121 {
1122    struct vtn_variable *vtn_var = void_var;
1123
1124    /* Handle decorations that apply to a vtn_variable as a whole */
1125    switch (dec->decoration) {
1126    case SpvDecorationBinding:
1127       vtn_var->binding = dec->literals[0];
1128       return;
1129    case SpvDecorationDescriptorSet:
1130       vtn_var->descriptor_set = dec->literals[0];
1131       return;
1132    case SpvDecorationInputAttachmentIndex:
1133       vtn_var->input_attachment_index = dec->literals[0];
1134       return;
1135    default:
1136       break;
1137    }
1138
1139    if (val->value_type == vtn_value_type_access_chain) {
1140       assert(val->access_chain->length == 0);
1141       assert(val->access_chain->var == void_var);
1142       assert(member == -1);
1143    } else {
1144       assert(val->value_type == vtn_value_type_type);
1145    }
1146
1147    /* Location is odd.  If applied to a split structure, we have to walk the
1148     * whole thing and accumulate the location.  It's easier to handle as a
1149     * special case.
1150     */
1151    if (dec->decoration == SpvDecorationLocation) {
1152       unsigned location = dec->literals[0];
1153       bool is_vertex_input;
1154       if (b->shader->stage == MESA_SHADER_FRAGMENT &&
1155           vtn_var->mode == vtn_variable_mode_output) {
1156          is_vertex_input = false;
1157          location += FRAG_RESULT_DATA0;
1158       } else if (b->shader->stage == MESA_SHADER_VERTEX &&
1159                  vtn_var->mode == vtn_variable_mode_input) {
1160          is_vertex_input = true;
1161          location += VERT_ATTRIB_GENERIC0;
1162       } else if (vtn_var->mode == vtn_variable_mode_input ||
1163                  vtn_var->mode == vtn_variable_mode_output) {
1164          is_vertex_input = false;
1165          location += VARYING_SLOT_VAR0;
1166       } else {
1167          unreachable("Location must be on input or output variable");
1168       }
1169
1170       if (vtn_var->var) {
1171          /* This handles the member and lone variable cases */
1172          vtn_var->var->data.location = location;
1173       } else {
1174          /* This handles the structure member case */
1175          assert(vtn_var->members);
1176          unsigned length =
1177             glsl_get_length(glsl_without_array(vtn_var->type->type));
1178          for (unsigned i = 0; i < length; i++) {
1179             vtn_var->members[i]->data.location = location;
1180             location +=
1181                glsl_count_attribute_slots(vtn_var->members[i]->interface_type,
1182                                           is_vertex_input);
1183          }
1184       }
1185       return;
1186    } else {
1187       if (vtn_var->var) {
1188          assert(member <= 0);
1189          apply_var_decoration(b, vtn_var->var, dec);
1190       } else if (vtn_var->members) {
1191          if (member >= 0) {
1192             assert(vtn_var->members);
1193             apply_var_decoration(b, vtn_var->members[member], dec);
1194          } else {
1195             unsigned length =
1196                glsl_get_length(glsl_without_array(vtn_var->type->type));
1197             for (unsigned i = 0; i < length; i++)
1198                apply_var_decoration(b, vtn_var->members[i], dec);
1199          }
1200       } else {
1201          /* A few variables, those with external storage, have no actual
1202           * nir_variables associated with them.  Fortunately, all decorations
1203           * we care about for those variables are on the type only.
1204           */
1205          assert(vtn_var->mode == vtn_variable_mode_ubo ||
1206                 vtn_var->mode == vtn_variable_mode_ssbo ||
1207                 vtn_var->mode == vtn_variable_mode_push_constant);
1208       }
1209    }
1210 }
1211
1212 void
1213 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1214                      const uint32_t *w, unsigned count)
1215 {
1216    switch (opcode) {
1217    case SpvOpVariable: {
1218       struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1219       var->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1220
1221       var->chain.var = var;
1222       var->chain.length = 0;
1223
1224       struct vtn_value *val =
1225          vtn_push_value(b, w[2], vtn_value_type_access_chain);
1226       val->access_chain = &var->chain;
1227
1228       struct vtn_type *without_array = var->type;
1229       while(glsl_type_is_array(without_array->type))
1230          without_array = without_array->array_element;
1231
1232       nir_variable_mode nir_mode;
1233       switch ((SpvStorageClass)w[3]) {
1234       case SpvStorageClassUniform:
1235       case SpvStorageClassUniformConstant:
1236          if (without_array->block) {
1237             var->mode = vtn_variable_mode_ubo;
1238             b->shader->info->num_ubos++;
1239          } else if (without_array->buffer_block) {
1240             var->mode = vtn_variable_mode_ssbo;
1241             b->shader->info->num_ssbos++;
1242          } else if (glsl_type_is_image(without_array->type)) {
1243             var->mode = vtn_variable_mode_image;
1244             nir_mode = nir_var_uniform;
1245             b->shader->info->num_images++;
1246          } else if (glsl_type_is_sampler(without_array->type)) {
1247             var->mode = vtn_variable_mode_sampler;
1248             nir_mode = nir_var_uniform;
1249             b->shader->info->num_textures++;
1250          } else {
1251             assert(!"Invalid uniform variable type");
1252          }
1253          break;
1254       case SpvStorageClassPushConstant:
1255          var->mode = vtn_variable_mode_push_constant;
1256          assert(b->shader->num_uniforms == 0);
1257          b->shader->num_uniforms = vtn_type_block_size(var->type);
1258          break;
1259       case SpvStorageClassInput:
1260          var->mode = vtn_variable_mode_input;
1261          nir_mode = nir_var_shader_in;
1262          break;
1263       case SpvStorageClassOutput:
1264          var->mode = vtn_variable_mode_output;
1265          nir_mode = nir_var_shader_out;
1266          break;
1267       case SpvStorageClassPrivate:
1268          var->mode = vtn_variable_mode_global;
1269          nir_mode = nir_var_global;
1270          break;
1271       case SpvStorageClassFunction:
1272          var->mode = vtn_variable_mode_local;
1273          nir_mode = nir_var_local;
1274          break;
1275       case SpvStorageClassWorkgroup:
1276          var->mode = vtn_variable_mode_workgroup;
1277          nir_mode = nir_var_shared;
1278          break;
1279       case SpvStorageClassCrossWorkgroup:
1280       case SpvStorageClassGeneric:
1281       case SpvStorageClassAtomicCounter:
1282       default:
1283          unreachable("Unhandled variable storage class");
1284       }
1285
1286       switch (var->mode) {
1287       case vtn_variable_mode_local:
1288       case vtn_variable_mode_global:
1289       case vtn_variable_mode_image:
1290       case vtn_variable_mode_sampler:
1291       case vtn_variable_mode_workgroup:
1292          /* For these, we create the variable normally */
1293          var->var = rzalloc(b->shader, nir_variable);
1294          var->var->name = ralloc_strdup(var->var, val->name);
1295          var->var->type = var->type->type;
1296          var->var->data.mode = nir_mode;
1297
1298          switch (var->mode) {
1299          case vtn_variable_mode_image:
1300          case vtn_variable_mode_sampler:
1301             var->var->interface_type = without_array->type;
1302             break;
1303          default:
1304             var->var->interface_type = NULL;
1305             break;
1306          }
1307          break;
1308
1309       case vtn_variable_mode_input:
1310       case vtn_variable_mode_output: {
1311          /* For inputs and outputs, we immediately split structures.  This
1312           * is for a couple of reasons.  For one, builtins may all come in
1313           * a struct and we really want those split out into separate
1314           * variables.  For another, interpolation qualifiers can be
1315           * applied to members of the top-level struct ane we need to be
1316           * able to preserve that information.
1317           */
1318
1319          int array_length = -1;
1320          struct vtn_type *interface_type = var->type;
1321          if (b->shader->stage == MESA_SHADER_GEOMETRY &&
1322              glsl_type_is_array(var->type->type)) {
1323             /* In Geometry shaders (and some tessellation), inputs come
1324              * in per-vertex arrays.  However, some builtins come in
1325              * non-per-vertex, hence the need for the is_array check.  In
1326              * any case, there are no non-builtin arrays allowed so this
1327              * check should be sufficient.
1328              */
1329             interface_type = var->type->array_element;
1330             array_length = glsl_get_length(var->type->type);
1331          }
1332
1333          if (glsl_type_is_struct(interface_type->type)) {
1334             /* It's a struct.  Split it. */
1335             unsigned num_members = glsl_get_length(interface_type->type);
1336             var->members = ralloc_array(b, nir_variable *, num_members);
1337
1338             for (unsigned i = 0; i < num_members; i++) {
1339                const struct glsl_type *mtype = interface_type->members[i]->type;
1340                if (array_length >= 0)
1341                   mtype = glsl_array_type(mtype, array_length);
1342
1343                var->members[i] = rzalloc(b->shader, nir_variable);
1344                var->members[i]->name =
1345                   ralloc_asprintf(var->members[i], "%s.%d", val->name, i);
1346                var->members[i]->type = mtype;
1347                var->members[i]->interface_type =
1348                   interface_type->members[i]->type;
1349                var->members[i]->data.mode = nir_mode;
1350             }
1351          } else {
1352             var->var = rzalloc(b->shader, nir_variable);
1353             var->var->name = ralloc_strdup(var->var, val->name);
1354             var->var->type = var->type->type;
1355             var->var->interface_type = interface_type->type;
1356             var->var->data.mode = nir_mode;
1357          }
1358
1359          /* For inputs and outputs, we need to grab locations and builtin
1360           * information from the interface type.
1361           */
1362          vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var);
1363          break;
1364
1365       case vtn_variable_mode_param:
1366          unreachable("Not created through OpVariable");
1367       }
1368
1369       case vtn_variable_mode_ubo:
1370       case vtn_variable_mode_ssbo:
1371       case vtn_variable_mode_push_constant:
1372          /* These don't need actual variables. */
1373          break;
1374       }
1375
1376       if (count > 4) {
1377          assert(count == 5);
1378          nir_constant *constant =
1379             vtn_value(b, w[4], vtn_value_type_constant)->constant;
1380          var->var->constant_initializer =
1381             nir_constant_clone(constant, var->var);
1382       }
1383
1384       vtn_foreach_decoration(b, val, var_decoration_cb, var);
1385
1386       if (var->mode == vtn_variable_mode_image ||
1387           var->mode == vtn_variable_mode_sampler) {
1388          /* XXX: We still need the binding information in the nir_variable
1389           * for these. We should fix that.
1390           */
1391          var->var->data.binding = var->binding;
1392          var->var->data.descriptor_set = var->descriptor_set;
1393          var->var->data.index = var->input_attachment_index;
1394
1395          if (var->mode == vtn_variable_mode_image)
1396             var->var->data.image.format = without_array->image_format;
1397       }
1398
1399       if (var->mode == vtn_variable_mode_local) {
1400          assert(var->members == NULL && var->var != NULL);
1401          nir_function_impl_add_variable(b->impl, var->var);
1402       } else if (var->var) {
1403          nir_shader_add_variable(b->shader, var->var);
1404       } else if (var->members) {
1405          unsigned count = glsl_get_length(without_array->type);
1406          for (unsigned i = 0; i < count; i++) {
1407             assert(var->members[i]->data.mode != nir_var_local);
1408             nir_shader_add_variable(b->shader, var->members[i]);
1409          }
1410       } else {
1411          assert(var->mode == vtn_variable_mode_ubo ||
1412                 var->mode == vtn_variable_mode_ssbo ||
1413                 var->mode == vtn_variable_mode_push_constant);
1414       }
1415       break;
1416    }
1417
1418    case SpvOpAccessChain:
1419    case SpvOpInBoundsAccessChain: {
1420       struct vtn_access_chain *base, *chain;
1421       struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1422       if (base_val->value_type == vtn_value_type_sampled_image) {
1423          /* This is rather insane.  SPIR-V allows you to use OpSampledImage
1424           * to combine an array of images with a single sampler to get an
1425           * array of sampled images that all share the same sampler.
1426           * Fortunately, this means that we can more-or-less ignore the
1427           * sampler when crawling the access chain, but it does leave us
1428           * with this rather awkward little special-case.
1429           */
1430          base = base_val->sampled_image->image;
1431       } else {
1432          assert(base_val->value_type == vtn_value_type_access_chain);
1433          base = base_val->access_chain;
1434       }
1435
1436       chain = vtn_access_chain_extend(b, base, count - 4);
1437
1438       unsigned idx = base->length;
1439       for (int i = 4; i < count; i++) {
1440          struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1441          if (link_val->value_type == vtn_value_type_constant) {
1442             chain->link[idx].mode = vtn_access_mode_literal;
1443             chain->link[idx].id = link_val->constant->values[0].u32[0];
1444          } else {
1445             chain->link[idx].mode = vtn_access_mode_id;
1446             chain->link[idx].id = w[i];
1447          }
1448          idx++;
1449       }
1450
1451       if (base_val->value_type == vtn_value_type_sampled_image) {
1452          struct vtn_value *val =
1453             vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1454          val->sampled_image = ralloc(b, struct vtn_sampled_image);
1455          val->sampled_image->image = chain;
1456          val->sampled_image->sampler = base_val->sampled_image->sampler;
1457       } else {
1458          struct vtn_value *val =
1459             vtn_push_value(b, w[2], vtn_value_type_access_chain);
1460          val->access_chain = chain;
1461       }
1462       break;
1463    }
1464
1465    case SpvOpCopyMemory: {
1466       struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_access_chain);
1467       struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_access_chain);
1468
1469       vtn_variable_copy(b, dest->access_chain, src->access_chain);
1470       break;
1471    }
1472
1473    case SpvOpLoad: {
1474       struct vtn_access_chain *src =
1475          vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1476
1477       if (src->var->mode == vtn_variable_mode_image ||
1478           src->var->mode == vtn_variable_mode_sampler) {
1479          vtn_push_value(b, w[2], vtn_value_type_access_chain)->access_chain = src;
1480          return;
1481       }
1482
1483       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1484       val->ssa = vtn_variable_load(b, src);
1485       break;
1486    }
1487
1488    case SpvOpStore: {
1489       struct vtn_access_chain *dest =
1490          vtn_value(b, w[1], vtn_value_type_access_chain)->access_chain;
1491       struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1492       vtn_variable_store(b, src, dest);
1493       break;
1494    }
1495
1496    case SpvOpArrayLength: {
1497       struct vtn_access_chain *chain =
1498          vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1499
1500       const uint32_t offset = chain->var->type->offsets[w[4]];
1501       const uint32_t stride = chain->var->type->members[w[4]]->stride;
1502
1503       unsigned chain_idx;
1504       struct vtn_type *type;
1505       nir_ssa_def *index =
1506          get_vulkan_resource_index(b, chain, &type, &chain_idx);
1507
1508       nir_intrinsic_instr *instr =
1509          nir_intrinsic_instr_create(b->nb.shader,
1510                                     nir_intrinsic_get_buffer_size);
1511       instr->src[0] = nir_src_for_ssa(index);
1512       nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1513       nir_builder_instr_insert(&b->nb, &instr->instr);
1514       nir_ssa_def *buf_size = &instr->dest.ssa;
1515
1516       /* array_length = max(buffer_size - offset, 0) / stride */
1517       nir_ssa_def *array_length =
1518          nir_idiv(&b->nb,
1519                   nir_imax(&b->nb,
1520                            nir_isub(&b->nb,
1521                                     buf_size,
1522                                     nir_imm_int(&b->nb, offset)),
1523                            nir_imm_int(&b->nb, 0u)),
1524                   nir_imm_int(&b->nb, stride));
1525
1526       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1527       val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
1528       val->ssa->def = array_length;
1529       break;
1530    }
1531
1532    case SpvOpCopyMemorySized:
1533    default:
1534       unreachable("Unhandled opcode");
1535    }
1536 }