src/compiler/spirv/vtn_variables.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 #include "vtn_private.h"
  29 #include "spirv_info.h"
  30
  31 static struct vtn_access_chain *
  32 vtn_access_chain_create(struct vtn_builder *b, unsigned length)
  33 {
  34    struct vtn_access_chain *chain;
  35
  36    /* Subtract 1 from the length since there's already one built in */
  37    size_t size = sizeof(*chain) +
  38                  (MAX2(length, 1) - 1) * sizeof(chain->link[0]);
  39    chain = rzalloc_size(b, size);
  40    chain->length = length;
  41
  42    return chain;
  43 }
  44
  45 static struct vtn_access_chain *
  46 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
  47                         unsigned new_ids)
  48 {
  49    struct vtn_access_chain *chain;
  50
  51    unsigned old_len = old ? old->length : 0;
  52    chain = vtn_access_chain_create(b, old_len + new_ids);
  53
  54    for (unsigned i = 0; i < old_len; i++)
  55       chain->link[i] = old->link[i];
  56
  57    return chain;
  58 }
  59
  60 /* Dereference the given base pointer by the access chain */
  61 static struct vtn_pointer *
  62 vtn_access_chain_pointer_dereference(struct vtn_builder *b,
  63                                      struct vtn_pointer *base,
  64                                      struct vtn_access_chain *deref_chain)
  65 {
  66    struct vtn_access_chain *chain =
  67       vtn_access_chain_extend(b, base->chain, deref_chain->length);
  68    struct vtn_type *type = base->type;
  69
  70    unsigned start = base->chain ? base->chain->length : 0;
  71    for (unsigned i = 0; i < deref_chain->length; i++) {
  72       chain->link[start + i] = deref_chain->link[i];
  73
  74       if (glsl_type_is_struct(type->type)) {
  75          assert(deref_chain->link[i].mode == vtn_access_mode_literal);
  76          type = type->members[deref_chain->link[i].id];
  77       } else {
  78          type = type->array_element;
  79       }
  80    }
  81
  82    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
  83    ptr->mode = base->mode;
  84    ptr->type = type;
  85    ptr->var = base->var;
  86    ptr->chain = chain;
  87
  88    return ptr;
  89 }
  90
  91 static nir_ssa_def *
  92 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
  93                        unsigned stride)
  94 {
  95    assert(stride > 0);
  96    if (link.mode == vtn_access_mode_literal) {
  97       return nir_imm_int(&b->nb, link.id * stride);
  98    } else if (stride == 1) {
  99       return vtn_ssa_value(b, link.id)->def;
 100    } else {
 101       return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
 102                               nir_imm_int(&b->nb, stride));
 103    }
 104 }
 105
 106 static nir_ssa_def *
 107 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var,
 108                             nir_ssa_def *desc_array_index)
 109 {
 110    if (!desc_array_index) {
 111       assert(glsl_type_is_struct(var->type->type));
 112       desc_array_index = nir_imm_int(&b->nb, 0);
 113    }
 114
 115    nir_intrinsic_instr *instr =
 116       nir_intrinsic_instr_create(b->nb.shader,
 117                                  nir_intrinsic_vulkan_resource_index);
 118    instr->src[0] = nir_src_for_ssa(desc_array_index);
 119    nir_intrinsic_set_desc_set(instr, var->descriptor_set);
 120    nir_intrinsic_set_binding(instr, var->binding);
 121
 122    nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 123    nir_builder_instr_insert(&b->nb, &instr->instr);
 124
 125    return &instr->dest.ssa;
 126 }
 127
 128 static struct vtn_pointer *
 129 vtn_ssa_offset_pointer_dereference(struct vtn_builder *b,
 130                                    struct vtn_pointer *base,
 131                                    struct vtn_access_chain *deref_chain)
 132 {
 133    nir_ssa_def *block_index = base->block_index;
 134    nir_ssa_def *offset = base->offset;
 135    struct vtn_type *type = base->type;
 136
 137    unsigned idx = 0;
 138    if (!block_index) {
 139       assert(base->var);
 140       if (glsl_type_is_array(type->type)) {
 141          /* We need at least one element in the chain */
 142          assert(deref_chain->length >= 1);
 143
 144          nir_ssa_def *desc_arr_idx =
 145             vtn_access_link_as_ssa(b, deref_chain->link[0], 1);
 146          block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
 147          type = type->array_element;
 148          idx++;
 149       } else {
 150          block_index = vtn_variable_resource_index(b, base->var, NULL);
 151       }
 152
 153       /* This is the first access chain so we also need an offset */
 154       assert(!offset);
 155       offset = nir_imm_int(&b->nb, 0);
 156    }
 157    assert(offset);
 158
 159    for (; idx < deref_chain->length; idx++) {
 160       switch (glsl_get_base_type(type->type)) {
 161       case GLSL_TYPE_UINT:
 162       case GLSL_TYPE_INT:
 163       case GLSL_TYPE_UINT64:
 164       case GLSL_TYPE_INT64:
 165       case GLSL_TYPE_FLOAT:
 166       case GLSL_TYPE_DOUBLE:
 167       case GLSL_TYPE_BOOL:
 168       case GLSL_TYPE_ARRAY: {
 169          nir_ssa_def *elem_offset =
 170             vtn_access_link_as_ssa(b, deref_chain->link[idx], type->stride);
 171          offset = nir_iadd(&b->nb, offset, elem_offset);
 172          type = type->array_element;
 173          break;
 174       }
 175
 176       case GLSL_TYPE_STRUCT: {
 177          assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
 178          unsigned member = deref_chain->link[idx].id;
 179          nir_ssa_def *mem_offset = nir_imm_int(&b->nb, type->offsets[member]);
 180          offset = nir_iadd(&b->nb, offset, mem_offset);
 181          type = type->members[member];
 182          break;
 183       }
 184
 185       default:
 186          unreachable("Invalid type for deref");
 187       }
 188    }
 189
 190    struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
 191    ptr->mode = base->mode;
 192    ptr->type = type;
 193    ptr->block_index = block_index;
 194    ptr->offset = offset;
 195
 196    return ptr;
 197 }
 198
 199 /* Dereference the given base pointer by the access chain */
 200 static struct vtn_pointer *
 201 vtn_pointer_dereference(struct vtn_builder *b,
 202                         struct vtn_pointer *base,
 203                         struct vtn_access_chain *deref_chain)
 204 {
 205    if (base->mode == vtn_variable_mode_ubo ||
 206        base->mode == vtn_variable_mode_ssbo) {
 207       return vtn_ssa_offset_pointer_dereference(b, base, deref_chain);
 208    } else {
 209       return vtn_access_chain_pointer_dereference(b, base, deref_chain);
 210    }
 211 }
 212
 213 /* Crawls a chain of array derefs and rewrites the types so that the
 214  * lengths stay the same but the terminal type is the one given by
 215  * tail_type.  This is useful for split structures.
 216  */
 217 static void
 218 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type)
 219 {
 220    deref->type = type;
 221    if (deref->child) {
 222       assert(deref->child->deref_type == nir_deref_type_array);
 223       assert(glsl_type_is_array(deref->type));
 224       rewrite_deref_types(deref->child, glsl_get_array_element(type));
 225    }
 226 }
 227
 228 struct vtn_pointer *
 229 vtn_pointer_for_variable(struct vtn_builder *b,
 230                          struct vtn_variable *var, struct vtn_type *ptr_type)
 231 {
 232    struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer);
 233
 234    pointer->mode = var->mode;
 235    pointer->type = var->type;
 236    pointer->ptr_type = ptr_type;
 237    pointer->var = var;
 238
 239    return pointer;
 240 }
 241
 242 nir_deref_var *
 243 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr)
 244 {
 245    /* Do on-the-fly copy propagation for samplers. */
 246    if (ptr->var->copy_prop_sampler)
 247       return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler);
 248
 249    nir_deref_var *deref_var;
 250    if (ptr->var->var) {
 251       deref_var = nir_deref_var_create(b, ptr->var->var);
 252       /* Raw variable access */
 253       if (!ptr->chain)
 254          return deref_var;
 255    } else {
 256       assert(ptr->var->members);
 257       /* Create the deref_var manually.  It will get filled out later. */
 258       deref_var = rzalloc(b, nir_deref_var);
 259       deref_var->deref.deref_type = nir_deref_type_var;
 260    }
 261
 262    struct vtn_access_chain *chain = ptr->chain;
 263    assert(chain);
 264
 265    struct vtn_type *deref_type = ptr->var->type;
 266    nir_deref *tail = &deref_var->deref;
 267    nir_variable **members = ptr->var->members;
 268
 269    for (unsigned i = 0; i < chain->length; i++) {
 270       enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
 271       switch (base_type) {
 272       case GLSL_TYPE_UINT:
 273       case GLSL_TYPE_INT:
 274       case GLSL_TYPE_UINT64:
 275       case GLSL_TYPE_INT64:
 276       case GLSL_TYPE_FLOAT:
 277       case GLSL_TYPE_DOUBLE:
 278       case GLSL_TYPE_BOOL:
 279       case GLSL_TYPE_ARRAY: {
 280          deref_type = deref_type->array_element;
 281
 282          nir_deref_array *deref_arr = nir_deref_array_create(b);
 283          deref_arr->deref.type = deref_type->type;
 284
 285          if (chain->link[i].mode == vtn_access_mode_literal) {
 286             deref_arr->deref_array_type = nir_deref_array_type_direct;
 287             deref_arr->base_offset = chain->link[i].id;
 288          } else {
 289             assert(chain->link[i].mode == vtn_access_mode_id);
 290             deref_arr->deref_array_type = nir_deref_array_type_indirect;
 291             deref_arr->base_offset = 0;
 292             deref_arr->indirect =
 293                nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
 294          }
 295          tail->child = &deref_arr->deref;
 296          tail = tail->child;
 297          break;
 298       }
 299
 300       case GLSL_TYPE_STRUCT: {
 301          assert(chain->link[i].mode == vtn_access_mode_literal);
 302          unsigned idx = chain->link[i].id;
 303          deref_type = deref_type->members[idx];
 304          if (members) {
 305             /* This is a pre-split structure. */
 306             deref_var->var = members[idx];
 307             rewrite_deref_types(&deref_var->deref, members[idx]->type);
 308             assert(tail->type == deref_type->type);
 309             members = NULL;
 310          } else {
 311             nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
 312             deref_struct->deref.type = deref_type->type;
 313             tail->child = &deref_struct->deref;
 314             tail = tail->child;
 315          }
 316          break;
 317       }
 318       default:
 319          unreachable("Invalid type for deref");
 320       }
 321    }
 322
 323    assert(members == NULL);
 324    return deref_var;
 325 }
 326
 327 static void
 328 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref,
 329                       nir_deref *tail, struct vtn_ssa_value *inout)
 330 {
 331    /* The deref tail may contain a deref to select a component of a vector (in
 332     * other words, it might not be an actual tail) so we have to save it away
 333     * here since we overwrite it later.
 334     */
 335    nir_deref *old_child = tail->child;
 336
 337    if (glsl_type_is_vector_or_scalar(tail->type)) {
 338       /* Terminate the deref chain in case there is one more link to pick
 339        * off a component of the vector.
 340        */
 341       tail->child = NULL;
 342
 343       nir_intrinsic_op op = load ? nir_intrinsic_load_var :
 344                                    nir_intrinsic_store_var;
 345
 346       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op);
 347       intrin->variables[0] = nir_deref_var_clone(deref, intrin);
 348       intrin->num_components = glsl_get_vector_elements(tail->type);
 349
 350       if (load) {
 351          nir_ssa_dest_init(&intrin->instr, &intrin->dest,
 352                            intrin->num_components,
 353                            glsl_get_bit_size(tail->type),
 354                            NULL);
 355          inout->def = &intrin->dest.ssa;
 356       } else {
 357          nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1);
 358          intrin->src[0] = nir_src_for_ssa(inout->def);
 359       }
 360
 361       nir_builder_instr_insert(&b->nb, &intrin->instr);
 362    } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY ||
 363               glsl_type_is_matrix(tail->type)) {
 364       unsigned elems = glsl_get_length(tail->type);
 365       nir_deref_array *deref_arr = nir_deref_array_create(b);
 366       deref_arr->deref_array_type = nir_deref_array_type_direct;
 367       deref_arr->deref.type = glsl_get_array_element(tail->type);
 368       tail->child = &deref_arr->deref;
 369       for (unsigned i = 0; i < elems; i++) {
 370          deref_arr->base_offset = i;
 371          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 372       }
 373    } else {
 374       assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT);
 375       unsigned elems = glsl_get_length(tail->type);
 376       nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0);
 377       tail->child = &deref_struct->deref;
 378       for (unsigned i = 0; i < elems; i++) {
 379          deref_struct->index = i;
 380          deref_struct->deref.type = glsl_get_struct_field(tail->type, i);
 381          _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
 382       }
 383    }
 384
 385    tail->child = old_child;
 386 }
 387
 388 nir_deref_var *
 389 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
 390 {
 391    struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer;
 392    return vtn_pointer_to_deref(b, ptr);
 393 }
 394
 395 /*
 396  * Gets the NIR-level deref tail, which may have as a child an array deref
 397  * selecting which component due to OpAccessChain supporting per-component
 398  * indexing in SPIR-V.
 399  */
 400 static nir_deref *
 401 get_deref_tail(nir_deref_var *deref)
 402 {
 403    nir_deref *cur = &deref->deref;
 404    while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child)
 405       cur = cur->child;
 406
 407    return cur;
 408 }
 409
 410 struct vtn_ssa_value *
 411 vtn_local_load(struct vtn_builder *b, nir_deref_var *src)
 412 {
 413    nir_deref *src_tail = get_deref_tail(src);
 414    struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
 415    _vtn_local_load_store(b, true, src, src_tail, val);
 416
 417    if (src_tail->child) {
 418       nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child);
 419       assert(vec_deref->deref.child == NULL);
 420       val->type = vec_deref->deref.type;
 421       if (vec_deref->deref_array_type == nir_deref_array_type_direct)
 422          val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset);
 423       else
 424          val->def = vtn_vector_extract_dynamic(b, val->def,
 425                                                vec_deref->indirect.ssa);
 426    }
 427
 428    return val;
 429 }
 430
 431 void
 432 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 433                 nir_deref_var *dest)
 434 {
 435    nir_deref *dest_tail = get_deref_tail(dest);
 436
 437    if (dest_tail->child) {
 438       struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
 439       _vtn_local_load_store(b, true, dest, dest_tail, val);
 440       nir_deref_array *deref = nir_deref_as_array(dest_tail->child);
 441       assert(deref->deref.child == NULL);
 442       if (deref->deref_array_type == nir_deref_array_type_direct)
 443          val->def = vtn_vector_insert(b, val->def, src->def,
 444                                       deref->base_offset);
 445       else
 446          val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
 447                                               deref->indirect.ssa);
 448       _vtn_local_load_store(b, false, dest, dest_tail, val);
 449    } else {
 450       _vtn_local_load_store(b, false, dest, dest_tail, src);
 451    }
 452 }
 453
 454 static nir_ssa_def *
 455 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_pointer *ptr,
 456                           struct vtn_type **type, unsigned *chain_idx)
 457 {
 458    /* Push constants have no explicit binding */
 459    if (ptr->mode == vtn_variable_mode_push_constant) {
 460       *chain_idx = 0;
 461       *type = ptr->var->type;
 462       return NULL;
 463    }
 464
 465    if (glsl_type_is_array(ptr->var->type->type)) {
 466       assert(ptr->chain->length > 0);
 467       nir_ssa_def *desc_array_index =
 468          vtn_access_link_as_ssa(b, ptr->chain->link[0], 1);
 469       *chain_idx = 1;
 470       *type = ptr->var->type->array_element;
 471       return vtn_variable_resource_index(b, ptr->var, desc_array_index);
 472    } else {
 473       *chain_idx = 0;
 474       *type = ptr->var->type;
 475       return vtn_variable_resource_index(b, ptr->var, NULL);
 476    }
 477 }
 478
 479 nir_ssa_def *
 480 vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr,
 481                       nir_ssa_def **index_out, unsigned *end_idx_out)
 482 {
 483    if (ptr->offset) {
 484       assert(ptr->block_index);
 485       *index_out = ptr->block_index;
 486       return ptr->offset;
 487    }
 488
 489    unsigned idx = 0;
 490    struct vtn_type *type;
 491    *index_out = get_vulkan_resource_index(b, ptr, &type, &idx);
 492
 493    nir_ssa_def *offset = nir_imm_int(&b->nb, 0);
 494    for (; idx < ptr->chain->length; idx++) {
 495       enum glsl_base_type base_type = glsl_get_base_type(type->type);
 496       switch (base_type) {
 497       case GLSL_TYPE_UINT:
 498       case GLSL_TYPE_INT:
 499       case GLSL_TYPE_UINT64:
 500       case GLSL_TYPE_INT64:
 501       case GLSL_TYPE_FLOAT:
 502       case GLSL_TYPE_DOUBLE:
 503       case GLSL_TYPE_BOOL:
 504       case GLSL_TYPE_ARRAY:
 505          offset = nir_iadd(&b->nb, offset,
 506                            vtn_access_link_as_ssa(b, ptr->chain->link[idx],
 507                                                   type->stride));
 508
 509          type = type->array_element;
 510          break;
 511
 512       case GLSL_TYPE_STRUCT: {
 513          assert(ptr->chain->link[idx].mode == vtn_access_mode_literal);
 514          unsigned member = ptr->chain->link[idx].id;
 515          offset = nir_iadd(&b->nb, offset,
 516                            nir_imm_int(&b->nb, type->offsets[member]));
 517          type = type->members[member];
 518          break;
 519       }
 520
 521       default:
 522          unreachable("Invalid type for deref");
 523       }
 524    }
 525
 526    assert(type == ptr->type);
 527    if (end_idx_out)
 528       *end_idx_out = idx;
 529
 530    return offset;
 531 }
 532
 533 /* Tries to compute the size of an interface block based on the strides and
 534  * offsets that are provided to us in the SPIR-V source.
 535  */
 536 static unsigned
 537 vtn_type_block_size(struct vtn_type *type)
 538 {
 539    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 540    switch (base_type) {
 541    case GLSL_TYPE_UINT:
 542    case GLSL_TYPE_INT:
 543    case GLSL_TYPE_UINT64:
 544    case GLSL_TYPE_INT64:
 545    case GLSL_TYPE_FLOAT:
 546    case GLSL_TYPE_BOOL:
 547    case GLSL_TYPE_DOUBLE: {
 548       unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
 549                                         glsl_get_matrix_columns(type->type);
 550       if (cols > 1) {
 551          assert(type->stride > 0);
 552          return type->stride * cols;
 553       } else if (base_type == GLSL_TYPE_DOUBLE ||
 554                  base_type == GLSL_TYPE_UINT64 ||
 555                  base_type == GLSL_TYPE_INT64) {
 556          return glsl_get_vector_elements(type->type) * 8;
 557       } else {
 558          return glsl_get_vector_elements(type->type) * 4;
 559       }
 560    }
 561
 562    case GLSL_TYPE_STRUCT:
 563    case GLSL_TYPE_INTERFACE: {
 564       unsigned size = 0;
 565       unsigned num_fields = glsl_get_length(type->type);
 566       for (unsigned f = 0; f < num_fields; f++) {
 567          unsigned field_end = type->offsets[f] +
 568                               vtn_type_block_size(type->members[f]);
 569          size = MAX2(size, field_end);
 570       }
 571       return size;
 572    }
 573
 574    case GLSL_TYPE_ARRAY:
 575       assert(type->stride > 0);
 576       assert(glsl_get_length(type->type) > 0);
 577       return type->stride * glsl_get_length(type->type);
 578
 579    default:
 580       assert(!"Invalid block type");
 581       return 0;
 582    }
 583 }
 584
 585 static void
 586 vtn_access_chain_get_offset_size(struct vtn_access_chain *chain,
 587                                  struct vtn_type *type,
 588                                  unsigned *access_offset,
 589                                  unsigned *access_size)
 590 {
 591    *access_offset = 0;
 592
 593    for (unsigned i = 0; i < chain->length; i++) {
 594       if (chain->link[i].mode != vtn_access_mode_literal)
 595          break;
 596
 597       if (glsl_type_is_struct(type->type)) {
 598          *access_offset += type->offsets[chain->link[i].id];
 599          type = type->members[chain->link[i].id];
 600       } else {
 601          *access_offset += type->stride * chain->link[i].id;
 602          type = type->array_element;
 603       }
 604    }
 605
 606    *access_size = vtn_type_block_size(type);
 607 }
 608
 609 static void
 610 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 611                      nir_ssa_def *index, nir_ssa_def *offset,
 612                      unsigned access_offset, unsigned access_size,
 613                      struct vtn_ssa_value **inout, const struct glsl_type *type)
 614 {
 615    nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
 616    instr->num_components = glsl_get_vector_elements(type);
 617
 618    int src = 0;
 619    if (!load) {
 620       nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
 621       instr->src[src++] = nir_src_for_ssa((*inout)->def);
 622    }
 623
 624    if (op == nir_intrinsic_load_push_constant) {
 625       assert(access_offset % 4 == 0);
 626
 627       nir_intrinsic_set_base(instr, access_offset);
 628       nir_intrinsic_set_range(instr, access_size);
 629    }
 630
 631    if (index)
 632       instr->src[src++] = nir_src_for_ssa(index);
 633
 634    if (op == nir_intrinsic_load_push_constant) {
 635       /* We need to subtract the offset from where the intrinsic will load the
 636        * data. */
 637       instr->src[src++] =
 638          nir_src_for_ssa(nir_isub(&b->nb, offset,
 639                                   nir_imm_int(&b->nb, access_offset)));
 640    } else {
 641       instr->src[src++] = nir_src_for_ssa(offset);
 642    }
 643
 644    if (load) {
 645       nir_ssa_dest_init(&instr->instr, &instr->dest,
 646                         instr->num_components,
 647                         glsl_get_bit_size(type), NULL);
 648       (*inout)->def = &instr->dest.ssa;
 649    }
 650
 651    nir_builder_instr_insert(&b->nb, &instr->instr);
 652
 653    if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
 654       (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
 655 }
 656
 657 static void
 658 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
 659                       nir_ssa_def *index, nir_ssa_def *offset,
 660                       unsigned access_offset, unsigned access_size,
 661                       struct vtn_access_chain *chain, unsigned chain_idx,
 662                       struct vtn_type *type, struct vtn_ssa_value **inout)
 663 {
 664    if (chain && chain_idx >= chain->length)
 665       chain = NULL;
 666
 667    if (load && chain == NULL && *inout == NULL)
 668       *inout = vtn_create_ssa_value(b, type->type);
 669
 670    enum glsl_base_type base_type = glsl_get_base_type(type->type);
 671    switch (base_type) {
 672    case GLSL_TYPE_UINT:
 673    case GLSL_TYPE_INT:
 674    case GLSL_TYPE_UINT64:
 675    case GLSL_TYPE_INT64:
 676    case GLSL_TYPE_FLOAT:
 677    case GLSL_TYPE_DOUBLE:
 678    case GLSL_TYPE_BOOL:
 679       /* This is where things get interesting.  At this point, we've hit
 680        * a vector, a scalar, or a matrix.
 681        */
 682       if (glsl_type_is_matrix(type->type)) {
 683          /* Loading the whole matrix */
 684          struct vtn_ssa_value *transpose;
 685          unsigned num_ops, vec_width, col_stride;
 686          if (type->row_major) {
 687             num_ops = glsl_get_vector_elements(type->type);
 688             vec_width = glsl_get_matrix_columns(type->type);
 689             col_stride = type->array_element->stride;
 690             if (load) {
 691                const struct glsl_type *transpose_type =
 692                   glsl_matrix_type(base_type, vec_width, num_ops);
 693                *inout = vtn_create_ssa_value(b, transpose_type);
 694             } else {
 695                transpose = vtn_ssa_transpose(b, *inout);
 696                inout = &transpose;
 697             }
 698          } else {
 699             num_ops = glsl_get_matrix_columns(type->type);
 700             vec_width = glsl_get_vector_elements(type->type);
 701             col_stride = type->stride;
 702          }
 703
 704          for (unsigned i = 0; i < num_ops; i++) {
 705             nir_ssa_def *elem_offset =
 706                nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * col_stride));
 707             _vtn_load_store_tail(b, op, load, index, elem_offset,
 708                                  access_offset, access_size,
 709                                  &(*inout)->elems[i],
 710                                  glsl_vector_type(base_type, vec_width));
 711          }
 712
 713          if (load && type->row_major)
 714             *inout = vtn_ssa_transpose(b, *inout);
 715       } else {
 716          unsigned elems = glsl_get_vector_elements(type->type);
 717          unsigned type_size = glsl_get_bit_size(type->type) / 8;
 718          if (elems == 1 || type->stride == type_size) {
 719             /* This is a tightly-packed normal scalar or vector load */
 720             assert(glsl_type_is_vector_or_scalar(type->type));
 721             _vtn_load_store_tail(b, op, load, index, offset,
 722                                  access_offset, access_size,
 723                                  inout, type->type);
 724          } else {
 725             /* This is a strided load.  We have to load N things separately.
 726              * This is the single column of a row-major matrix case.
 727              */
 728             assert(type->stride > type_size);
 729             assert(type->stride % type_size == 0);
 730
 731             nir_ssa_def *per_comp[4];
 732             for (unsigned i = 0; i < elems; i++) {
 733                nir_ssa_def *elem_offset =
 734                   nir_iadd(&b->nb, offset,
 735                                    nir_imm_int(&b->nb, i * type->stride));
 736                struct vtn_ssa_value *comp, temp_val;
 737                if (!load) {
 738                   temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
 739                   temp_val.type = glsl_scalar_type(base_type);
 740                }
 741                comp = &temp_val;
 742                _vtn_load_store_tail(b, op, load, index, elem_offset,
 743                                     access_offset, access_size,
 744                                     &comp, glsl_scalar_type(base_type));
 745                per_comp[i] = comp->def;
 746             }
 747
 748             if (load) {
 749                if (*inout == NULL)
 750                   *inout = vtn_create_ssa_value(b, type->type);
 751                (*inout)->def = nir_vec(&b->nb, per_comp, elems);
 752             }
 753          }
 754       }
 755       return;
 756
 757    case GLSL_TYPE_ARRAY: {
 758       unsigned elems = glsl_get_length(type->type);
 759       for (unsigned i = 0; i < elems; i++) {
 760          nir_ssa_def *elem_off =
 761             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
 762          _vtn_block_load_store(b, op, load, index, elem_off,
 763                                access_offset, access_size,
 764                                NULL, 0,
 765                                type->array_element, &(*inout)->elems[i]);
 766       }
 767       return;
 768    }
 769
 770    case GLSL_TYPE_STRUCT: {
 771       unsigned elems = glsl_get_length(type->type);
 772       for (unsigned i = 0; i < elems; i++) {
 773          nir_ssa_def *elem_off =
 774             nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
 775          _vtn_block_load_store(b, op, load, index, elem_off,
 776                                access_offset, access_size,
 777                                NULL, 0,
 778                                type->members[i], &(*inout)->elems[i]);
 779       }
 780       return;
 781    }
 782
 783    default:
 784       unreachable("Invalid block member type");
 785    }
 786 }
 787
 788 static struct vtn_ssa_value *
 789 vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src)
 790 {
 791    nir_intrinsic_op op;
 792    unsigned access_offset = 0, access_size = 0;
 793    switch (src->mode) {
 794    case vtn_variable_mode_ubo:
 795       op = nir_intrinsic_load_ubo;
 796       break;
 797    case vtn_variable_mode_ssbo:
 798       op = nir_intrinsic_load_ssbo;
 799       break;
 800    case vtn_variable_mode_push_constant:
 801       op = nir_intrinsic_load_push_constant;
 802       vtn_access_chain_get_offset_size(src->chain, src->var->type,
 803                                        &access_offset, &access_size);
 804       break;
 805    default:
 806       assert(!"Invalid block variable mode");
 807    }
 808
 809    nir_ssa_def *offset, *index = NULL;
 810    unsigned chain_idx;
 811    offset = vtn_pointer_to_offset(b, src, &index, &chain_idx);
 812
 813    struct vtn_ssa_value *value = NULL;
 814    _vtn_block_load_store(b, op, true, index, offset,
 815                          access_offset, access_size,
 816                          src->chain, chain_idx, src->type, &value);
 817    return value;
 818 }
 819
 820 static void
 821 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 822                 struct vtn_pointer *dst)
 823 {
 824    nir_ssa_def *offset, *index = NULL;
 825    unsigned chain_idx;
 826    offset = vtn_pointer_to_offset(b, dst, &index, &chain_idx);
 827
 828    _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset,
 829                          0, 0, dst->chain, chain_idx, dst->type, &src);
 830 }
 831
 832 static bool
 833 vtn_pointer_is_external_block(struct vtn_pointer *ptr)
 834 {
 835    return ptr->mode == vtn_variable_mode_ssbo ||
 836           ptr->mode == vtn_variable_mode_ubo ||
 837           ptr->mode == vtn_variable_mode_push_constant;
 838 }
 839
 840 static void
 841 _vtn_variable_load_store(struct vtn_builder *b, bool load,
 842                          struct vtn_pointer *ptr,
 843                          struct vtn_ssa_value **inout)
 844 {
 845    enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type);
 846    switch (base_type) {
 847    case GLSL_TYPE_UINT:
 848    case GLSL_TYPE_INT:
 849    case GLSL_TYPE_UINT64:
 850    case GLSL_TYPE_INT64:
 851    case GLSL_TYPE_FLOAT:
 852    case GLSL_TYPE_BOOL:
 853    case GLSL_TYPE_DOUBLE:
 854       /* At this point, we have a scalar, vector, or matrix so we know that
 855        * there cannot be any structure splitting still in the way.  By
 856        * stopping at the matrix level rather than the vector level, we
 857        * ensure that matrices get loaded in the optimal way even if they
 858        * are storred row-major in a UBO.
 859        */
 860       if (load) {
 861          *inout = vtn_local_load(b, vtn_pointer_to_deref(b, ptr));
 862       } else {
 863          vtn_local_store(b, *inout, vtn_pointer_to_deref(b, ptr));
 864       }
 865       return;
 866
 867    case GLSL_TYPE_ARRAY:
 868    case GLSL_TYPE_STRUCT: {
 869       unsigned elems = glsl_get_length(ptr->type->type);
 870       if (load) {
 871          assert(*inout == NULL);
 872          *inout = rzalloc(b, struct vtn_ssa_value);
 873          (*inout)->type = ptr->type->type;
 874          (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
 875       }
 876
 877       struct vtn_access_chain chain = {
 878          .length = 1,
 879          .link = {
 880             { .mode = vtn_access_mode_literal, },
 881          }
 882       };
 883       for (unsigned i = 0; i < elems; i++) {
 884          chain.link[0].id = i;
 885          struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain);
 886          _vtn_variable_load_store(b, load, elem, &(*inout)->elems[i]);
 887       }
 888       return;
 889    }
 890
 891    default:
 892       unreachable("Invalid access chain type");
 893    }
 894 }
 895
 896 struct vtn_ssa_value *
 897 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src)
 898 {
 899    if (vtn_pointer_is_external_block(src)) {
 900       return vtn_block_load(b, src);
 901    } else {
 902       struct vtn_ssa_value *val = NULL;
 903       _vtn_variable_load_store(b, true, src, &val);
 904       return val;
 905    }
 906 }
 907
 908 void
 909 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
 910                    struct vtn_pointer *dest)
 911 {
 912    if (vtn_pointer_is_external_block(dest)) {
 913       assert(dest->mode == vtn_variable_mode_ssbo);
 914       vtn_block_store(b, src, dest);
 915    } else {
 916       _vtn_variable_load_store(b, false, dest, &src);
 917    }
 918 }
 919
 920 static void
 921 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
 922                    struct vtn_pointer *src)
 923 {
 924    assert(src->type->type == dest->type->type);
 925    enum glsl_base_type base_type = glsl_get_base_type(src->type->type);
 926    switch (base_type) {
 927    case GLSL_TYPE_UINT:
 928    case GLSL_TYPE_INT:
 929    case GLSL_TYPE_UINT64:
 930    case GLSL_TYPE_INT64:
 931    case GLSL_TYPE_FLOAT:
 932    case GLSL_TYPE_DOUBLE:
 933    case GLSL_TYPE_BOOL:
 934       /* At this point, we have a scalar, vector, or matrix so we know that
 935        * there cannot be any structure splitting still in the way.  By
 936        * stopping at the matrix level rather than the vector level, we
 937        * ensure that matrices get loaded in the optimal way even if they
 938        * are storred row-major in a UBO.
 939        */
 940       vtn_variable_store(b, vtn_variable_load(b, src), dest);
 941       return;
 942
 943    case GLSL_TYPE_ARRAY:
 944    case GLSL_TYPE_STRUCT: {
 945       struct vtn_access_chain chain = {
 946          .length = 1,
 947          .link = {
 948             { .mode = vtn_access_mode_literal, },
 949          }
 950       };
 951       unsigned elems = glsl_get_length(src->type->type);
 952       for (unsigned i = 0; i < elems; i++) {
 953          chain.link[0].id = i;
 954          struct vtn_pointer *src_elem =
 955             vtn_pointer_dereference(b, src, &chain);
 956          struct vtn_pointer *dest_elem =
 957             vtn_pointer_dereference(b, dest, &chain);
 958
 959          _vtn_variable_copy(b, dest_elem, src_elem);
 960       }
 961       return;
 962    }
 963
 964    default:
 965       unreachable("Invalid access chain type");
 966    }
 967 }
 968
 969 static void
 970 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
 971                   struct vtn_pointer *src)
 972 {
 973    /* TODO: At some point, we should add a special-case for when we can
 974     * just emit a copy_var intrinsic.
 975     */
 976    _vtn_variable_copy(b, dest, src);
 977 }
 978
 979 static void
 980 set_mode_system_value(nir_variable_mode *mode)
 981 {
 982    assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
 983    *mode = nir_var_system_value;
 984 }
 985
 986 static void
 987 vtn_get_builtin_location(struct vtn_builder *b,
 988                          SpvBuiltIn builtin, int *location,
 989                          nir_variable_mode *mode)
 990 {
 991    switch (builtin) {
 992    case SpvBuiltInPosition:
 993       *location = VARYING_SLOT_POS;
 994       break;
 995    case SpvBuiltInPointSize:
 996       *location = VARYING_SLOT_PSIZ;
 997       break;
 998    case SpvBuiltInClipDistance:
 999       *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
1000       break;
1001    case SpvBuiltInCullDistance:
1002       *location = VARYING_SLOT_CULL_DIST0;
1003       break;
1004    case SpvBuiltInVertexIndex:
1005       *location = SYSTEM_VALUE_VERTEX_ID;
1006       set_mode_system_value(mode);
1007       break;
1008    case SpvBuiltInVertexId:
1009       /* Vulkan defines VertexID to be zero-based and reserves the new
1010        * builtin keyword VertexIndex to indicate the non-zero-based value.
1011        */
1012       *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
1013       set_mode_system_value(mode);
1014       break;
1015    case SpvBuiltInInstanceIndex:
1016       *location = SYSTEM_VALUE_INSTANCE_INDEX;
1017       set_mode_system_value(mode);
1018       break;
1019    case SpvBuiltInInstanceId:
1020       *location = SYSTEM_VALUE_INSTANCE_ID;
1021       set_mode_system_value(mode);
1022       break;
1023    case SpvBuiltInPrimitiveId:
1024       if (b->shader->stage == MESA_SHADER_FRAGMENT) {
1025          assert(*mode == nir_var_shader_in);
1026          *location = VARYING_SLOT_PRIMITIVE_ID;
1027       } else if (*mode == nir_var_shader_out) {
1028          *location = VARYING_SLOT_PRIMITIVE_ID;
1029       } else {
1030          *location = SYSTEM_VALUE_PRIMITIVE_ID;
1031          set_mode_system_value(mode);
1032       }
1033       break;
1034    case SpvBuiltInInvocationId:
1035       *location = SYSTEM_VALUE_INVOCATION_ID;
1036       set_mode_system_value(mode);
1037       break;
1038    case SpvBuiltInLayer:
1039       *location = VARYING_SLOT_LAYER;
1040       if (b->shader->stage == MESA_SHADER_FRAGMENT)
1041          *mode = nir_var_shader_in;
1042       else if (b->shader->stage == MESA_SHADER_GEOMETRY)
1043          *mode = nir_var_shader_out;
1044       else
1045          unreachable("invalid stage for SpvBuiltInLayer");
1046       break;
1047    case SpvBuiltInViewportIndex:
1048       *location = VARYING_SLOT_VIEWPORT;
1049       if (b->shader->stage == MESA_SHADER_GEOMETRY)
1050          *mode = nir_var_shader_out;
1051       else if (b->shader->stage == MESA_SHADER_FRAGMENT)
1052          *mode = nir_var_shader_in;
1053       else
1054          unreachable("invalid stage for SpvBuiltInViewportIndex");
1055       break;
1056    case SpvBuiltInTessLevelOuter:
1057       *location = VARYING_SLOT_TESS_LEVEL_OUTER;
1058       break;
1059    case SpvBuiltInTessLevelInner:
1060       *location = VARYING_SLOT_TESS_LEVEL_INNER;
1061       break;
1062    case SpvBuiltInTessCoord:
1063       *location = SYSTEM_VALUE_TESS_COORD;
1064       set_mode_system_value(mode);
1065       break;
1066    case SpvBuiltInPatchVertices:
1067       *location = SYSTEM_VALUE_VERTICES_IN;
1068       set_mode_system_value(mode);
1069       break;
1070    case SpvBuiltInFragCoord:
1071       *location = VARYING_SLOT_POS;
1072       assert(*mode == nir_var_shader_in);
1073       break;
1074    case SpvBuiltInPointCoord:
1075       *location = VARYING_SLOT_PNTC;
1076       assert(*mode == nir_var_shader_in);
1077       break;
1078    case SpvBuiltInFrontFacing:
1079       *location = SYSTEM_VALUE_FRONT_FACE;
1080       set_mode_system_value(mode);
1081       break;
1082    case SpvBuiltInSampleId:
1083       *location = SYSTEM_VALUE_SAMPLE_ID;
1084       set_mode_system_value(mode);
1085       break;
1086    case SpvBuiltInSamplePosition:
1087       *location = SYSTEM_VALUE_SAMPLE_POS;
1088       set_mode_system_value(mode);
1089       break;
1090    case SpvBuiltInSampleMask:
1091       if (*mode == nir_var_shader_out) {
1092          *location = FRAG_RESULT_SAMPLE_MASK;
1093       } else {
1094          *location = SYSTEM_VALUE_SAMPLE_MASK_IN;
1095          set_mode_system_value(mode);
1096       }
1097       break;
1098    case SpvBuiltInFragDepth:
1099       *location = FRAG_RESULT_DEPTH;
1100       assert(*mode == nir_var_shader_out);
1101       break;
1102    case SpvBuiltInNumWorkgroups:
1103       *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
1104       set_mode_system_value(mode);
1105       break;
1106    case SpvBuiltInWorkgroupSize:
1107       /* This should already be handled */
1108       unreachable("unsupported builtin");
1109       break;
1110    case SpvBuiltInWorkgroupId:
1111       *location = SYSTEM_VALUE_WORK_GROUP_ID;
1112       set_mode_system_value(mode);
1113       break;
1114    case SpvBuiltInLocalInvocationId:
1115       *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
1116       set_mode_system_value(mode);
1117       break;
1118    case SpvBuiltInLocalInvocationIndex:
1119       *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1120       set_mode_system_value(mode);
1121       break;
1122    case SpvBuiltInGlobalInvocationId:
1123       *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1124       set_mode_system_value(mode);
1125       break;
1126    case SpvBuiltInBaseVertex:
1127       *location = SYSTEM_VALUE_BASE_VERTEX;
1128       set_mode_system_value(mode);
1129       break;
1130    case SpvBuiltInBaseInstance:
1131       *location = SYSTEM_VALUE_BASE_INSTANCE;
1132       set_mode_system_value(mode);
1133       break;
1134    case SpvBuiltInDrawIndex:
1135       *location = SYSTEM_VALUE_DRAW_ID;
1136       set_mode_system_value(mode);
1137       break;
1138    case SpvBuiltInViewIndex:
1139       *location = SYSTEM_VALUE_VIEW_INDEX;
1140       set_mode_system_value(mode);
1141       break;
1142    case SpvBuiltInHelperInvocation:
1143    default:
1144       unreachable("unsupported builtin");
1145    }
1146 }
1147
1148 static void
1149 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var,
1150                      const struct vtn_decoration *dec)
1151 {
1152    switch (dec->decoration) {
1153    case SpvDecorationRelaxedPrecision:
1154       break; /* FIXME: Do nothing with this for now. */
1155    case SpvDecorationNoPerspective:
1156       nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1157       break;
1158    case SpvDecorationFlat:
1159       nir_var->data.interpolation = INTERP_MODE_FLAT;
1160       break;
1161    case SpvDecorationCentroid:
1162       nir_var->data.centroid = true;
1163       break;
1164    case SpvDecorationSample:
1165       nir_var->data.sample = true;
1166       break;
1167    case SpvDecorationInvariant:
1168       nir_var->data.invariant = true;
1169       break;
1170    case SpvDecorationConstant:
1171       assert(nir_var->constant_initializer != NULL);
1172       nir_var->data.read_only = true;
1173       break;
1174    case SpvDecorationNonReadable:
1175       nir_var->data.image.write_only = true;
1176       break;
1177    case SpvDecorationNonWritable:
1178       nir_var->data.read_only = true;
1179       nir_var->data.image.read_only = true;
1180       break;
1181    case SpvDecorationComponent:
1182       nir_var->data.location_frac = dec->literals[0];
1183       break;
1184    case SpvDecorationIndex:
1185       nir_var->data.index = dec->literals[0];
1186       break;
1187    case SpvDecorationBuiltIn: {
1188       SpvBuiltIn builtin = dec->literals[0];
1189
1190       if (builtin == SpvBuiltInWorkgroupSize) {
1191          /* This shouldn't be a builtin.  It's actually a constant. */
1192          nir_var->data.mode = nir_var_global;
1193          nir_var->data.read_only = true;
1194
1195          nir_constant *c = rzalloc(nir_var, nir_constant);
1196          c->values[0].u32[0] = b->shader->info.cs.local_size[0];
1197          c->values[0].u32[1] = b->shader->info.cs.local_size[1];
1198          c->values[0].u32[2] = b->shader->info.cs.local_size[2];
1199          nir_var->constant_initializer = c;
1200          break;
1201       }
1202
1203       nir_variable_mode mode = nir_var->data.mode;
1204       vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode);
1205       nir_var->data.mode = mode;
1206
1207       switch (builtin) {
1208       case SpvBuiltInTessLevelOuter:
1209       case SpvBuiltInTessLevelInner:
1210          nir_var->data.compact = true;
1211          break;
1212       case SpvBuiltInSamplePosition:
1213          nir_var->data.origin_upper_left = b->origin_upper_left;
1214          /* fallthrough */
1215       case SpvBuiltInFragCoord:
1216          nir_var->data.pixel_center_integer = b->pixel_center_integer;
1217          break;
1218       default:
1219          break;
1220       }
1221    }
1222
1223    case SpvDecorationSpecId:
1224    case SpvDecorationRowMajor:
1225    case SpvDecorationColMajor:
1226    case SpvDecorationMatrixStride:
1227    case SpvDecorationRestrict:
1228    case SpvDecorationAliased:
1229    case SpvDecorationVolatile:
1230    case SpvDecorationCoherent:
1231    case SpvDecorationUniform:
1232    case SpvDecorationStream:
1233    case SpvDecorationOffset:
1234    case SpvDecorationLinkageAttributes:
1235       break; /* Do nothing with these here */
1236
1237    case SpvDecorationPatch:
1238       nir_var->data.patch = true;
1239       break;
1240
1241    case SpvDecorationLocation:
1242       unreachable("Handled above");
1243
1244    case SpvDecorationBlock:
1245    case SpvDecorationBufferBlock:
1246    case SpvDecorationArrayStride:
1247    case SpvDecorationGLSLShared:
1248    case SpvDecorationGLSLPacked:
1249       break; /* These can apply to a type but we don't care about them */
1250
1251    case SpvDecorationBinding:
1252    case SpvDecorationDescriptorSet:
1253    case SpvDecorationNoContraction:
1254    case SpvDecorationInputAttachmentIndex:
1255       vtn_warn("Decoration not allowed for variable or structure member: %s",
1256                spirv_decoration_to_string(dec->decoration));
1257       break;
1258
1259    case SpvDecorationXfbBuffer:
1260    case SpvDecorationXfbStride:
1261       vtn_warn("Vulkan does not have transform feedback: %s",
1262                spirv_decoration_to_string(dec->decoration));
1263       break;
1264
1265    case SpvDecorationCPacked:
1266    case SpvDecorationSaturatedConversion:
1267    case SpvDecorationFuncParamAttr:
1268    case SpvDecorationFPRoundingMode:
1269    case SpvDecorationFPFastMathMode:
1270    case SpvDecorationAlignment:
1271       vtn_warn("Decoration only allowed for CL-style kernels: %s",
1272                spirv_decoration_to_string(dec->decoration));
1273       break;
1274
1275    default:
1276       unreachable("Unhandled decoration");
1277    }
1278 }
1279
1280 static void
1281 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1282                 const struct vtn_decoration *dec, void *out_is_patch)
1283 {
1284    if (dec->decoration == SpvDecorationPatch) {
1285       *((bool *) out_is_patch) = true;
1286    }
1287 }
1288
1289 static void
1290 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1291                   const struct vtn_decoration *dec, void *void_var)
1292 {
1293    struct vtn_variable *vtn_var = void_var;
1294
1295    /* Handle decorations that apply to a vtn_variable as a whole */
1296    switch (dec->decoration) {
1297    case SpvDecorationBinding:
1298       vtn_var->binding = dec->literals[0];
1299       return;
1300    case SpvDecorationDescriptorSet:
1301       vtn_var->descriptor_set = dec->literals[0];
1302       return;
1303    case SpvDecorationInputAttachmentIndex:
1304       vtn_var->input_attachment_index = dec->literals[0];
1305       return;
1306    case SpvDecorationPatch:
1307       vtn_var->patch = true;
1308       break;
1309    default:
1310       break;
1311    }
1312
1313    if (val->value_type == vtn_value_type_pointer) {
1314       assert(val->pointer->var == void_var);
1315       assert(val->pointer->chain == NULL);
1316       assert(member == -1);
1317    } else {
1318       assert(val->value_type == vtn_value_type_type);
1319    }
1320
1321    /* Location is odd.  If applied to a split structure, we have to walk the
1322     * whole thing and accumulate the location.  It's easier to handle as a
1323     * special case.
1324     */
1325    if (dec->decoration == SpvDecorationLocation) {
1326       unsigned location = dec->literals[0];
1327       bool is_vertex_input;
1328       if (b->shader->stage == MESA_SHADER_FRAGMENT &&
1329           vtn_var->mode == vtn_variable_mode_output) {
1330          is_vertex_input = false;
1331          location += FRAG_RESULT_DATA0;
1332       } else if (b->shader->stage == MESA_SHADER_VERTEX &&
1333                  vtn_var->mode == vtn_variable_mode_input) {
1334          is_vertex_input = true;
1335          location += VERT_ATTRIB_GENERIC0;
1336       } else if (vtn_var->mode == vtn_variable_mode_input ||
1337                  vtn_var->mode == vtn_variable_mode_output) {
1338          is_vertex_input = false;
1339          location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1340       } else {
1341          vtn_warn("Location must be on input or output variable");
1342          return;
1343       }
1344
1345       if (vtn_var->var) {
1346          /* This handles the member and lone variable cases */
1347          vtn_var->var->data.location = location;
1348       } else {
1349          /* This handles the structure member case */
1350          assert(vtn_var->members);
1351          unsigned length =
1352             glsl_get_length(glsl_without_array(vtn_var->type->type));
1353          for (unsigned i = 0; i < length; i++) {
1354             vtn_var->members[i]->data.location = location;
1355             location +=
1356                glsl_count_attribute_slots(vtn_var->members[i]->interface_type,
1357                                           is_vertex_input);
1358          }
1359       }
1360       return;
1361    } else {
1362       if (vtn_var->var) {
1363          assert(member <= 0);
1364          apply_var_decoration(b, vtn_var->var, dec);
1365       } else if (vtn_var->members) {
1366          if (member >= 0) {
1367             assert(vtn_var->members);
1368             apply_var_decoration(b, vtn_var->members[member], dec);
1369          } else {
1370             unsigned length =
1371                glsl_get_length(glsl_without_array(vtn_var->type->type));
1372             for (unsigned i = 0; i < length; i++)
1373                apply_var_decoration(b, vtn_var->members[i], dec);
1374          }
1375       } else {
1376          /* A few variables, those with external storage, have no actual
1377           * nir_variables associated with them.  Fortunately, all decorations
1378           * we care about for those variables are on the type only.
1379           */
1380          assert(vtn_var->mode == vtn_variable_mode_ubo ||
1381                 vtn_var->mode == vtn_variable_mode_ssbo ||
1382                 vtn_var->mode == vtn_variable_mode_push_constant);
1383       }
1384    }
1385 }
1386
1387 static enum vtn_variable_mode
1388 vtn_storage_class_to_mode(SpvStorageClass class,
1389                           struct vtn_type *interface_type,
1390                           nir_variable_mode *nir_mode_out)
1391 {
1392    enum vtn_variable_mode mode;
1393    nir_variable_mode nir_mode;
1394    switch (class) {
1395    case SpvStorageClassUniform:
1396       if (interface_type->block) {
1397          mode = vtn_variable_mode_ubo;
1398          nir_mode = 0;
1399       } else if (interface_type->buffer_block) {
1400          mode = vtn_variable_mode_ssbo;
1401          nir_mode = 0;
1402       } else {
1403          assert(!"Invalid uniform variable type");
1404       }
1405       break;
1406    case SpvStorageClassUniformConstant:
1407       if (glsl_type_is_image(interface_type->type)) {
1408          mode = vtn_variable_mode_image;
1409          nir_mode = nir_var_uniform;
1410       } else if (glsl_type_is_sampler(interface_type->type)) {
1411          mode = vtn_variable_mode_sampler;
1412          nir_mode = nir_var_uniform;
1413       } else {
1414          assert(!"Invalid uniform constant variable type");
1415       }
1416       break;
1417    case SpvStorageClassPushConstant:
1418       mode = vtn_variable_mode_push_constant;
1419       nir_mode = nir_var_uniform;
1420       break;
1421    case SpvStorageClassInput:
1422       mode = vtn_variable_mode_input;
1423       nir_mode = nir_var_shader_in;
1424       break;
1425    case SpvStorageClassOutput:
1426       mode = vtn_variable_mode_output;
1427       nir_mode = nir_var_shader_out;
1428       break;
1429    case SpvStorageClassPrivate:
1430       mode = vtn_variable_mode_global;
1431       nir_mode = nir_var_global;
1432       break;
1433    case SpvStorageClassFunction:
1434       mode = vtn_variable_mode_local;
1435       nir_mode = nir_var_local;
1436       break;
1437    case SpvStorageClassWorkgroup:
1438       mode = vtn_variable_mode_workgroup;
1439       nir_mode = nir_var_shared;
1440       break;
1441    case SpvStorageClassCrossWorkgroup:
1442    case SpvStorageClassGeneric:
1443    case SpvStorageClassAtomicCounter:
1444    default:
1445       unreachable("Unhandled variable storage class");
1446    }
1447
1448    if (nir_mode_out)
1449       *nir_mode_out = nir_mode;
1450
1451    return mode;
1452 }
1453
1454 static bool
1455 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
1456 {
1457    if (var->patch || !glsl_type_is_array(var->type->type))
1458       return false;
1459
1460    if (var->mode == vtn_variable_mode_input) {
1461       return stage == MESA_SHADER_TESS_CTRL ||
1462              stage == MESA_SHADER_TESS_EVAL ||
1463              stage == MESA_SHADER_GEOMETRY;
1464    }
1465
1466    if (var->mode == vtn_variable_mode_output)
1467       return stage == MESA_SHADER_TESS_CTRL;
1468
1469    return false;
1470 }
1471
1472 static void
1473 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val,
1474                     struct vtn_type *ptr_type, SpvStorageClass storage_class,
1475                     nir_constant *initializer)
1476 {
1477    assert(ptr_type->base_type == vtn_base_type_pointer);
1478    struct vtn_type *type = ptr_type->deref;
1479    assert(type->base_type != vtn_base_type_pointer);
1480
1481    struct vtn_type *without_array = type;
1482    while(glsl_type_is_array(without_array->type))
1483       without_array = without_array->array_element;
1484
1485    enum vtn_variable_mode mode;
1486    nir_variable_mode nir_mode;
1487    mode = vtn_storage_class_to_mode(storage_class, without_array, &nir_mode);
1488
1489    switch (mode) {
1490    case vtn_variable_mode_ubo:
1491       b->shader->info.num_ubos++;
1492       break;
1493    case vtn_variable_mode_ssbo:
1494       b->shader->info.num_ssbos++;
1495       break;
1496    case vtn_variable_mode_image:
1497       b->shader->info.num_images++;
1498       break;
1499    case vtn_variable_mode_sampler:
1500       b->shader->info.num_textures++;
1501       break;
1502    case vtn_variable_mode_push_constant:
1503       b->shader->num_uniforms = vtn_type_block_size(type);
1504       break;
1505    default:
1506       /* No tallying is needed */
1507       break;
1508    }
1509
1510    struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1511    var->type = type;
1512    var->mode = mode;
1513
1514    assert(val->value_type == vtn_value_type_pointer);
1515    val->pointer = vtn_pointer_for_variable(b, var, ptr_type);
1516
1517    switch (var->mode) {
1518    case vtn_variable_mode_local:
1519    case vtn_variable_mode_global:
1520    case vtn_variable_mode_image:
1521    case vtn_variable_mode_sampler:
1522    case vtn_variable_mode_workgroup:
1523       /* For these, we create the variable normally */
1524       var->var = rzalloc(b->shader, nir_variable);
1525       var->var->name = ralloc_strdup(var->var, val->name);
1526       var->var->type = var->type->type;
1527       var->var->data.mode = nir_mode;
1528
1529       switch (var->mode) {
1530       case vtn_variable_mode_image:
1531       case vtn_variable_mode_sampler:
1532          var->var->interface_type = without_array->type;
1533          break;
1534       default:
1535          var->var->interface_type = NULL;
1536          break;
1537       }
1538       break;
1539
1540    case vtn_variable_mode_input:
1541    case vtn_variable_mode_output: {
1542       /* In order to know whether or not we're a per-vertex inout, we need
1543        * the patch qualifier.  This means walking the variable decorations
1544        * early before we actually create any variables.  Not a big deal.
1545        *
1546        * GLSLang really likes to place decorations in the most interior
1547        * thing it possibly can.  In particular, if you have a struct, it
1548        * will place the patch decorations on the struct members.  This
1549        * should be handled by the variable splitting below just fine.
1550        *
1551        * If you have an array-of-struct, things get even more weird as it
1552        * will place the patch decorations on the struct even though it's
1553        * inside an array and some of the members being patch and others not
1554        * makes no sense whatsoever.  Since the only sensible thing is for
1555        * it to be all or nothing, we'll call it patch if any of the members
1556        * are declared patch.
1557        */
1558       var->patch = false;
1559       vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
1560       if (glsl_type_is_array(var->type->type) &&
1561           glsl_type_is_struct(without_array->type)) {
1562          vtn_foreach_decoration(b, without_array->val,
1563                                 var_is_patch_cb, &var->patch);
1564       }
1565
1566       /* For inputs and outputs, we immediately split structures.  This
1567        * is for a couple of reasons.  For one, builtins may all come in
1568        * a struct and we really want those split out into separate
1569        * variables.  For another, interpolation qualifiers can be
1570        * applied to members of the top-level struct ane we need to be
1571        * able to preserve that information.
1572        */
1573
1574       int array_length = -1;
1575       struct vtn_type *interface_type = var->type;
1576       if (is_per_vertex_inout(var, b->shader->stage)) {
1577          /* In Geometry shaders (and some tessellation), inputs come
1578           * in per-vertex arrays.  However, some builtins come in
1579           * non-per-vertex, hence the need for the is_array check.  In
1580           * any case, there are no non-builtin arrays allowed so this
1581           * check should be sufficient.
1582           */
1583          interface_type = var->type->array_element;
1584          array_length = glsl_get_length(var->type->type);
1585       }
1586
1587       if (glsl_type_is_struct(interface_type->type)) {
1588          /* It's a struct.  Split it. */
1589          unsigned num_members = glsl_get_length(interface_type->type);
1590          var->members = ralloc_array(b, nir_variable *, num_members);
1591
1592          for (unsigned i = 0; i < num_members; i++) {
1593             const struct glsl_type *mtype = interface_type->members[i]->type;
1594             if (array_length >= 0)
1595                mtype = glsl_array_type(mtype, array_length);
1596
1597             var->members[i] = rzalloc(b->shader, nir_variable);
1598             var->members[i]->name =
1599                ralloc_asprintf(var->members[i], "%s.%d", val->name, i);
1600             var->members[i]->type = mtype;
1601             var->members[i]->interface_type =
1602                interface_type->members[i]->type;
1603             var->members[i]->data.mode = nir_mode;
1604             var->members[i]->data.patch = var->patch;
1605          }
1606       } else {
1607          var->var = rzalloc(b->shader, nir_variable);
1608          var->var->name = ralloc_strdup(var->var, val->name);
1609          var->var->type = var->type->type;
1610          var->var->interface_type = interface_type->type;
1611          var->var->data.mode = nir_mode;
1612          var->var->data.patch = var->patch;
1613       }
1614
1615       /* For inputs and outputs, we need to grab locations and builtin
1616        * information from the interface type.
1617        */
1618       vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var);
1619       break;
1620    }
1621
1622    case vtn_variable_mode_param:
1623       unreachable("Not created through OpVariable");
1624
1625    case vtn_variable_mode_ubo:
1626    case vtn_variable_mode_ssbo:
1627    case vtn_variable_mode_push_constant:
1628       /* These don't need actual variables. */
1629       break;
1630    }
1631
1632    if (initializer) {
1633       var->var->constant_initializer =
1634          nir_constant_clone(initializer, var->var);
1635    }
1636
1637    vtn_foreach_decoration(b, val, var_decoration_cb, var);
1638
1639    if (var->mode == vtn_variable_mode_image ||
1640        var->mode == vtn_variable_mode_sampler) {
1641       /* XXX: We still need the binding information in the nir_variable
1642        * for these. We should fix that.
1643        */
1644       var->var->data.binding = var->binding;
1645       var->var->data.descriptor_set = var->descriptor_set;
1646       var->var->data.index = var->input_attachment_index;
1647
1648       if (var->mode == vtn_variable_mode_image)
1649          var->var->data.image.format = without_array->image_format;
1650    }
1651
1652    if (var->mode == vtn_variable_mode_local) {
1653       assert(var->members == NULL && var->var != NULL);
1654       nir_function_impl_add_variable(b->impl, var->var);
1655    } else if (var->var) {
1656       nir_shader_add_variable(b->shader, var->var);
1657    } else if (var->members) {
1658       unsigned count = glsl_get_length(without_array->type);
1659       for (unsigned i = 0; i < count; i++) {
1660          assert(var->members[i]->data.mode != nir_var_local);
1661          nir_shader_add_variable(b->shader, var->members[i]);
1662       }
1663    } else {
1664       assert(var->mode == vtn_variable_mode_ubo ||
1665              var->mode == vtn_variable_mode_ssbo ||
1666              var->mode == vtn_variable_mode_push_constant);
1667    }
1668 }
1669
1670 void
1671 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1672                      const uint32_t *w, unsigned count)
1673 {
1674    switch (opcode) {
1675    case SpvOpUndef: {
1676       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef);
1677       val->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1678       break;
1679    }
1680
1681    case SpvOpVariable: {
1682       struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type;
1683
1684       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer);
1685
1686       SpvStorageClass storage_class = w[3];
1687       nir_constant *initializer = NULL;
1688       if (count > 4)
1689          initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant;
1690
1691       vtn_create_variable(b, val, ptr_type, storage_class, initializer);
1692       break;
1693    }
1694
1695    case SpvOpAccessChain:
1696    case SpvOpInBoundsAccessChain: {
1697       struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4);
1698
1699       unsigned idx = 0;
1700       for (int i = 4; i < count; i++) {
1701          struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1702          if (link_val->value_type == vtn_value_type_constant) {
1703             chain->link[idx].mode = vtn_access_mode_literal;
1704             chain->link[idx].id = link_val->constant->values[0].u32[0];
1705          } else {
1706             chain->link[idx].mode = vtn_access_mode_id;
1707             chain->link[idx].id = w[i];
1708
1709          }
1710          idx++;
1711       }
1712
1713       struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type;
1714       struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1715       if (base_val->value_type == vtn_value_type_sampled_image) {
1716          /* This is rather insane.  SPIR-V allows you to use OpSampledImage
1717           * to combine an array of images with a single sampler to get an
1718           * array of sampled images that all share the same sampler.
1719           * Fortunately, this means that we can more-or-less ignore the
1720           * sampler when crawling the access chain, but it does leave us
1721           * with this rather awkward little special-case.
1722           */
1723          struct vtn_value *val =
1724             vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1725          val->sampled_image = ralloc(b, struct vtn_sampled_image);
1726          val->sampled_image->image =
1727             vtn_pointer_dereference(b, base_val->sampled_image->image, chain);
1728          val->sampled_image->sampler = base_val->sampled_image->sampler;
1729       } else {
1730          assert(base_val->value_type == vtn_value_type_pointer);
1731          struct vtn_value *val =
1732             vtn_push_value(b, w[2], vtn_value_type_pointer);
1733          val->pointer = vtn_pointer_dereference(b, base_val->pointer, chain);
1734          val->pointer->ptr_type = ptr_type;
1735       }
1736       break;
1737    }
1738
1739    case SpvOpCopyMemory: {
1740       struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer);
1741       struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer);
1742
1743       vtn_variable_copy(b, dest->pointer, src->pointer);
1744       break;
1745    }
1746
1747    case SpvOpLoad: {
1748       struct vtn_pointer *src =
1749          vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
1750
1751       if (src->mode == vtn_variable_mode_image ||
1752           src->mode == vtn_variable_mode_sampler) {
1753          vtn_push_value(b, w[2], vtn_value_type_pointer)->pointer = src;
1754          return;
1755       }
1756
1757       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1758       val->ssa = vtn_variable_load(b, src);
1759       break;
1760    }
1761
1762    case SpvOpStore: {
1763       struct vtn_pointer *dest =
1764          vtn_value(b, w[1], vtn_value_type_pointer)->pointer;
1765
1766       if (glsl_type_is_sampler(dest->type->type)) {
1767          vtn_warn("OpStore of a sampler detected.  Doing on-the-fly copy "
1768                   "propagation to workaround the problem.");
1769          assert(dest->var->copy_prop_sampler == NULL);
1770          dest->var->copy_prop_sampler =
1771             vtn_value(b, w[2], vtn_value_type_pointer)->pointer;
1772          break;
1773       }
1774
1775       struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1776       vtn_variable_store(b, src, dest);
1777       break;
1778    }
1779
1780    case SpvOpArrayLength: {
1781       struct vtn_pointer *ptr =
1782          vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
1783
1784       const uint32_t offset = ptr->var->type->offsets[w[4]];
1785       const uint32_t stride = ptr->var->type->members[w[4]]->stride;
1786
1787       unsigned chain_idx;
1788       struct vtn_type *type;
1789       nir_ssa_def *index =
1790          get_vulkan_resource_index(b, ptr, &type, &chain_idx);
1791
1792       nir_intrinsic_instr *instr =
1793          nir_intrinsic_instr_create(b->nb.shader,
1794                                     nir_intrinsic_get_buffer_size);
1795       instr->src[0] = nir_src_for_ssa(index);
1796       nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1797       nir_builder_instr_insert(&b->nb, &instr->instr);
1798       nir_ssa_def *buf_size = &instr->dest.ssa;
1799
1800       /* array_length = max(buffer_size - offset, 0) / stride */
1801       nir_ssa_def *array_length =
1802          nir_idiv(&b->nb,
1803                   nir_imax(&b->nb,
1804                            nir_isub(&b->nb,
1805                                     buf_size,
1806                                     nir_imm_int(&b->nb, offset)),
1807                            nir_imm_int(&b->nb, 0u)),
1808                   nir_imm_int(&b->nb, stride));
1809
1810       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1811       val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
1812       val->ssa->def = array_length;
1813       break;
1814    }
1815
1816    case SpvOpCopyMemorySized:
1817    default:
1818       unreachable("Unhandled opcode");
1819    }
1820 }