src/compiler/nir/nir_lower_io_arrays_to_elements.c

   1 /*
   2  * Copyright © 2017 Timothy Arceri
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "nir.h"
  25 #include "nir_builder.h"
  26 #include "nir_deref.h"
  27
  28 /** @file nir_lower_io_arrays_to_elements.c
  29  *
  30  * Split arrays/matrices with direct indexing into individual elements. This
  31  * will allow optimisation passes to better clean up unused elements.
  32  *
  33  */
  34
  35 static unsigned
  36 get_io_offset(nir_builder *b, nir_deref_instr *deref, nir_variable *var,
  37               unsigned *element_index, nir_ssa_def **vertex_index)
  38 {
  39    nir_deref_path path;
  40    nir_deref_path_init(&path, deref, NULL);
  41
  42    assert(path.path[0]->deref_type == nir_deref_type_var);
  43    nir_deref_instr **p = &path.path[1];
  44
  45    /* For per-vertex input arrays (i.e. geometry shader inputs), skip the
  46     * outermost array index.  Process the rest normally.
  47     */
  48    if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
  49       *vertex_index = nir_ssa_for_src(b, (*p)->arr.index, 1);
  50       p++;
  51    }
  52
  53    unsigned offset = 0;
  54    for (; *p; p++) {
  55       if ((*p)->deref_type == nir_deref_type_array) {
  56          /* must not be indirect dereference */
  57          unsigned index = nir_src_as_uint((*p)->arr.index);
  58
  59          unsigned size = glsl_count_attribute_slots((*p)->type, false);
  60          offset += size * index;
  61
  62          unsigned num_elements = glsl_type_is_array((*p)->type) ?
  63             glsl_get_aoa_size((*p)->type) : 1;
  64
  65          num_elements *= glsl_type_is_matrix(glsl_without_array((*p)->type)) ?
  66             glsl_get_matrix_columns(glsl_without_array((*p)->type)) : 1;
  67
  68          *element_index += num_elements * index;
  69       } else if ((*p)->deref_type == nir_deref_type_struct) {
  70          /* TODO: we could also add struct splitting support to this pass */
  71          break;
  72       }
  73    }
  74
  75    nir_deref_path_finish(&path);
  76
  77    return offset;
  78 }
  79
  80 static nir_variable **
  81 get_array_elements(struct hash_table *ht, nir_variable *var,
  82                    gl_shader_stage stage)
  83 {
  84    nir_variable **elements;
  85    struct hash_entry *entry = _mesa_hash_table_search(ht, var);
  86    if (!entry) {
  87       const struct glsl_type *type = var->type;
  88       if (nir_is_per_vertex_io(var, stage)) {
  89          assert(glsl_type_is_array(type));
  90          type = glsl_get_array_element(type);
  91       }
  92
  93       unsigned num_elements = glsl_type_is_array(type) ?
  94          glsl_get_aoa_size(type) : 1;
  95
  96       num_elements *= glsl_type_is_matrix(glsl_without_array(type)) ?
  97          glsl_get_matrix_columns(glsl_without_array(type)) : 1;
  98
  99       elements = (nir_variable **) calloc(num_elements, sizeof(nir_variable *));
 100       _mesa_hash_table_insert(ht, var, elements);
 101    } else {
 102       elements = (nir_variable **) entry->data;
 103    }
 104
 105    return elements;
 106 }
 107
 108 static void
 109 lower_array(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var,
 110             struct hash_table *varyings)
 111 {
 112    b->cursor = nir_before_instr(&intr->instr);
 113
 114    nir_variable **elements =
 115       get_array_elements(varyings, var, b->shader->info.stage);
 116
 117    nir_ssa_def *vertex_index = NULL;
 118    unsigned elements_index = 0;
 119    unsigned io_offset = get_io_offset(b, nir_src_as_deref(intr->src[0]),
 120                                       var, &elements_index, &vertex_index);
 121
 122    nir_variable *element = elements[elements_index];
 123    if (!element) {
 124          element = nir_variable_clone(var, b->shader);
 125          element->data.location =  var->data.location + io_offset;
 126
 127          const struct glsl_type *type = glsl_without_array(element->type);
 128
 129          /* This pass also splits matrices so we need give them a new type. */
 130          if (glsl_type_is_matrix(type)) {
 131             type = glsl_vector_type(glsl_get_base_type(type),
 132                                     glsl_get_vector_elements(type));
 133          }
 134
 135          if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
 136             type = glsl_get_array_instance(type,
 137                                            glsl_get_length(element->type));
 138          }
 139
 140          element->type = type;
 141          elements[elements_index] = element;
 142
 143          nir_shader_add_variable(b->shader, element);
 144    }
 145
 146    nir_deref_instr *element_deref = nir_build_deref_var(b, element);
 147
 148    if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
 149       assert(vertex_index);
 150       element_deref = nir_build_deref_array(b, element_deref, vertex_index);
 151    }
 152
 153    nir_intrinsic_instr *element_intr =
 154       nir_intrinsic_instr_create(b->shader, intr->intrinsic);
 155    element_intr->num_components = intr->num_components;
 156    element_intr->src[0] = nir_src_for_ssa(&element_deref->dest.ssa);
 157
 158    if (intr->intrinsic != nir_intrinsic_store_deref) {
 159       nir_ssa_dest_init(&element_intr->instr, &element_intr->dest,
 160                         intr->num_components, intr->dest.ssa.bit_size, NULL);
 161
 162       if (intr->intrinsic == nir_intrinsic_interp_deref_at_offset ||
 163           intr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
 164          nir_src_copy(&element_intr->src[1], &intr->src[1],
 165                       &element_intr->instr);
 166       }
 167
 168       nir_ssa_def_rewrite_uses(&intr->dest.ssa,
 169                                nir_src_for_ssa(&element_intr->dest.ssa));
 170    } else {
 171       nir_intrinsic_set_write_mask(element_intr,
 172                                    nir_intrinsic_write_mask(intr));
 173       nir_src_copy(&element_intr->src[1], &intr->src[1],
 174                    &element_intr->instr);
 175    }
 176
 177    nir_builder_instr_insert(b, &element_intr->instr);
 178
 179    /* Remove the old load intrinsic */
 180    nir_instr_remove(&intr->instr);
 181 }
 182
 183 static bool
 184 deref_has_indirect(nir_builder *b, nir_variable *var, nir_deref_path *path)
 185 {
 186    assert(path->path[0]->deref_type == nir_deref_type_var);
 187    nir_deref_instr **p = &path->path[1];
 188
 189    if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
 190       p++;
 191    }
 192
 193    for (; *p; p++) {
 194       if ((*p)->deref_type != nir_deref_type_array)
 195          continue;
 196
 197       if (!nir_src_is_const((*p)->arr.index))
 198          return true;
 199    }
 200
 201    return false;
 202 }
 203
 204 /* Creates a mask of locations that contains arrays that are indexed via
 205  * indirect indexing.
 206  */
 207 static void
 208 create_indirects_mask(nir_shader *shader, uint64_t *indirects,
 209                       uint64_t *patch_indirects, nir_variable_mode mode)
 210 {
 211    nir_foreach_function(function, shader) {
 212       if (function->impl) {
 213          nir_builder b;
 214          nir_builder_init(&b, function->impl);
 215
 216          nir_foreach_block(block, function->impl) {
 217             nir_foreach_instr_safe(instr, block) {
 218
 219                if (instr->type != nir_instr_type_intrinsic)
 220                   continue;
 221
 222                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 223
 224                if (intr->intrinsic != nir_intrinsic_load_deref &&
 225                    intr->intrinsic != nir_intrinsic_store_deref &&
 226                    intr->intrinsic != nir_intrinsic_interp_deref_at_centroid &&
 227                    intr->intrinsic != nir_intrinsic_interp_deref_at_sample &&
 228                    intr->intrinsic != nir_intrinsic_interp_deref_at_offset)
 229                   continue;
 230
 231                nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
 232                nir_variable *var = nir_deref_instr_get_variable(deref);
 233
 234                if (var->data.mode != mode)
 235                   continue;
 236
 237                nir_deref_path path;
 238                nir_deref_path_init(&path, deref, NULL);
 239
 240                uint64_t loc_mask = ((uint64_t)1) << var->data.location;
 241                if (var->data.patch) {
 242                   if (deref_has_indirect(&b, var, &path))
 243                      patch_indirects[var->data.location_frac] |= loc_mask;
 244                } else {
 245                   if (deref_has_indirect(&b, var, &path))
 246                      indirects[var->data.location_frac] |= loc_mask;
 247                }
 248
 249                nir_deref_path_finish(&path);
 250             }
 251          }
 252       }
 253    }
 254 }
 255
 256 static void
 257 lower_io_arrays_to_elements(nir_shader *shader, nir_variable_mode mask,
 258                             uint64_t *indirects, uint64_t *patch_indirects,
 259                             struct hash_table *varyings,
 260                             bool after_cross_stage_opts)
 261 {
 262    nir_foreach_function(function, shader) {
 263       if (function->impl) {
 264          nir_builder b;
 265          nir_builder_init(&b, function->impl);
 266
 267          nir_foreach_block(block, function->impl) {
 268             nir_foreach_instr_safe(instr, block) {
 269                if (instr->type != nir_instr_type_intrinsic)
 270                   continue;
 271
 272                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 273
 274                if (intr->intrinsic != nir_intrinsic_load_deref &&
 275                    intr->intrinsic != nir_intrinsic_store_deref &&
 276                    intr->intrinsic != nir_intrinsic_interp_deref_at_centroid &&
 277                    intr->intrinsic != nir_intrinsic_interp_deref_at_sample &&
 278                    intr->intrinsic != nir_intrinsic_interp_deref_at_offset)
 279                   continue;
 280
 281                nir_variable *var =
 282                   nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0]));
 283
 284                /* Skip indirects */
 285                uint64_t loc_mask = ((uint64_t)1) << var->data.location;
 286                if (var->data.patch) {
 287                   if (patch_indirects[var->data.location_frac] & loc_mask)
 288                      continue;
 289                } else {
 290                   if (indirects[var->data.location_frac] & loc_mask)
 291                      continue;
 292                }
 293
 294                nir_variable_mode mode = var->data.mode;
 295
 296                const struct glsl_type *type = var->type;
 297                if (nir_is_per_vertex_io(var, b.shader->info.stage)) {
 298                   assert(glsl_type_is_array(type));
 299                   type = glsl_get_array_element(type);
 300                }
 301
 302                /* Skip types we cannot split.
 303                 *
 304                 * TODO: Add support for struct splitting.
 305                 */
 306                if ((!glsl_type_is_array(type) && !glsl_type_is_matrix(type))||
 307                    glsl_type_is_struct(glsl_without_array(type)))
 308                   continue;
 309
 310                /* Skip builtins */
 311                if (!after_cross_stage_opts &&
 312                    var->data.location < VARYING_SLOT_VAR0 &&
 313                    var->data.location >= 0)
 314                   continue;
 315
 316                /* Don't bother splitting if we can't opt away any unused
 317                 * elements.
 318                 */
 319                if (!after_cross_stage_opts && var->data.always_active_io)
 320                   continue;
 321
 322                switch (intr->intrinsic) {
 323                case nir_intrinsic_interp_deref_at_centroid:
 324                case nir_intrinsic_interp_deref_at_sample:
 325                case nir_intrinsic_interp_deref_at_offset:
 326                case nir_intrinsic_load_deref:
 327                case nir_intrinsic_store_deref:
 328                   if ((mask & nir_var_shader_in && mode == nir_var_shader_in) ||
 329                       (mask & nir_var_shader_out && mode == nir_var_shader_out))
 330                      lower_array(&b, intr, var, varyings);
 331                   break;
 332                default:
 333                   break;
 334                }
 335             }
 336          }
 337       }
 338    }
 339 }
 340
 341 void
 342 nir_lower_io_arrays_to_elements_no_indirects(nir_shader *shader,
 343                                              bool outputs_only)
 344 {
 345    struct hash_table *split_inputs =
 346       _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 347                               _mesa_key_pointer_equal);
 348    struct hash_table *split_outputs =
 349       _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 350                               _mesa_key_pointer_equal);
 351
 352    uint64_t indirects[4] = {0}, patch_indirects[4] = {0};
 353
 354    lower_io_arrays_to_elements(shader, nir_var_shader_out, indirects,
 355                                patch_indirects, split_outputs, true);
 356
 357    if (!outputs_only) {
 358       lower_io_arrays_to_elements(shader, nir_var_shader_in, indirects,
 359                                   patch_indirects, split_inputs, true);
 360
 361       /* Remove old input from the shaders inputs list */
 362       struct hash_entry *entry;
 363       hash_table_foreach(split_inputs, entry) {
 364          nir_variable *var = (nir_variable *) entry->key;
 365          exec_node_remove(&var->node);
 366
 367          free(entry->data);
 368       }
 369    }
 370
 371    /* Remove old output from the shaders outputs list */
 372    struct hash_entry *entry;
 373    hash_table_foreach(split_outputs, entry) {
 374       nir_variable *var = (nir_variable *) entry->key;
 375       exec_node_remove(&var->node);
 376
 377       free(entry->data);
 378    }
 379
 380    _mesa_hash_table_destroy(split_inputs, NULL);
 381    _mesa_hash_table_destroy(split_outputs, NULL);
 382
 383    nir_remove_dead_derefs(shader);
 384 }
 385
 386 void
 387 nir_lower_io_arrays_to_elements(nir_shader *producer, nir_shader *consumer)
 388 {
 389    struct hash_table *split_inputs =
 390       _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 391                               _mesa_key_pointer_equal);
 392    struct hash_table *split_outputs =
 393       _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 394                               _mesa_key_pointer_equal);
 395
 396    uint64_t indirects[4] = {0}, patch_indirects[4] = {0};
 397    create_indirects_mask(producer, indirects, patch_indirects,
 398                          nir_var_shader_out);
 399    create_indirects_mask(consumer, indirects, patch_indirects,
 400                          nir_var_shader_in);
 401
 402    lower_io_arrays_to_elements(producer, nir_var_shader_out, indirects,
 403                                patch_indirects, split_outputs, false);
 404
 405    lower_io_arrays_to_elements(consumer, nir_var_shader_in, indirects,
 406                                patch_indirects, split_inputs, false);
 407
 408    /* Remove old input from the shaders inputs list */
 409    struct hash_entry *entry;
 410    hash_table_foreach(split_inputs, entry) {
 411       nir_variable *var = (nir_variable *) entry->key;
 412       exec_node_remove(&var->node);
 413
 414       free(entry->data);
 415    }
 416
 417    /* Remove old output from the shaders outputs list */
 418    hash_table_foreach(split_outputs, entry) {
 419       nir_variable *var = (nir_variable *) entry->key;
 420       exec_node_remove(&var->node);
 421
 422       free(entry->data);
 423    }
 424
 425    _mesa_hash_table_destroy(split_inputs, NULL);
 426    _mesa_hash_table_destroy(split_outputs, NULL);
 427
 428    nir_remove_dead_derefs(producer);
 429    nir_remove_dead_derefs(consumer);
 430 }