src/compiler/nir/nir_lower_io_arrays_to_elements.c

   1 /*
   2  * Copyright © 2017 Timothy Arceri
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "nir.h"
  25 #include "nir_builder.h"
  26 #include "nir_deref.h"
  27
  28 /** @file nir_lower_io_arrays_to_elements.c
  29  *
  30  * Split arrays/matrices with direct indexing into individual elements. This
  31  * will allow optimisation passes to better clean up unused elements.
  32  *
  33  */
  34
  35 static unsigned
  36 get_io_offset(nir_builder *b, nir_deref_instr *deref, nir_variable *var,
  37               unsigned *element_index, unsigned *xfb_offset,
  38               nir_ssa_def **vertex_index)
  39 {
  40    nir_deref_path path;
  41    nir_deref_path_init(&path, deref, NULL);
  42
  43    assert(path.path[0]->deref_type == nir_deref_type_var);
  44    nir_deref_instr **p = &path.path[1];
  45
  46    /* For per-vertex input arrays (i.e. geometry shader inputs), skip the
  47     * outermost array index.  Process the rest normally.
  48     */
  49    if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
  50       *vertex_index = nir_ssa_for_src(b, (*p)->arr.index, 1);
  51       p++;
  52    }
  53
  54    unsigned offset = 0;
  55    *xfb_offset = 0;
  56    for (; *p; p++) {
  57       if ((*p)->deref_type == nir_deref_type_array) {
  58          /* must not be indirect dereference */
  59          unsigned index = nir_src_as_uint((*p)->arr.index);
  60
  61          unsigned size = glsl_count_attribute_slots((*p)->type, false);
  62          offset += size * index;
  63
  64          xfb_offset += index * glsl_get_component_slots((*p)->type) * 4;
  65
  66          unsigned num_elements = glsl_type_is_array((*p)->type) ?
  67             glsl_get_aoa_size((*p)->type) : 1;
  68
  69          num_elements *= glsl_type_is_matrix(glsl_without_array((*p)->type)) ?
  70             glsl_get_matrix_columns(glsl_without_array((*p)->type)) : 1;
  71
  72          *element_index += num_elements * index;
  73       } else if ((*p)->deref_type == nir_deref_type_struct) {
  74          /* TODO: we could also add struct splitting support to this pass */
  75          break;
  76       }
  77    }
  78
  79    nir_deref_path_finish(&path);
  80
  81    return offset;
  82 }
  83
  84 static nir_variable **
  85 get_array_elements(struct hash_table *ht, nir_variable *var,
  86                    gl_shader_stage stage)
  87 {
  88    nir_variable **elements;
  89    struct hash_entry *entry = _mesa_hash_table_search(ht, var);
  90    if (!entry) {
  91       const struct glsl_type *type = var->type;
  92       if (nir_is_per_vertex_io(var, stage)) {
  93          assert(glsl_type_is_array(type));
  94          type = glsl_get_array_element(type);
  95       }
  96
  97       unsigned num_elements = glsl_type_is_array(type) ?
  98          glsl_get_aoa_size(type) : 1;
  99
 100       num_elements *= glsl_type_is_matrix(glsl_without_array(type)) ?
 101          glsl_get_matrix_columns(glsl_without_array(type)) : 1;
 102
 103       elements = (nir_variable **) calloc(num_elements, sizeof(nir_variable *));
 104       _mesa_hash_table_insert(ht, var, elements);
 105    } else {
 106       elements = (nir_variable **) entry->data;
 107    }
 108
 109    return elements;
 110 }
 111
 112 static void
 113 lower_array(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var,
 114             struct hash_table *varyings)
 115 {
 116    b->cursor = nir_before_instr(&intr->instr);
 117
 118    nir_variable **elements =
 119       get_array_elements(varyings, var, b->shader->info.stage);
 120
 121    nir_ssa_def *vertex_index = NULL;
 122    unsigned elements_index = 0;
 123    unsigned xfb_offset = 0;
 124    unsigned io_offset = get_io_offset(b, nir_src_as_deref(intr->src[0]),
 125                                       var, &elements_index, &xfb_offset,
 126                                       &vertex_index);
 127
 128    nir_variable *element = elements[elements_index];
 129    if (!element) {
 130          element = nir_variable_clone(var, b->shader);
 131          element->data.location =  var->data.location + io_offset;
 132
 133          if (var->data.explicit_offset)
 134             element->data.offset = var->data.offset + xfb_offset;
 135
 136          const struct glsl_type *type = glsl_without_array(element->type);
 137
 138          /* This pass also splits matrices so we need give them a new type. */
 139          if (glsl_type_is_matrix(type))
 140             type = glsl_get_column_type(type);
 141
 142          if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
 143             type = glsl_array_type(type, glsl_get_length(element->type),
 144                                    glsl_get_explicit_stride(element->type));
 145          }
 146
 147          element->type = type;
 148          elements[elements_index] = element;
 149
 150          nir_shader_add_variable(b->shader, element);
 151    }
 152
 153    nir_deref_instr *element_deref = nir_build_deref_var(b, element);
 154
 155    if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
 156       assert(vertex_index);
 157       element_deref = nir_build_deref_array(b, element_deref, vertex_index);
 158    }
 159
 160    nir_intrinsic_instr *element_intr =
 161       nir_intrinsic_instr_create(b->shader, intr->intrinsic);
 162    element_intr->num_components = intr->num_components;
 163    element_intr->src[0] = nir_src_for_ssa(&element_deref->dest.ssa);
 164
 165    if (intr->intrinsic != nir_intrinsic_store_deref) {
 166       nir_ssa_dest_init(&element_intr->instr, &element_intr->dest,
 167                         intr->num_components, intr->dest.ssa.bit_size, NULL);
 168
 169       if (intr->intrinsic == nir_intrinsic_interp_deref_at_offset ||
 170           intr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
 171          nir_src_copy(&element_intr->src[1], &intr->src[1],
 172                       &element_intr->instr);
 173       }
 174
 175       nir_ssa_def_rewrite_uses(&intr->dest.ssa,
 176                                nir_src_for_ssa(&element_intr->dest.ssa));
 177    } else {
 178       nir_intrinsic_set_write_mask(element_intr,
 179                                    nir_intrinsic_write_mask(intr));
 180       nir_src_copy(&element_intr->src[1], &intr->src[1],
 181                    &element_intr->instr);
 182    }
 183
 184    nir_builder_instr_insert(b, &element_intr->instr);
 185
 186    /* Remove the old load intrinsic */
 187    nir_instr_remove(&intr->instr);
 188 }
 189
 190 static bool
 191 deref_has_indirect(nir_builder *b, nir_variable *var, nir_deref_path *path)
 192 {
 193    assert(path->path[0]->deref_type == nir_deref_type_var);
 194    nir_deref_instr **p = &path->path[1];
 195
 196    if (nir_is_per_vertex_io(var, b->shader->info.stage)) {
 197       p++;
 198    }
 199
 200    for (; *p; p++) {
 201       if ((*p)->deref_type != nir_deref_type_array)
 202          continue;
 203
 204       if (!nir_src_is_const((*p)->arr.index))
 205          return true;
 206    }
 207
 208    return false;
 209 }
 210
 211 /* Creates a mask of locations that contains arrays that are indexed via
 212  * indirect indexing.
 213  */
 214 static void
 215 create_indirects_mask(nir_shader *shader, uint64_t *indirects,
 216                       uint64_t *patch_indirects, nir_variable_mode mode)
 217 {
 218    nir_foreach_function(function, shader) {
 219       if (function->impl) {
 220          nir_builder b;
 221          nir_builder_init(&b, function->impl);
 222
 223          nir_foreach_block(block, function->impl) {
 224             nir_foreach_instr_safe(instr, block) {
 225
 226                if (instr->type != nir_instr_type_intrinsic)
 227                   continue;
 228
 229                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 230
 231                if (intr->intrinsic != nir_intrinsic_load_deref &&
 232                    intr->intrinsic != nir_intrinsic_store_deref &&
 233                    intr->intrinsic != nir_intrinsic_interp_deref_at_centroid &&
 234                    intr->intrinsic != nir_intrinsic_interp_deref_at_sample &&
 235                    intr->intrinsic != nir_intrinsic_interp_deref_at_offset)
 236                   continue;
 237
 238                nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
 239                if (deref->mode != mode)
 240                   continue;
 241
 242                nir_variable *var = nir_deref_instr_get_variable(deref);
 243
 244                nir_deref_path path;
 245                nir_deref_path_init(&path, deref, NULL);
 246
 247                uint64_t loc_mask = ((uint64_t)1) << var->data.location;
 248                if (var->data.patch) {
 249                   if (deref_has_indirect(&b, var, &path))
 250                      patch_indirects[var->data.location_frac] |= loc_mask;
 251                } else {
 252                   if (deref_has_indirect(&b, var, &path))
 253                      indirects[var->data.location_frac] |= loc_mask;
 254                }
 255
 256                nir_deref_path_finish(&path);
 257             }
 258          }
 259       }
 260    }
 261 }
 262
 263 static void
 264 lower_io_arrays_to_elements(nir_shader *shader, nir_variable_mode mask,
 265                             uint64_t *indirects, uint64_t *patch_indirects,
 266                             struct hash_table *varyings,
 267                             bool after_cross_stage_opts)
 268 {
 269    nir_foreach_function(function, shader) {
 270       if (function->impl) {
 271          nir_builder b;
 272          nir_builder_init(&b, function->impl);
 273
 274          nir_foreach_block(block, function->impl) {
 275             nir_foreach_instr_safe(instr, block) {
 276                if (instr->type != nir_instr_type_intrinsic)
 277                   continue;
 278
 279                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 280
 281                if (intr->intrinsic != nir_intrinsic_load_deref &&
 282                    intr->intrinsic != nir_intrinsic_store_deref &&
 283                    intr->intrinsic != nir_intrinsic_interp_deref_at_centroid &&
 284                    intr->intrinsic != nir_intrinsic_interp_deref_at_sample &&
 285                    intr->intrinsic != nir_intrinsic_interp_deref_at_offset)
 286                   continue;
 287
 288                nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
 289                if (!(deref->mode & mask))
 290                   continue;
 291
 292                nir_variable *var = nir_deref_instr_get_variable(deref);
 293
 294                /* Drivers assume compact arrays are, in fact, arrays. */
 295                if (var->data.compact)
 296                   continue;
 297
 298                /* Skip indirects */
 299                uint64_t loc_mask = ((uint64_t)1) << var->data.location;
 300                if (var->data.patch) {
 301                   if (patch_indirects[var->data.location_frac] & loc_mask)
 302                      continue;
 303                } else {
 304                   if (indirects[var->data.location_frac] & loc_mask)
 305                      continue;
 306                }
 307
 308                nir_variable_mode mode = var->data.mode;
 309
 310                const struct glsl_type *type = var->type;
 311                if (nir_is_per_vertex_io(var, b.shader->info.stage)) {
 312                   assert(glsl_type_is_array(type));
 313                   type = glsl_get_array_element(type);
 314                }
 315
 316                /* Skip types we cannot split.
 317                 *
 318                 * TODO: Add support for struct splitting.
 319                 */
 320                if ((!glsl_type_is_array(type) && !glsl_type_is_matrix(type))||
 321                    glsl_type_is_struct_or_ifc(glsl_without_array(type)))
 322                   continue;
 323
 324                /* Skip builtins */
 325                if (!after_cross_stage_opts &&
 326                    var->data.location < VARYING_SLOT_VAR0 &&
 327                    var->data.location >= 0)
 328                   continue;
 329
 330                /* Don't bother splitting if we can't opt away any unused
 331                 * elements.
 332                 */
 333                if (!after_cross_stage_opts && var->data.always_active_io)
 334                   continue;
 335
 336                switch (intr->intrinsic) {
 337                case nir_intrinsic_interp_deref_at_centroid:
 338                case nir_intrinsic_interp_deref_at_sample:
 339                case nir_intrinsic_interp_deref_at_offset:
 340                case nir_intrinsic_load_deref:
 341                case nir_intrinsic_store_deref:
 342                   if ((mask & nir_var_shader_in && mode == nir_var_shader_in) ||
 343                       (mask & nir_var_shader_out && mode == nir_var_shader_out))
 344                      lower_array(&b, intr, var, varyings);
 345                   break;
 346                default:
 347                   break;
 348                }
 349             }
 350          }
 351       }
 352    }
 353 }
 354
 355 void
 356 nir_lower_io_arrays_to_elements_no_indirects(nir_shader *shader,
 357                                              bool outputs_only)
 358 {
 359    struct hash_table *split_inputs = _mesa_pointer_hash_table_create(NULL);
 360    struct hash_table *split_outputs = _mesa_pointer_hash_table_create(NULL);
 361
 362    uint64_t indirects[4] = {0}, patch_indirects[4] = {0};
 363
 364    lower_io_arrays_to_elements(shader, nir_var_shader_out, indirects,
 365                                patch_indirects, split_outputs, true);
 366
 367    if (!outputs_only) {
 368       lower_io_arrays_to_elements(shader, nir_var_shader_in, indirects,
 369                                   patch_indirects, split_inputs, true);
 370
 371       /* Remove old input from the shaders inputs list */
 372       hash_table_foreach(split_inputs, entry) {
 373          nir_variable *var = (nir_variable *) entry->key;
 374          exec_node_remove(&var->node);
 375
 376          free(entry->data);
 377       }
 378    }
 379
 380    /* Remove old output from the shaders outputs list */
 381    hash_table_foreach(split_outputs, entry) {
 382       nir_variable *var = (nir_variable *) entry->key;
 383       exec_node_remove(&var->node);
 384
 385       free(entry->data);
 386    }
 387
 388    _mesa_hash_table_destroy(split_inputs, NULL);
 389    _mesa_hash_table_destroy(split_outputs, NULL);
 390
 391    nir_remove_dead_derefs(shader);
 392 }
 393
 394 void
 395 nir_lower_io_arrays_to_elements(nir_shader *producer, nir_shader *consumer)
 396 {
 397    struct hash_table *split_inputs = _mesa_pointer_hash_table_create(NULL);
 398    struct hash_table *split_outputs = _mesa_pointer_hash_table_create(NULL);
 399
 400    uint64_t indirects[4] = {0}, patch_indirects[4] = {0};
 401    create_indirects_mask(producer, indirects, patch_indirects,
 402                          nir_var_shader_out);
 403    create_indirects_mask(consumer, indirects, patch_indirects,
 404                          nir_var_shader_in);
 405
 406    lower_io_arrays_to_elements(producer, nir_var_shader_out, indirects,
 407                                patch_indirects, split_outputs, false);
 408
 409    lower_io_arrays_to_elements(consumer, nir_var_shader_in, indirects,
 410                                patch_indirects, split_inputs, false);
 411
 412    /* Remove old input from the shaders inputs list */
 413    hash_table_foreach(split_inputs, entry) {
 414       nir_variable *var = (nir_variable *) entry->key;
 415       exec_node_remove(&var->node);
 416
 417       free(entry->data);
 418    }
 419
 420    /* Remove old output from the shaders outputs list */
 421    hash_table_foreach(split_outputs, entry) {
 422       nir_variable *var = (nir_variable *) entry->key;
 423       exec_node_remove(&var->node);
 424
 425       free(entry->data);
 426    }
 427
 428    _mesa_hash_table_destroy(split_inputs, NULL);
 429    _mesa_hash_table_destroy(split_outputs, NULL);
 430
 431    nir_remove_dead_derefs(producer);
 432    nir_remove_dead_derefs(consumer);
 433 }