src/compiler/nir/nir_lower_array_deref_of_vec.c

   1 /*
   2  * Copyright © 2019 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "nir.h"
  25 #include "nir_builder.h"
  26
  27 static void
  28 build_write_masked_store(nir_builder *b, nir_deref_instr *vec_deref,
  29                          nir_ssa_def *value, unsigned component)
  30 {
  31    assert(value->num_components == 1);
  32    unsigned num_components = glsl_get_components(vec_deref->type);
  33    assert(num_components > 1 && num_components <= NIR_MAX_VEC_COMPONENTS);
  34
  35    nir_ssa_def *u = nir_ssa_undef(b, 1, value->bit_size);
  36    nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS];
  37    for (unsigned i = 0; i < num_components; i++)
  38       comps[i] = (i == component) ? value : u;
  39
  40    nir_ssa_def *vec = nir_vec(b, comps, num_components);
  41    nir_store_deref(b, vec_deref, vec, (1u << component));
  42 }
  43
  44 static void
  45 build_write_masked_stores(nir_builder *b, nir_deref_instr *vec_deref,
  46                           nir_ssa_def *value, nir_ssa_def *index,
  47                           unsigned start, unsigned end)
  48 {
  49    if (start == end - 1) {
  50       build_write_masked_store(b, vec_deref, value, start);
  51    } else {
  52       unsigned mid = start + (end - start) / 2;
  53       nir_push_if(b, nir_ilt(b, index, nir_imm_int(b, mid)));
  54       build_write_masked_stores(b, vec_deref, value, index, start, mid);
  55       nir_push_else(b, NULL);
  56       build_write_masked_stores(b, vec_deref, value, index, mid, end);
  57       nir_pop_if(b, NULL);
  58    }
  59 }
  60
  61 static bool
  62 nir_lower_array_deref_of_vec_impl(nir_function_impl *impl,
  63                                   nir_variable_mode modes,
  64                                   nir_lower_array_deref_of_vec_options options)
  65 {
  66    bool progress = false;
  67
  68    nir_builder b;
  69    nir_builder_init(&b, impl);
  70
  71    nir_foreach_block(block, impl) {
  72       nir_foreach_instr_safe(instr, block) {
  73          if (instr->type != nir_instr_type_intrinsic)
  74             continue;
  75
  76          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  77          assert(intrin->intrinsic != nir_intrinsic_copy_deref);
  78
  79          if (intrin->intrinsic != nir_intrinsic_load_deref &&
  80              intrin->intrinsic != nir_intrinsic_interp_deref_at_centroid &&
  81              intrin->intrinsic != nir_intrinsic_interp_deref_at_sample &&
  82              intrin->intrinsic != nir_intrinsic_interp_deref_at_offset &&
  83              intrin->intrinsic != nir_intrinsic_interp_deref_at_vertex &&
  84              intrin->intrinsic != nir_intrinsic_store_deref)
  85             continue;
  86
  87          nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
  88          if (!(deref->mode & modes))
  89             continue;
  90
  91          /* We only care about array derefs that act on vectors */
  92          if (deref->deref_type != nir_deref_type_array)
  93             continue;
  94
  95          nir_deref_instr *vec_deref = nir_deref_instr_parent(deref);
  96          if (!glsl_type_is_vector(vec_deref->type))
  97             continue;
  98
  99          assert(intrin->num_components == 1);
 100          unsigned num_components = glsl_get_components(vec_deref->type);
 101          assert(num_components > 1 && num_components <= NIR_MAX_VEC_COMPONENTS);
 102
 103          b.cursor = nir_after_instr(&intrin->instr);
 104
 105          if (intrin->intrinsic == nir_intrinsic_store_deref) {
 106             assert(intrin->src[1].is_ssa);
 107             nir_ssa_def *value = intrin->src[1].ssa;
 108
 109             if (nir_src_is_const(deref->arr.index)) {
 110                if (!(options & nir_lower_direct_array_deref_of_vec_store))
 111                   continue;
 112
 113                unsigned index = nir_src_as_uint(deref->arr.index);
 114                /* If index is OOB, we throw the old store away and don't
 115                 * replace it with anything.
 116                 */
 117                if (index < num_components)
 118                   build_write_masked_store(&b, vec_deref, value, index);
 119             } else {
 120                if (!(options & nir_lower_indirect_array_deref_of_vec_store))
 121                   continue;
 122
 123                nir_ssa_def *index = nir_ssa_for_src(&b, deref->arr.index, 1);
 124                build_write_masked_stores(&b, vec_deref, value, index,
 125                                          0, num_components);
 126             }
 127             nir_instr_remove(&intrin->instr);
 128
 129             progress = true;
 130          } else {
 131             if (nir_src_is_const(deref->arr.index)) {
 132                if (!(options & nir_lower_direct_array_deref_of_vec_load))
 133                   continue;
 134             } else {
 135                if (!(options & nir_lower_indirect_array_deref_of_vec_load))
 136                   continue;
 137             }
 138
 139             /* Turn the load into a vector load */
 140             nir_instr_rewrite_src(&intrin->instr, &intrin->src[0],
 141                                   nir_src_for_ssa(&vec_deref->dest.ssa));
 142             intrin->dest.ssa.num_components = num_components;
 143             intrin->num_components = num_components;
 144
 145             nir_ssa_def *index = nir_ssa_for_src(&b, deref->arr.index, 1);
 146             nir_ssa_def *scalar =
 147                nir_vector_extract(&b, &intrin->dest.ssa, index);
 148             if (scalar->parent_instr->type == nir_instr_type_ssa_undef) {
 149                nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
 150                                         nir_src_for_ssa(scalar));
 151                nir_instr_remove(&intrin->instr);
 152             } else {
 153                nir_ssa_def_rewrite_uses_after(&intrin->dest.ssa,
 154                                               nir_src_for_ssa(scalar),
 155                                               scalar->parent_instr);
 156             }
 157             progress = true;
 158          }
 159       }
 160    }
 161
 162    if (progress) {
 163       nir_metadata_preserve(impl, nir_metadata_block_index |
 164                                   nir_metadata_dominance);
 165    } else {
 166       nir_metadata_preserve(impl, nir_metadata_all);
 167    }
 168
 169    return progress;
 170 }
 171
 172 /* Lowers away array dereferences on vectors
 173  *
 174  * These are allowed on certain variable types such as SSBOs and TCS outputs.
 175  * However, not everyone can actually handle them everywhere.  There are also
 176  * cases where we want to lower them for performance reasons.
 177  *
 178  * This patch assumes that copy_deref instructions have already been lowered.
 179  */
 180 bool
 181 nir_lower_array_deref_of_vec(nir_shader *shader, nir_variable_mode modes,
 182                              nir_lower_array_deref_of_vec_options options)
 183 {
 184    bool progress = false;
 185
 186    nir_foreach_function(function, shader) {
 187       if (function->impl &&
 188           nir_lower_array_deref_of_vec_impl(function->impl, modes, options))
 189          progress = true;
 190    }
 191
 192    return progress;
 193 }