src/compiler/nir/nir_lower_scratch.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 /*
  29  * This lowering pass converts references to variables with loads/stores to
  30  * scratch space based on a few configurable parameters.
  31  */
  32
  33 #include "nir.h"
  34 #include "nir_builder.h"
  35 #include "nir_deref.h"
  36
  37 static void
  38 lower_load_store(nir_builder *b,
  39                  nir_intrinsic_instr *intrin,
  40                  glsl_type_size_align_func size_align)
  41 {
  42    b->cursor = nir_before_instr(&intrin->instr);
  43
  44    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
  45    nir_variable *var = nir_deref_instr_get_variable(deref);
  46
  47    nir_ssa_def *offset =
  48       nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align),
  49                       var->data.location);
  50
  51    unsigned align, UNUSED size;
  52    size_align(deref->type, &size, &align);
  53
  54    if (intrin->intrinsic == nir_intrinsic_load_deref) {
  55       nir_intrinsic_instr *load =
  56          nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch);
  57       load->num_components = intrin->num_components;
  58       load->src[0] = nir_src_for_ssa(offset);
  59       nir_intrinsic_set_align(load, align, 0);
  60       nir_ssa_dest_init(&load->instr, &load->dest,
  61                         intrin->dest.ssa.num_components,
  62                         intrin->dest.ssa.bit_size, NULL);
  63       nir_builder_instr_insert(b, &load->instr);
  64
  65       nir_ssa_def *value = &load->dest.ssa;
  66       if (glsl_type_is_boolean(deref->type))
  67          value = nir_b2i32(b, value);
  68
  69       nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
  70                                nir_src_for_ssa(&load->dest.ssa));
  71    } else {
  72       assert(intrin->intrinsic == nir_intrinsic_store_deref);
  73
  74       assert(intrin->src[1].is_ssa);
  75       nir_ssa_def *value = intrin->src[1].ssa;
  76       if (glsl_type_is_boolean(deref->type))
  77          value = nir_i2b(b, value);
  78
  79       nir_intrinsic_instr *store =
  80          nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch);
  81       store->num_components = intrin->num_components;
  82       store->src[0] = nir_src_for_ssa(value);
  83       store->src[1] = nir_src_for_ssa(offset);
  84       nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin));
  85       nir_intrinsic_set_align(store, align, 0);
  86       nir_builder_instr_insert(b, &store->instr);
  87    }
  88
  89    nir_instr_remove(&intrin->instr);
  90    nir_deref_instr_remove_if_unused(deref);
  91 }
  92
  93 bool
  94 nir_lower_vars_to_scratch(nir_shader *shader,
  95                           nir_variable_mode modes,
  96                           int size_threshold,
  97                           glsl_type_size_align_func size_align)
  98 {
  99    /* First, we walk the instructions and flag any variables we want to lower
 100     * by removing them from their respective list and setting the mode to 0.
 101     */
 102    nir_foreach_function(function, shader) {
 103       nir_foreach_block(block, function->impl) {
 104          nir_foreach_instr(instr, block) {
 105             if (instr->type != nir_instr_type_intrinsic)
 106                continue;
 107
 108             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 109             if (intrin->intrinsic != nir_intrinsic_load_deref &&
 110                 intrin->intrinsic != nir_intrinsic_store_deref)
 111                continue;
 112
 113             nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
 114             if (!(deref->mode & modes))
 115                continue;
 116
 117             if (!nir_deref_instr_has_indirect(nir_src_as_deref(intrin->src[0])))
 118                continue;
 119
 120             nir_variable *var = nir_deref_instr_get_variable(deref);
 121
 122             /* We set var->mode to 0 to indicate that a variable will be moved
 123              * to scratch.  Don't assign a scratch location twice.
 124              */
 125             if (var->data.mode == 0)
 126                continue;
 127
 128             unsigned var_size, var_align;
 129             size_align(var->type, &var_size, &var_align);
 130             if (var_size <= size_threshold)
 131                continue;
 132
 133             /* Remove it from its list */
 134             exec_node_remove(&var->node);
 135             /* Invalid mode used to flag "moving to scratch" */
 136             var->data.mode = 0;
 137
 138             var->data.location = ALIGN_POT(shader->scratch_size, var_align);
 139             shader->scratch_size = var->data.location + var_size;
 140          }
 141       }
 142    }
 143
 144    bool progress = false;
 145    nir_foreach_function(function, shader) {
 146       if (!function->impl)
 147          continue;
 148
 149       nir_builder build;
 150       nir_builder_init(&build, function->impl);
 151
 152       bool impl_progress = false;
 153       nir_foreach_block(block, function->impl) {
 154          nir_foreach_instr_safe(instr, block) {
 155             if (instr->type != nir_instr_type_intrinsic)
 156                continue;
 157
 158             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 159             if (intrin->intrinsic != nir_intrinsic_load_deref &&
 160                 intrin->intrinsic != nir_intrinsic_store_deref)
 161                continue;
 162
 163             nir_variable *var = nir_intrinsic_get_var(intrin, 0);
 164             /* Variables flagged for lowering above have mode == 0 */
 165             if (!var || var->data.mode)
 166                continue;
 167
 168             lower_load_store(&build, intrin, size_align);
 169             impl_progress = true;
 170          }
 171       }
 172
 173       if (impl_progress) {
 174          progress = true;
 175          nir_metadata_preserve(function->impl, nir_metadata_block_index |
 176                                                nir_metadata_dominance);
 177       }
 178    }
 179
 180    return progress;
 181 }