src/compiler/nir/nir_lower_scratch.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 /*
  29  * This lowering pass converts references to variables with loads/stores to
  30  * scratch space based on a few configurable parameters.
  31  */
  32
  33 #include "nir.h"
  34 #include "nir_builder.h"
  35 #include "nir_deref.h"
  36
  37 static bool
  38 deref_has_indirect(nir_deref_instr *deref)
  39 {
  40    while (deref->deref_type != nir_deref_type_var) {
  41       if (deref->deref_type == nir_deref_type_array &&
  42           nir_src_as_const_value(deref->arr.index) == NULL)
  43          return true;
  44
  45       deref = nir_deref_instr_parent(deref);
  46    }
  47
  48    return false;
  49 }
  50
  51 static void
  52 lower_load_store(nir_builder *b,
  53                  nir_intrinsic_instr *intrin,
  54                  glsl_type_size_align_func size_align)
  55 {
  56    b->cursor = nir_before_instr(&intrin->instr);
  57
  58    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
  59    nir_variable *var = nir_deref_instr_get_variable(deref);
  60
  61    nir_ssa_def *offset =
  62       nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align),
  63                       var->data.location);
  64
  65    unsigned align, UNUSED size;
  66    size_align(deref->type, &size, &align);
  67
  68    if (intrin->intrinsic == nir_intrinsic_load_deref) {
  69       nir_intrinsic_instr *load =
  70          nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch);
  71       load->num_components = intrin->num_components;
  72       load->src[0] = nir_src_for_ssa(offset);
  73       nir_intrinsic_set_align(load, align, 0);
  74       nir_ssa_dest_init(&load->instr, &load->dest,
  75                         intrin->dest.ssa.num_components,
  76                         intrin->dest.ssa.bit_size, NULL);
  77       nir_builder_instr_insert(b, &load->instr);
  78
  79       nir_ssa_def *value = &load->dest.ssa;
  80       if (glsl_type_is_boolean(deref->type))
  81          value = nir_b2i32(b, value);
  82
  83       nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
  84                                nir_src_for_ssa(&load->dest.ssa));
  85    } else {
  86       assert(intrin->intrinsic == nir_intrinsic_store_deref);
  87
  88       assert(intrin->src[1].is_ssa);
  89       nir_ssa_def *value = intrin->src[1].ssa;
  90       if (glsl_type_is_boolean(deref->type))
  91          value = nir_i2b(b, value);
  92
  93       nir_intrinsic_instr *store =
  94          nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch);
  95       store->num_components = intrin->num_components;
  96       store->src[0] = nir_src_for_ssa(value);
  97       store->src[1] = nir_src_for_ssa(offset);
  98       nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin));
  99       nir_intrinsic_set_align(store, align, 0);
 100       nir_builder_instr_insert(b, &store->instr);
 101    }
 102
 103    nir_instr_remove(&intrin->instr);
 104    nir_deref_instr_remove_if_unused(deref);
 105 }
 106
 107 bool
 108 nir_lower_vars_to_scratch(nir_shader *shader,
 109                           nir_variable_mode modes,
 110                           int size_threshold,
 111                           glsl_type_size_align_func size_align)
 112 {
 113    /* First, we walk the instructions and flag any variables we want to lower
 114     * by removing them from their respective list and setting the mode to 0.
 115     */
 116    nir_foreach_function(function, shader) {
 117       nir_foreach_block(block, function->impl) {
 118          nir_foreach_instr(instr, block) {
 119             if (instr->type != nir_instr_type_intrinsic)
 120                continue;
 121
 122             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 123             if (intrin->intrinsic != nir_intrinsic_load_deref &&
 124                 intrin->intrinsic != nir_intrinsic_store_deref)
 125                continue;
 126
 127             nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
 128             if (!(deref->mode & modes))
 129                continue;
 130
 131             if (!deref_has_indirect(nir_src_as_deref(intrin->src[0])))
 132                continue;
 133
 134             nir_variable *var = nir_deref_instr_get_variable(deref);
 135
 136             /* We set var->mode to 0 to indicate that a variable will be moved
 137              * to scratch.  Don't assign a scratch location twice.
 138              */
 139             if (var->data.mode == 0)
 140                continue;
 141
 142             unsigned var_size, var_align;
 143             size_align(var->type, &var_size, &var_align);
 144             if (var_size <= size_threshold)
 145                continue;
 146
 147             /* Remove it from its list */
 148             exec_node_remove(&var->node);
 149             /* Invalid mode used to flag "moving to scratch" */
 150             var->data.mode = 0;
 151
 152             var->data.location = ALIGN_POT(shader->scratch_size, var_align);
 153             shader->scratch_size = var->data.location + var_size;
 154          }
 155       }
 156    }
 157
 158    bool progress = false;
 159    nir_foreach_function(function, shader) {
 160       if (!function->impl)
 161          continue;
 162
 163       nir_builder build;
 164       nir_builder_init(&build, function->impl);
 165
 166       bool impl_progress = false;
 167       nir_foreach_block(block, function->impl) {
 168          nir_foreach_instr_safe(instr, block) {
 169             if (instr->type != nir_instr_type_intrinsic)
 170                continue;
 171
 172             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 173             if (intrin->intrinsic != nir_intrinsic_load_deref &&
 174                 intrin->intrinsic != nir_intrinsic_store_deref)
 175                continue;
 176
 177             nir_variable *var = nir_intrinsic_get_var(intrin, 0);
 178             /* Variables flagged for lowering above have mode == 0 */
 179             if (!var || var->data.mode)
 180                continue;
 181
 182             lower_load_store(&build, intrin, size_align);
 183             impl_progress = true;
 184          }
 185       }
 186
 187       if (impl_progress) {
 188          progress = true;
 189          nir_metadata_preserve(function->impl, nir_metadata_block_index |
 190                                                nir_metadata_dominance);
 191       }
 192    }
 193
 194    return progress;
 195 }