src/compiler/nir/nir_lower_scratch.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 /*
  29  * This lowering pass converts references to variables with loads/stores to
  30  * scratch space based on a few configurable parameters.
  31  */
  32
  33 #include "nir.h"
  34 #include "nir_builder.h"
  35 #include "nir_deref.h"
  36
  37 static void
  38 lower_load_store(nir_builder *b,
  39                  nir_intrinsic_instr *intrin,
  40                  glsl_type_size_align_func size_align)
  41 {
  42    b->cursor = nir_before_instr(&intrin->instr);
  43
  44    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
  45    nir_variable *var = nir_deref_instr_get_variable(deref);
  46
  47    nir_ssa_def *offset =
  48       nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align),
  49                       var->data.location);
  50
  51    unsigned align, UNUSED size;
  52    size_align(deref->type, &size, &align);
  53
  54    if (intrin->intrinsic == nir_intrinsic_load_deref) {
  55       nir_intrinsic_instr *load =
  56          nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch);
  57       load->num_components = intrin->num_components;
  58       load->src[0] = nir_src_for_ssa(offset);
  59       nir_intrinsic_set_align(load, align, 0);
  60       unsigned bit_size = intrin->dest.ssa.bit_size;
  61       nir_ssa_dest_init(&load->instr, &load->dest,
  62                         intrin->dest.ssa.num_components,
  63                         bit_size == 1 ? 32 : bit_size, NULL);
  64       nir_builder_instr_insert(b, &load->instr);
  65
  66       nir_ssa_def *value = &load->dest.ssa;
  67       if (bit_size == 1)
  68          value = nir_b2b1(b, value);
  69
  70       nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
  71                                nir_src_for_ssa(value));
  72    } else {
  73       assert(intrin->intrinsic == nir_intrinsic_store_deref);
  74
  75       assert(intrin->src[1].is_ssa);
  76       nir_ssa_def *value = intrin->src[1].ssa;
  77       if (value->bit_size == 1)
  78          value = nir_b2b32(b, value);
  79
  80       nir_intrinsic_instr *store =
  81          nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch);
  82       store->num_components = intrin->num_components;
  83       store->src[0] = nir_src_for_ssa(value);
  84       store->src[1] = nir_src_for_ssa(offset);
  85       nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin));
  86       nir_intrinsic_set_align(store, align, 0);
  87       nir_builder_instr_insert(b, &store->instr);
  88    }
  89
  90    nir_instr_remove(&intrin->instr);
  91    nir_deref_instr_remove_if_unused(deref);
  92 }
  93
  94 bool
  95 nir_lower_vars_to_scratch(nir_shader *shader,
  96                           nir_variable_mode modes,
  97                           int size_threshold,
  98                           glsl_type_size_align_func size_align)
  99 {
 100    /* First, we walk the instructions and flag any variables we want to lower
 101     * by removing them from their respective list and setting the mode to 0.
 102     */
 103    nir_foreach_function(function, shader) {
 104       nir_foreach_block(block, function->impl) {
 105          nir_foreach_instr(instr, block) {
 106             if (instr->type != nir_instr_type_intrinsic)
 107                continue;
 108
 109             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 110             if (intrin->intrinsic != nir_intrinsic_load_deref &&
 111                 intrin->intrinsic != nir_intrinsic_store_deref)
 112                continue;
 113
 114             nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
 115             if (!(deref->mode & modes))
 116                continue;
 117
 118             if (!nir_deref_instr_has_indirect(nir_src_as_deref(intrin->src[0])))
 119                continue;
 120
 121             nir_variable *var = nir_deref_instr_get_variable(deref);
 122
 123             /* We set var->mode to 0 to indicate that a variable will be moved
 124              * to scratch.  Don't assign a scratch location twice.
 125              */
 126             if (var->data.mode == 0)
 127                continue;
 128
 129             unsigned var_size, var_align;
 130             size_align(var->type, &var_size, &var_align);
 131             if (var_size <= size_threshold)
 132                continue;
 133
 134             /* Remove it from its list */
 135             exec_node_remove(&var->node);
 136             /* Invalid mode used to flag "moving to scratch" */
 137             var->data.mode = 0;
 138
 139             var->data.location = ALIGN_POT(shader->scratch_size, var_align);
 140             shader->scratch_size = var->data.location + var_size;
 141          }
 142       }
 143    }
 144
 145    bool progress = false;
 146    nir_foreach_function(function, shader) {
 147       if (!function->impl)
 148          continue;
 149
 150       nir_builder build;
 151       nir_builder_init(&build, function->impl);
 152
 153       bool impl_progress = false;
 154       nir_foreach_block(block, function->impl) {
 155          nir_foreach_instr_safe(instr, block) {
 156             if (instr->type != nir_instr_type_intrinsic)
 157                continue;
 158
 159             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 160             if (intrin->intrinsic != nir_intrinsic_load_deref &&
 161                 intrin->intrinsic != nir_intrinsic_store_deref)
 162                continue;
 163
 164             nir_variable *var = nir_intrinsic_get_var(intrin, 0);
 165             /* Variables flagged for lowering above have mode == 0 */
 166             if (!var || var->data.mode)
 167                continue;
 168
 169             lower_load_store(&build, intrin, size_align);
 170             impl_progress = true;
 171          }
 172       }
 173
 174       if (impl_progress) {
 175          progress = true;
 176          nir_metadata_preserve(function->impl, nir_metadata_block_index |
 177                                                nir_metadata_dominance);
 178       } else {
 179          nir_metadata_preserve(function->impl, nir_metadata_all);
 180       }
 181    }
 182
 183    return progress;
 184 }