src/compiler/nir/nir_opt_rematerialize_compares.c

   1 /*
   2  * Copyright © 2019 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "nir.h"
  25 #include "nir/nir_builder.h"
  26 #include "nir_constant_expressions.h"
  27 #include "nir_control_flow.h"
  28 #include "nir_loop_analyze.h"
  29
  30 static bool
  31 is_two_src_comparison(const nir_alu_instr *instr)
  32 {
  33    switch (instr->op) {
  34    case nir_op_flt:
  35    case nir_op_flt32:
  36    case nir_op_fge:
  37    case nir_op_fge32:
  38    case nir_op_feq:
  39    case nir_op_feq32:
  40    case nir_op_fneu:
  41    case nir_op_fneu32:
  42    case nir_op_ilt:
  43    case nir_op_ilt32:
  44    case nir_op_ult:
  45    case nir_op_ult32:
  46    case nir_op_ige:
  47    case nir_op_ige32:
  48    case nir_op_uge:
  49    case nir_op_uge32:
  50    case nir_op_ieq:
  51    case nir_op_ieq32:
  52    case nir_op_ine:
  53    case nir_op_ine32:
  54       return true;
  55    default:
  56       return false;
  57    }
  58 }
  59
  60 static bool
  61 all_srcs_are_ssa(const nir_alu_instr *instr)
  62 {
  63    for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
  64       if (!instr->src[i].src.is_ssa)
  65          return false;
  66    }
  67
  68    return true;
  69 }
  70
  71
  72 static bool
  73 all_uses_are_bcsel(const nir_alu_instr *instr)
  74 {
  75    if (!instr->dest.dest.is_ssa)
  76       return false;
  77
  78    nir_foreach_use(use, &instr->dest.dest.ssa) {
  79       if (use->parent_instr->type != nir_instr_type_alu)
  80          return false;
  81
  82       nir_alu_instr *const alu = nir_instr_as_alu(use->parent_instr);
  83       if (alu->op != nir_op_bcsel &&
  84           alu->op != nir_op_b32csel)
  85          return false;
  86
  87       /* Not only must the result be used by a bcsel, but it must be used as
  88        * the first source (the condition).
  89        */
  90       if (alu->src[0].src.ssa != &instr->dest.dest.ssa)
  91          return false;
  92    }
  93
  94    return true;
  95 }
  96
  97 static bool
  98 nir_opt_rematerialize_compares_impl(nir_shader *shader, nir_function_impl *impl)
  99 {
 100    bool progress = false;
 101
 102    nir_foreach_block(block, impl) {
 103       nir_foreach_instr(instr, block) {
 104          if (instr->type != nir_instr_type_alu)
 105             continue;
 106
 107          nir_alu_instr *const alu = nir_instr_as_alu(instr);
 108          if (!is_two_src_comparison(alu))
 109             continue;
 110
 111          if (!all_srcs_are_ssa(alu))
 112             continue;
 113
 114          if (!all_uses_are_bcsel(alu))
 115             continue;
 116
 117          /* At this point it is known that alu is a comparison instruction
 118           * that is only used by nir_op_bcsel and possibly by if-statements
 119           * (though the latter has not been explicitly checked).
 120           *
 121           * Iterate through each use of the comparison.  For every use (or use
 122           * by an if-statement) that is in a different block, emit a copy of
 123           * the comparison.  Care must be taken here.  The original
 124           * instruction must be duplicated only once in each block because CSE
 125           * cannot be run after this pass.
 126           */
 127          nir_foreach_use_safe(use, &alu->dest.dest.ssa) {
 128             nir_instr *const use_instr = use->parent_instr;
 129
 130             /* If the use is in the same block as the def, don't
 131              * rematerialize.
 132              */
 133             if (use_instr->block == alu->instr.block)
 134                continue;
 135
 136             nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
 137
 138             nir_instr_insert_before(use_instr, &clone->instr);
 139
 140             nir_alu_instr *const use_alu = nir_instr_as_alu(use_instr);
 141             for (unsigned i = 0; i < nir_op_infos[use_alu->op].num_inputs; i++) {
 142                if (use_alu->src[i].src.ssa == &alu->dest.dest.ssa) {
 143                   nir_instr_rewrite_src(&use_alu->instr,
 144                                         &use_alu->src[i].src,
 145                                         nir_src_for_ssa(&clone->dest.dest.ssa));
 146                   progress = true;
 147                }
 148             }
 149          }
 150
 151          nir_foreach_if_use_safe(use, &alu->dest.dest.ssa) {
 152             nir_if *const if_stmt = use->parent_if;
 153
 154             nir_block *const prev_block =
 155                nir_cf_node_as_block(nir_cf_node_prev(&if_stmt->cf_node));
 156
 157             /* If the compare is from the previous block, don't
 158              * rematerialize.
 159              */
 160             if (prev_block == alu->instr.block)
 161                continue;
 162
 163             nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
 164
 165             nir_instr_insert_after_block(prev_block, &clone->instr);
 166
 167             nir_if_rewrite_condition(if_stmt,
 168                                      nir_src_for_ssa(&clone->dest.dest.ssa));
 169             progress = true;
 170          }
 171       }
 172    }
 173
 174    if (progress) {
 175       nir_metadata_preserve(impl, nir_metadata_block_index |
 176                                   nir_metadata_dominance);
 177    } else {
 178       nir_metadata_preserve(impl, nir_metadata_all);
 179    }
 180
 181    return progress;
 182 }
 183
 184 bool
 185 nir_opt_rematerialize_compares(nir_shader *shader)
 186 {
 187    bool progress = false;
 188
 189    nir_foreach_function(function, shader) {
 190       if (function->impl == NULL)
 191          continue;
 192
 193       progress = nir_opt_rematerialize_compares_impl(shader, function->impl)
 194          || progress;
 195    }
 196
 197    return progress;
 198 }