src/compiler/glsl/gl_nir_lower_atomics.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Connor Abbott (cwabbott0@gmail.com)
  25  *
  26  */
  27
  28 #include "compiler/nir/nir.h"
  29 #include "compiler/nir/nir_builder.h"
  30 #include "gl_nir.h"
  31 #include "ir_uniform.h"
  32 #include "main/config.h"
  33 #include "main/mtypes.h"
  34 #include <assert.h>
  35
  36 /*
  37  * replace atomic counter intrinsics that use a variable with intrinsics
  38  * that directly store the buffer index and byte offset
  39  */
  40
  41 static bool
  42 lower_deref_instr(nir_builder *b, nir_intrinsic_instr *instr,
  43                   const struct gl_shader_program *shader_program,
  44                   nir_shader *shader, bool use_binding_as_idx)
  45 {
  46    nir_intrinsic_op op;
  47    switch (instr->intrinsic) {
  48    case nir_intrinsic_atomic_counter_read_deref:
  49       op = nir_intrinsic_atomic_counter_read;
  50       break;
  51
  52    case nir_intrinsic_atomic_counter_inc_deref:
  53       op = nir_intrinsic_atomic_counter_inc;
  54       break;
  55
  56    case nir_intrinsic_atomic_counter_pre_dec_deref:
  57       op = nir_intrinsic_atomic_counter_pre_dec;
  58       break;
  59
  60    case nir_intrinsic_atomic_counter_post_dec_deref:
  61       op = nir_intrinsic_atomic_counter_post_dec;
  62       break;
  63
  64    case nir_intrinsic_atomic_counter_add_deref:
  65       op = nir_intrinsic_atomic_counter_add;
  66       break;
  67
  68    case nir_intrinsic_atomic_counter_min_deref:
  69       op = nir_intrinsic_atomic_counter_min;
  70       break;
  71
  72    case nir_intrinsic_atomic_counter_max_deref:
  73       op = nir_intrinsic_atomic_counter_max;
  74       break;
  75
  76    case nir_intrinsic_atomic_counter_and_deref:
  77       op = nir_intrinsic_atomic_counter_and;
  78       break;
  79
  80    case nir_intrinsic_atomic_counter_or_deref:
  81       op = nir_intrinsic_atomic_counter_or;
  82       break;
  83
  84    case nir_intrinsic_atomic_counter_xor_deref:
  85       op = nir_intrinsic_atomic_counter_xor;
  86       break;
  87
  88    case nir_intrinsic_atomic_counter_exchange_deref:
  89       op = nir_intrinsic_atomic_counter_exchange;
  90       break;
  91
  92    case nir_intrinsic_atomic_counter_comp_swap_deref:
  93       op = nir_intrinsic_atomic_counter_comp_swap;
  94       break;
  95
  96    default:
  97       return false;
  98    }
  99
 100    nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
 101    nir_variable *var = nir_deref_instr_get_variable(deref);
 102
 103    if (var->data.mode != nir_var_uniform &&
 104        var->data.mode != nir_var_mem_ssbo &&
 105        var->data.mode != nir_var_mem_shared)
 106       return false; /* atomics passed as function arguments can't be lowered */
 107
 108    const unsigned uniform_loc = var->data.location;
 109    const unsigned idx = use_binding_as_idx ? var->data.binding :
 110       shader_program->data->UniformStorage[uniform_loc].opaque[shader->info.stage].index;
 111
 112    b->cursor = nir_before_instr(&instr->instr);
 113
 114    nir_ssa_def *offset = nir_imm_int(b, var->data.offset);
 115    for (nir_deref_instr *d = deref; d->deref_type != nir_deref_type_var;
 116         d = nir_deref_instr_parent(d)) {
 117       assert(d->deref_type == nir_deref_type_array);
 118       assert(d->arr.index.is_ssa);
 119
 120       unsigned array_stride = ATOMIC_COUNTER_SIZE;
 121       if (glsl_type_is_array(d->type))
 122          array_stride *= glsl_get_aoa_size(d->type);
 123
 124       offset = nir_iadd(b, offset, nir_imul(b, d->arr.index.ssa,
 125                                             nir_imm_int(b, array_stride)));
 126    }
 127
 128    /* Since the first source is a deref and the first source in the lowered
 129     * instruction is the offset, we can just swap it out and change the
 130     * opcode.
 131     */
 132    instr->intrinsic = op;
 133    nir_instr_rewrite_src(&instr->instr, &instr->src[0],
 134                          nir_src_for_ssa(offset));
 135    nir_intrinsic_set_base(instr, idx);
 136
 137    nir_deref_instr_remove_if_unused(deref);
 138
 139    return true;
 140 }
 141
 142 bool
 143 gl_nir_lower_atomics(nir_shader *shader,
 144                      const struct gl_shader_program *shader_program,
 145                      bool use_binding_as_idx)
 146 {
 147    bool progress = false;
 148
 149    nir_foreach_function(function, shader) {
 150       if (!function->impl)
 151          continue;
 152
 153       bool impl_progress = false;
 154
 155       nir_builder build;
 156       nir_builder_init(&build, function->impl);
 157
 158       nir_foreach_block(block, function->impl) {
 159          nir_foreach_instr_safe(instr, block) {
 160             if (instr->type != nir_instr_type_intrinsic)
 161                continue;
 162
 163             impl_progress |= lower_deref_instr(&build,
 164                                                nir_instr_as_intrinsic(instr),
 165                                                shader_program, shader,
 166                                                use_binding_as_idx);
 167          }
 168       }
 169
 170       if (impl_progress) {
 171          nir_metadata_preserve(function->impl, nir_metadata_block_index |
 172                                                nir_metadata_dominance);
 173          progress = true;
 174       }
 175    }
 176
 177    return progress;
 178 }