src/compiler/nir/nir_lower_memory_model.c

   1 /*
   2  * Copyright © 2020 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 /*
  26  * Replaces make availability/visible semantics on barriers with
  27  * ACCESS_COHERENT on memory loads/stores
  28  */
  29
  30 #include "nir/nir.h"
  31 #include "shader_enums.h"
  32
  33 static bool
  34 get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *mode,
  35                    bool *reads, bool *writes)
  36 {
  37    switch (intrin->intrinsic) {
  38    case nir_intrinsic_image_deref_load:
  39       *mode = nir_src_as_deref(intrin->src[0])->mode;
  40       *reads = true;
  41       break;
  42    case nir_intrinsic_image_deref_store:
  43       *mode = nir_src_as_deref(intrin->src[0])->mode;
  44       *writes = true;
  45       break;
  46    case nir_intrinsic_image_deref_atomic_add:
  47    case nir_intrinsic_image_deref_atomic_umin:
  48    case nir_intrinsic_image_deref_atomic_imin:
  49    case nir_intrinsic_image_deref_atomic_umax:
  50    case nir_intrinsic_image_deref_atomic_imax:
  51    case nir_intrinsic_image_deref_atomic_and:
  52    case nir_intrinsic_image_deref_atomic_or:
  53    case nir_intrinsic_image_deref_atomic_xor:
  54    case nir_intrinsic_image_deref_atomic_exchange:
  55    case nir_intrinsic_image_deref_atomic_comp_swap:
  56       *mode = nir_src_as_deref(intrin->src[0])->mode;
  57       *reads = true;
  58       *writes = true;
  59       break;
  60    case nir_intrinsic_load_ssbo:
  61       *mode = nir_var_mem_ssbo;
  62       *reads = true;
  63       break;
  64    case nir_intrinsic_store_ssbo:
  65       *mode = nir_var_mem_ssbo;
  66       *writes = true;
  67       break;
  68    case nir_intrinsic_ssbo_atomic_add:
  69    case nir_intrinsic_ssbo_atomic_imin:
  70    case nir_intrinsic_ssbo_atomic_umin:
  71    case nir_intrinsic_ssbo_atomic_imax:
  72    case nir_intrinsic_ssbo_atomic_umax:
  73    case nir_intrinsic_ssbo_atomic_and:
  74    case nir_intrinsic_ssbo_atomic_or:
  75    case nir_intrinsic_ssbo_atomic_xor:
  76    case nir_intrinsic_ssbo_atomic_exchange:
  77    case nir_intrinsic_ssbo_atomic_comp_swap:
  78       *mode = nir_var_mem_ssbo;
  79       *reads = true;
  80       *writes = true;
  81       break;
  82    case nir_intrinsic_load_global:
  83       *mode = nir_var_mem_global;
  84       *reads = true;
  85       break;
  86    case nir_intrinsic_store_global:
  87       *mode = nir_var_mem_global;
  88       *writes = true;
  89       break;
  90    case nir_intrinsic_global_atomic_add:
  91    case nir_intrinsic_global_atomic_imin:
  92    case nir_intrinsic_global_atomic_umin:
  93    case nir_intrinsic_global_atomic_imax:
  94    case nir_intrinsic_global_atomic_umax:
  95    case nir_intrinsic_global_atomic_and:
  96    case nir_intrinsic_global_atomic_or:
  97    case nir_intrinsic_global_atomic_xor:
  98    case nir_intrinsic_global_atomic_exchange:
  99    case nir_intrinsic_global_atomic_comp_swap:
 100       *mode = nir_var_mem_global;
 101       *reads = true;
 102       *writes = true;
 103       break;
 104    case nir_intrinsic_load_deref:
 105       *mode = nir_src_as_deref(intrin->src[0])->mode;
 106       *reads = true;
 107       break;
 108    case nir_intrinsic_store_deref:
 109       *mode = nir_src_as_deref(intrin->src[0])->mode;
 110       *writes = true;
 111       break;
 112    case nir_intrinsic_deref_atomic_add:
 113    case nir_intrinsic_deref_atomic_imin:
 114    case nir_intrinsic_deref_atomic_umin:
 115    case nir_intrinsic_deref_atomic_imax:
 116    case nir_intrinsic_deref_atomic_umax:
 117    case nir_intrinsic_deref_atomic_and:
 118    case nir_intrinsic_deref_atomic_or:
 119    case nir_intrinsic_deref_atomic_xor:
 120    case nir_intrinsic_deref_atomic_exchange:
 121    case nir_intrinsic_deref_atomic_comp_swap:
 122       *mode = nir_src_as_deref(intrin->src[0])->mode;
 123       *reads = true;
 124       *writes = true;
 125       break;
 126    default:
 127       return false;
 128    }
 129    return true;
 130 }
 131
 132 static bool
 133 visit_instr(nir_instr *instr, uint32_t *cur_modes, unsigned vis_avail_sem)
 134 {
 135    if (instr->type != nir_instr_type_intrinsic)
 136       return false;
 137    nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 138
 139    if (intrin->intrinsic == nir_intrinsic_scoped_barrier &&
 140        (nir_intrinsic_memory_semantics(intrin) & vis_avail_sem)) {
 141       *cur_modes |= nir_intrinsic_memory_modes(intrin);
 142
 143       unsigned semantics = nir_intrinsic_memory_semantics(intrin);
 144       nir_intrinsic_set_memory_semantics(
 145          intrin, semantics & ~vis_avail_sem);
 146       return true;
 147    }
 148
 149    if (!*cur_modes)
 150       return false; /* early exit */
 151
 152    nir_variable_mode mode;
 153    bool reads = false, writes = false;
 154    if (!get_intrinsic_info(intrin, &mode, &reads, &writes))
 155       return false;
 156
 157    if (!reads && vis_avail_sem == NIR_MEMORY_MAKE_VISIBLE)
 158       return false;
 159    if (!writes && vis_avail_sem == NIR_MEMORY_MAKE_AVAILABLE)
 160       return false;
 161
 162    unsigned access = nir_intrinsic_access(intrin);
 163
 164    if (access & (ACCESS_NON_READABLE | ACCESS_NON_WRITEABLE | ACCESS_CAN_REORDER | ACCESS_COHERENT))
 165       return false;
 166
 167    if (*cur_modes & mode) {
 168       nir_intrinsic_set_access(intrin, access | ACCESS_COHERENT);
 169       return true;
 170    }
 171
 172    return false;
 173 }
 174
 175 static bool
 176 lower_make_visible(nir_cf_node *cf_node, uint32_t *cur_modes)
 177 {
 178    bool progress = false;
 179    switch (cf_node->type) {
 180    case nir_cf_node_block: {
 181       nir_block *block = nir_cf_node_as_block(cf_node);
 182       nir_foreach_instr(instr, block)
 183          visit_instr(instr, cur_modes, NIR_MEMORY_MAKE_VISIBLE);
 184       break;
 185    }
 186    case nir_cf_node_if: {
 187       nir_if *nif = nir_cf_node_as_if(cf_node);
 188       uint32_t cur_modes_then = *cur_modes;
 189       uint32_t cur_modes_else = *cur_modes;
 190       foreach_list_typed(nir_cf_node, if_node, node, &nif->then_list)
 191          progress |= lower_make_visible(if_node, &cur_modes_then);
 192       foreach_list_typed(nir_cf_node, if_node, node, &nif->else_list)
 193          progress |= lower_make_visible(if_node, &cur_modes_else);
 194       *cur_modes |= cur_modes_then | cur_modes_else;
 195       break;
 196    }
 197    case nir_cf_node_loop: {
 198       nir_loop *loop = nir_cf_node_as_loop(cf_node);
 199       bool loop_progress;
 200       do {
 201          loop_progress = false;
 202          foreach_list_typed(nir_cf_node, loop_node, node, &loop->body)
 203             loop_progress |= lower_make_visible(loop_node, cur_modes);
 204          progress |= loop_progress;
 205       } while (loop_progress);
 206       break;
 207    }
 208    case nir_cf_node_function:
 209       unreachable("Invalid cf type");
 210    }
 211    return progress;
 212 }
 213
 214 static bool
 215 lower_make_available(nir_cf_node *cf_node, uint32_t *cur_modes)
 216 {
 217    bool progress = false;
 218    switch (cf_node->type) {
 219    case nir_cf_node_block: {
 220       nir_block *block = nir_cf_node_as_block(cf_node);
 221       nir_foreach_instr_reverse(instr, block)
 222          visit_instr(instr, cur_modes, NIR_MEMORY_MAKE_AVAILABLE);
 223       break;
 224    }
 225    case nir_cf_node_if: {
 226       nir_if *nif = nir_cf_node_as_if(cf_node);
 227       uint32_t cur_modes_then = *cur_modes;
 228       uint32_t cur_modes_else = *cur_modes;
 229       foreach_list_typed_reverse(nir_cf_node, if_node, node, &nif->then_list)
 230          progress |= lower_make_available(if_node, &cur_modes_then);
 231       foreach_list_typed_reverse(nir_cf_node, if_node, node, &nif->else_list)
 232          progress |= lower_make_available(if_node, &cur_modes_else);
 233       *cur_modes |= cur_modes_then | cur_modes_else;
 234       break;
 235    }
 236    case nir_cf_node_loop: {
 237       nir_loop *loop = nir_cf_node_as_loop(cf_node);
 238       bool loop_progress;
 239       do {
 240          loop_progress = false;
 241          foreach_list_typed_reverse(nir_cf_node, loop_node, node, &loop->body)
 242             loop_progress |= lower_make_available(loop_node, cur_modes);
 243          progress |= loop_progress;
 244       } while (loop_progress);
 245       break;
 246    }
 247    case nir_cf_node_function:
 248       unreachable("Invalid cf type");
 249    }
 250    return progress;
 251 }
 252
 253 bool
 254 nir_lower_memory_model(nir_shader *shader)
 255 {
 256    bool progress = false;
 257
 258    struct exec_list *cf_list = &nir_shader_get_entrypoint(shader)->body;
 259
 260    uint32_t modes = 0;
 261    foreach_list_typed(nir_cf_node, cf_node, node, cf_list)
 262       progress |= lower_make_visible(cf_node, &modes);
 263
 264    modes = 0;
 265    foreach_list_typed_reverse(nir_cf_node, cf_node, node, cf_list)
 266       progress |= lower_make_available(cf_node, &modes);
 267
 268    return progress;
 269 }