2 * Copyright © 2020 Valve Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 * Replaces make availability/visible semantics on barriers with
27 * ACCESS_COHERENT on memory loads/stores
31 #include "shader_enums.h"
34 get_intrinsic_info(nir_intrinsic_instr
*intrin
, nir_variable_mode
*mode
,
35 bool *reads
, bool *writes
)
37 switch (intrin
->intrinsic
) {
38 case nir_intrinsic_image_deref_load
:
39 *mode
= nir_src_as_deref(intrin
->src
[0])->mode
;
42 case nir_intrinsic_image_deref_store
:
43 *mode
= nir_src_as_deref(intrin
->src
[0])->mode
;
46 case nir_intrinsic_image_deref_atomic_add
:
47 case nir_intrinsic_image_deref_atomic_umin
:
48 case nir_intrinsic_image_deref_atomic_imin
:
49 case nir_intrinsic_image_deref_atomic_umax
:
50 case nir_intrinsic_image_deref_atomic_imax
:
51 case nir_intrinsic_image_deref_atomic_and
:
52 case nir_intrinsic_image_deref_atomic_or
:
53 case nir_intrinsic_image_deref_atomic_xor
:
54 case nir_intrinsic_image_deref_atomic_exchange
:
55 case nir_intrinsic_image_deref_atomic_comp_swap
:
56 *mode
= nir_src_as_deref(intrin
->src
[0])->mode
;
60 case nir_intrinsic_load_ssbo
:
61 *mode
= nir_var_mem_ssbo
;
64 case nir_intrinsic_store_ssbo
:
65 *mode
= nir_var_mem_ssbo
;
68 case nir_intrinsic_ssbo_atomic_add
:
69 case nir_intrinsic_ssbo_atomic_imin
:
70 case nir_intrinsic_ssbo_atomic_umin
:
71 case nir_intrinsic_ssbo_atomic_imax
:
72 case nir_intrinsic_ssbo_atomic_umax
:
73 case nir_intrinsic_ssbo_atomic_and
:
74 case nir_intrinsic_ssbo_atomic_or
:
75 case nir_intrinsic_ssbo_atomic_xor
:
76 case nir_intrinsic_ssbo_atomic_exchange
:
77 case nir_intrinsic_ssbo_atomic_comp_swap
:
78 *mode
= nir_var_mem_ssbo
;
82 case nir_intrinsic_load_global
:
83 *mode
= nir_var_mem_global
;
86 case nir_intrinsic_store_global
:
87 *mode
= nir_var_mem_global
;
90 case nir_intrinsic_global_atomic_add
:
91 case nir_intrinsic_global_atomic_imin
:
92 case nir_intrinsic_global_atomic_umin
:
93 case nir_intrinsic_global_atomic_imax
:
94 case nir_intrinsic_global_atomic_umax
:
95 case nir_intrinsic_global_atomic_and
:
96 case nir_intrinsic_global_atomic_or
:
97 case nir_intrinsic_global_atomic_xor
:
98 case nir_intrinsic_global_atomic_exchange
:
99 case nir_intrinsic_global_atomic_comp_swap
:
100 *mode
= nir_var_mem_global
;
104 case nir_intrinsic_load_deref
:
105 *mode
= nir_src_as_deref(intrin
->src
[0])->mode
;
108 case nir_intrinsic_store_deref
:
109 *mode
= nir_src_as_deref(intrin
->src
[0])->mode
;
112 case nir_intrinsic_deref_atomic_add
:
113 case nir_intrinsic_deref_atomic_imin
:
114 case nir_intrinsic_deref_atomic_umin
:
115 case nir_intrinsic_deref_atomic_imax
:
116 case nir_intrinsic_deref_atomic_umax
:
117 case nir_intrinsic_deref_atomic_and
:
118 case nir_intrinsic_deref_atomic_or
:
119 case nir_intrinsic_deref_atomic_xor
:
120 case nir_intrinsic_deref_atomic_exchange
:
121 case nir_intrinsic_deref_atomic_comp_swap
:
122 *mode
= nir_src_as_deref(intrin
->src
[0])->mode
;
133 visit_instr(nir_instr
*instr
, uint32_t *cur_modes
, unsigned vis_avail_sem
)
135 if (instr
->type
!= nir_instr_type_intrinsic
)
137 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
139 if (intrin
->intrinsic
== nir_intrinsic_scoped_barrier
&&
140 (nir_intrinsic_memory_semantics(intrin
) & vis_avail_sem
)) {
141 *cur_modes
|= nir_intrinsic_memory_modes(intrin
);
143 unsigned semantics
= nir_intrinsic_memory_semantics(intrin
);
144 nir_intrinsic_set_memory_semantics(
145 intrin
, semantics
& ~vis_avail_sem
);
150 return false; /* early exit */
152 nir_variable_mode mode
;
153 bool reads
= false, writes
= false;
154 if (!get_intrinsic_info(intrin
, &mode
, &reads
, &writes
))
157 if (!reads
&& vis_avail_sem
== NIR_MEMORY_MAKE_VISIBLE
)
159 if (!writes
&& vis_avail_sem
== NIR_MEMORY_MAKE_AVAILABLE
)
162 unsigned access
= nir_intrinsic_access(intrin
);
164 if (access
& (ACCESS_NON_READABLE
| ACCESS_NON_WRITEABLE
| ACCESS_CAN_REORDER
| ACCESS_COHERENT
))
167 if (*cur_modes
& mode
) {
168 nir_intrinsic_set_access(intrin
, access
| ACCESS_COHERENT
);
176 lower_make_visible(nir_cf_node
*cf_node
, uint32_t *cur_modes
)
178 bool progress
= false;
179 switch (cf_node
->type
) {
180 case nir_cf_node_block
: {
181 nir_block
*block
= nir_cf_node_as_block(cf_node
);
182 nir_foreach_instr(instr
, block
)
183 visit_instr(instr
, cur_modes
, NIR_MEMORY_MAKE_VISIBLE
);
186 case nir_cf_node_if
: {
187 nir_if
*nif
= nir_cf_node_as_if(cf_node
);
188 uint32_t cur_modes_then
= *cur_modes
;
189 uint32_t cur_modes_else
= *cur_modes
;
190 foreach_list_typed(nir_cf_node
, if_node
, node
, &nif
->then_list
)
191 progress
|= lower_make_visible(if_node
, &cur_modes_then
);
192 foreach_list_typed(nir_cf_node
, if_node
, node
, &nif
->else_list
)
193 progress
|= lower_make_visible(if_node
, &cur_modes_else
);
194 *cur_modes
|= cur_modes_then
| cur_modes_else
;
197 case nir_cf_node_loop
: {
198 nir_loop
*loop
= nir_cf_node_as_loop(cf_node
);
201 loop_progress
= false;
202 foreach_list_typed(nir_cf_node
, loop_node
, node
, &loop
->body
)
203 loop_progress
|= lower_make_visible(loop_node
, cur_modes
);
204 progress
|= loop_progress
;
205 } while (loop_progress
);
208 case nir_cf_node_function
:
209 unreachable("Invalid cf type");
215 lower_make_available(nir_cf_node
*cf_node
, uint32_t *cur_modes
)
217 bool progress
= false;
218 switch (cf_node
->type
) {
219 case nir_cf_node_block
: {
220 nir_block
*block
= nir_cf_node_as_block(cf_node
);
221 nir_foreach_instr_reverse(instr
, block
)
222 visit_instr(instr
, cur_modes
, NIR_MEMORY_MAKE_AVAILABLE
);
225 case nir_cf_node_if
: {
226 nir_if
*nif
= nir_cf_node_as_if(cf_node
);
227 uint32_t cur_modes_then
= *cur_modes
;
228 uint32_t cur_modes_else
= *cur_modes
;
229 foreach_list_typed_reverse(nir_cf_node
, if_node
, node
, &nif
->then_list
)
230 progress
|= lower_make_available(if_node
, &cur_modes_then
);
231 foreach_list_typed_reverse(nir_cf_node
, if_node
, node
, &nif
->else_list
)
232 progress
|= lower_make_available(if_node
, &cur_modes_else
);
233 *cur_modes
|= cur_modes_then
| cur_modes_else
;
236 case nir_cf_node_loop
: {
237 nir_loop
*loop
= nir_cf_node_as_loop(cf_node
);
240 loop_progress
= false;
241 foreach_list_typed_reverse(nir_cf_node
, loop_node
, node
, &loop
->body
)
242 loop_progress
|= lower_make_available(loop_node
, cur_modes
);
243 progress
|= loop_progress
;
244 } while (loop_progress
);
247 case nir_cf_node_function
:
248 unreachable("Invalid cf type");
254 nir_lower_memory_model(nir_shader
*shader
)
256 bool progress
= false;
258 struct exec_list
*cf_list
= &nir_shader_get_entrypoint(shader
)->body
;
261 foreach_list_typed(nir_cf_node
, cf_node
, node
, cf_list
)
262 progress
|= lower_make_visible(cf_node
, &modes
);
265 foreach_list_typed_reverse(nir_cf_node
, cf_node
, node
, cf_list
)
266 progress
|= lower_make_available(cf_node
, &modes
);