2 * Copyright © 2019 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "nir_builder.h"
28 read_first_invocation(nir_builder
*b
, nir_ssa_def
*x
)
30 nir_intrinsic_instr
*first
=
31 nir_intrinsic_instr_create(b
->shader
,
32 nir_intrinsic_read_first_invocation
);
33 first
->num_components
= x
->num_components
;
34 first
->src
[0] = nir_src_for_ssa(x
);
35 nir_ssa_dest_init(&first
->instr
, &first
->dest
,
36 x
->num_components
, x
->bit_size
, NULL
);
37 nir_builder_instr_insert(b
, &first
->instr
);
38 return &first
->dest
.ssa
;
42 lower_non_uniform_tex_access(nir_builder
*b
, nir_tex_instr
*tex
)
44 if (!tex
->texture_non_uniform
&& !tex
->sampler_non_uniform
)
47 /* We can have at most one texture and one sampler handle */
48 nir_ssa_def
*handles
[2];
49 unsigned handle_count
= 0;
50 for (unsigned i
= 0; i
< tex
->num_srcs
; i
++) {
51 switch (tex
->src
[i
].src_type
) {
52 case nir_tex_src_texture_offset
:
53 case nir_tex_src_texture_handle
:
54 if (!tex
->texture_non_uniform
)
58 case nir_tex_src_sampler_offset
:
59 case nir_tex_src_sampler_handle
:
60 if (!tex
->sampler_non_uniform
)
68 assert(tex
->src
[i
].src
.is_ssa
);
69 assert(tex
->src
[i
].src
.ssa
->num_components
== 1);
70 assert(handle_count
< 2);
71 handles
[handle_count
++] = tex
->src
[i
].src
.ssa
;
74 if (handle_count
== 0)
77 b
->cursor
= nir_instr_remove(&tex
->instr
);
81 nir_ssa_def
*all_equal_first
= nir_imm_true(b
);
82 for (unsigned i
= 0; i
< handle_count
; i
++) {
83 nir_ssa_def
*equal_first
=
84 nir_ieq(b
, read_first_invocation(b
, handles
[i
]), handles
[i
]);
85 all_equal_first
= nir_iand(b
, all_equal_first
, equal_first
);
88 nir_push_if(b
, all_equal_first
);
90 nir_builder_instr_insert(b
, &tex
->instr
);
91 nir_jump(b
, nir_jump_break
);
97 lower_non_uniform_access_intrin(nir_builder
*b
, nir_intrinsic_instr
*intrin
,
100 if (!(nir_intrinsic_access(intrin
) & ACCESS_NON_UNIFORM
))
103 /* If it's constant, it's automatically uniform; don't bother. */
104 if (nir_src_is_const(intrin
->src
[handle_src
]))
107 b
->cursor
= nir_instr_remove(&intrin
->instr
);
111 assert(intrin
->src
[handle_src
].is_ssa
);
112 assert(intrin
->src
[handle_src
].ssa
->num_components
== 1);
113 nir_ssa_def
*handle
= intrin
->src
[handle_src
].ssa
;
115 nir_push_if(b
, nir_ieq(b
, read_first_invocation(b
, handle
), handle
));
117 nir_builder_instr_insert(b
, &intrin
->instr
);
118 nir_jump(b
, nir_jump_break
);
124 nir_lower_non_uniform_access_impl(nir_function_impl
*impl
,
125 enum nir_lower_non_uniform_access_type types
)
127 bool progress
= false;
130 nir_builder_init(&b
, impl
);
132 nir_foreach_block_safe(block
, impl
) {
133 nir_foreach_instr_safe(instr
, block
) {
134 switch (instr
->type
) {
135 case nir_instr_type_tex
: {
136 nir_tex_instr
*tex
= nir_instr_as_tex(instr
);
137 if ((types
& nir_lower_non_uniform_texture_access
) &&
138 lower_non_uniform_tex_access(&b
, tex
))
143 case nir_instr_type_intrinsic
: {
144 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
145 switch (intrin
->intrinsic
) {
146 case nir_intrinsic_load_ubo
:
147 if ((types
& nir_lower_non_uniform_ubo_access
) &&
148 lower_non_uniform_access_intrin(&b
, intrin
, 0))
152 case nir_intrinsic_load_ssbo
:
153 case nir_intrinsic_ssbo_atomic_add
:
154 case nir_intrinsic_ssbo_atomic_imin
:
155 case nir_intrinsic_ssbo_atomic_umin
:
156 case nir_intrinsic_ssbo_atomic_imax
:
157 case nir_intrinsic_ssbo_atomic_umax
:
158 case nir_intrinsic_ssbo_atomic_and
:
159 case nir_intrinsic_ssbo_atomic_or
:
160 case nir_intrinsic_ssbo_atomic_xor
:
161 case nir_intrinsic_ssbo_atomic_exchange
:
162 case nir_intrinsic_ssbo_atomic_comp_swap
:
163 case nir_intrinsic_ssbo_atomic_fadd
:
164 case nir_intrinsic_ssbo_atomic_fmin
:
165 case nir_intrinsic_ssbo_atomic_fmax
:
166 case nir_intrinsic_ssbo_atomic_fcomp_swap
:
167 if ((types
& nir_lower_non_uniform_ssbo_access
) &&
168 lower_non_uniform_access_intrin(&b
, intrin
, 0))
172 case nir_intrinsic_store_ssbo
:
173 /* SSBO Stores put the index in the second source */
174 if ((types
& nir_lower_non_uniform_ssbo_access
) &&
175 lower_non_uniform_access_intrin(&b
, intrin
, 1))
179 case nir_intrinsic_image_load
:
180 case nir_intrinsic_image_store
:
181 case nir_intrinsic_image_atomic_add
:
182 case nir_intrinsic_image_atomic_min
:
183 case nir_intrinsic_image_atomic_max
:
184 case nir_intrinsic_image_atomic_and
:
185 case nir_intrinsic_image_atomic_or
:
186 case nir_intrinsic_image_atomic_xor
:
187 case nir_intrinsic_image_atomic_exchange
:
188 case nir_intrinsic_image_atomic_comp_swap
:
189 case nir_intrinsic_image_atomic_fadd
:
190 case nir_intrinsic_image_size
:
191 case nir_intrinsic_image_samples
:
192 case nir_intrinsic_bindless_image_load
:
193 case nir_intrinsic_bindless_image_store
:
194 case nir_intrinsic_bindless_image_atomic_add
:
195 case nir_intrinsic_bindless_image_atomic_min
:
196 case nir_intrinsic_bindless_image_atomic_max
:
197 case nir_intrinsic_bindless_image_atomic_and
:
198 case nir_intrinsic_bindless_image_atomic_or
:
199 case nir_intrinsic_bindless_image_atomic_xor
:
200 case nir_intrinsic_bindless_image_atomic_exchange
:
201 case nir_intrinsic_bindless_image_atomic_comp_swap
:
202 case nir_intrinsic_bindless_image_atomic_fadd
:
203 case nir_intrinsic_bindless_image_size
:
204 case nir_intrinsic_bindless_image_samples
:
205 if ((types
& nir_lower_non_uniform_image_access
) &&
206 lower_non_uniform_access_intrin(&b
, intrin
, 0))
225 nir_metadata_preserve(impl
, nir_metadata_none
);
231 * Lowers non-uniform resource access by using a loop
233 * This pass lowers non-uniform resource access by using subgroup operations
234 * and a loop. Most hardware requires things like textures and UBO access
235 * operations to happen on a dynamically uniform (or at least subgroup
236 * uniform) resource. This pass allows for non-uniform access by placing the
237 * texture instruction in a loop that looks something like this:
240 * bool tex_eq_first = readFirstInvocationARB(texture) == texture;
241 * bool smp_eq_first = readFirstInvocationARB(sampler) == sampler;
242 * if (tex_eq_first && smp_eq_first) {
243 * res = texture(texture, sampler, ...);
248 * Fortunately, because the instruction is immediately followed by the only
249 * break in the loop, the block containing the instruction dominates the end
250 * of the loop. Therefore, it's safe to move the instruction into the loop
251 * without fixing up SSA in any way.
254 nir_lower_non_uniform_access(nir_shader
*shader
,
255 enum nir_lower_non_uniform_access_type types
)
257 bool progress
= false;
259 nir_foreach_function(function
, shader
) {
260 if (function
->impl
&&
261 nir_lower_non_uniform_access_impl(function
->impl
, types
))