2 * Copyright 2020 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 #include "si_shader_internal.h"
30 * Return a value that is equal to the given i32 \p index if it lies in [0,num)
31 * or an undefined value in the same interval otherwise.
33 static LLVMValueRef
si_llvm_bound_index(struct si_shader_context
*ctx
, LLVMValueRef index
,
36 LLVMBuilderRef builder
= ctx
->ac
.builder
;
37 LLVMValueRef c_max
= LLVMConstInt(ctx
->ac
.i32
, num
- 1, 0);
40 if (util_is_power_of_two_or_zero(num
)) {
41 index
= LLVMBuildAnd(builder
, index
, c_max
, "");
43 /* In theory, this MAX pattern should result in code that is
44 * as good as the bit-wise AND above.
46 * In practice, LLVM generates worse code (at the time of
47 * writing), because its value tracking is not strong enough.
49 cc
= LLVMBuildICmp(builder
, LLVMIntULE
, index
, c_max
, "");
50 index
= LLVMBuildSelect(builder
, cc
, index
, c_max
, "");
56 static LLVMValueRef
load_const_buffer_desc_fast_path(struct si_shader_context
*ctx
)
58 LLVMValueRef ptr
= ac_get_arg(&ctx
->ac
, ctx
->const_and_shader_buffers
);
59 struct si_shader_selector
*sel
= ctx
->shader
->selector
;
61 /* Do the bounds checking with a descriptor, because
62 * doing computation and manual bounds checking of 64-bit
63 * addresses generates horrible VALU code with very high
64 * VGPR usage and very low SIMD occupancy.
66 ptr
= LLVMBuildPtrToInt(ctx
->ac
.builder
, ptr
, ctx
->ac
.intptr
, "");
68 LLVMValueRef desc0
, desc1
;
70 desc1
= LLVMConstInt(ctx
->ac
.i32
, S_008F04_BASE_ADDRESS_HI(ctx
->screen
->info
.address32_hi
), 0);
72 uint32_t rsrc3
= S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X
) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y
) |
73 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z
) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W
);
75 if (ctx
->screen
->info
.chip_class
>= GFX10
)
76 rsrc3
|= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT
) |
77 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW
) | S_008F0C_RESOURCE_LEVEL(1);
79 rsrc3
|= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT
) |
80 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32
);
82 LLVMValueRef desc_elems
[] = {desc0
, desc1
,
83 LLVMConstInt(ctx
->ac
.i32
, sel
->info
.constbuf0_num_slots
* 16, 0),
84 LLVMConstInt(ctx
->ac
.i32
, rsrc3
, false)};
86 return ac_build_gather_values(&ctx
->ac
, desc_elems
, 4);
89 static LLVMValueRef
load_ubo(struct ac_shader_abi
*abi
, LLVMValueRef index
)
91 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
92 struct si_shader_selector
*sel
= ctx
->shader
->selector
;
94 LLVMValueRef ptr
= ac_get_arg(&ctx
->ac
, ctx
->const_and_shader_buffers
);
96 if (sel
->info
.const_buffers_declared
== 1 && sel
->info
.shader_buffers_declared
== 0) {
97 return load_const_buffer_desc_fast_path(ctx
);
100 index
= si_llvm_bound_index(ctx
, index
, ctx
->num_const_buffers
);
102 LLVMBuildAdd(ctx
->ac
.builder
, index
, LLVMConstInt(ctx
->ac
.i32
, SI_NUM_SHADER_BUFFERS
, 0), "");
104 return ac_build_load_to_sgpr(&ctx
->ac
, ptr
, index
);
107 static LLVMValueRef
load_ssbo(struct ac_shader_abi
*abi
, LLVMValueRef index
, bool write
)
109 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
110 LLVMValueRef rsrc_ptr
= ac_get_arg(&ctx
->ac
, ctx
->const_and_shader_buffers
);
112 index
= si_llvm_bound_index(ctx
, index
, ctx
->num_shader_buffers
);
113 index
= LLVMBuildSub(ctx
->ac
.builder
, LLVMConstInt(ctx
->ac
.i32
, SI_NUM_SHADER_BUFFERS
- 1, 0),
116 return ac_build_load_to_sgpr(&ctx
->ac
, rsrc_ptr
, index
);
120 * Given a 256-bit resource descriptor, force the DCC enable bit to off.
122 * At least on Tonga, executing image stores on images with DCC enabled and
123 * non-trivial can eventually lead to lockups. This can occur when an
124 * application binds an image as read-only but then uses a shader that writes
125 * to it. The OpenGL spec allows almost arbitrarily bad behavior (including
126 * program termination) in this case, but it doesn't cost much to be a bit
127 * nicer: disabling DCC in the shader still leads to undefined results but
130 static LLVMValueRef
force_dcc_off(struct si_shader_context
*ctx
, LLVMValueRef rsrc
)
132 if (ctx
->screen
->info
.chip_class
<= GFX7
) {
135 LLVMValueRef i32_6
= LLVMConstInt(ctx
->ac
.i32
, 6, 0);
136 LLVMValueRef i32_C
= LLVMConstInt(ctx
->ac
.i32
, C_008F28_COMPRESSION_EN
, 0);
139 tmp
= LLVMBuildExtractElement(ctx
->ac
.builder
, rsrc
, i32_6
, "");
140 tmp
= LLVMBuildAnd(ctx
->ac
.builder
, tmp
, i32_C
, "");
141 return LLVMBuildInsertElement(ctx
->ac
.builder
, rsrc
, tmp
, i32_6
, "");
145 /* AC_DESC_FMASK is handled exactly like AC_DESC_IMAGE. The caller should
146 * adjust "index" to point to FMASK. */
147 static LLVMValueRef
si_load_image_desc(struct si_shader_context
*ctx
, LLVMValueRef list
,
148 LLVMValueRef index
, enum ac_descriptor_type desc_type
,
149 bool uses_store
, bool bindless
)
151 LLVMBuilderRef builder
= ctx
->ac
.builder
;
154 if (desc_type
== AC_DESC_BUFFER
) {
155 index
= ac_build_imad(&ctx
->ac
, index
, LLVMConstInt(ctx
->ac
.i32
, 2, 0), ctx
->ac
.i32_1
);
156 list
= LLVMBuildPointerCast(builder
, list
, ac_array_in_const32_addr_space(ctx
->ac
.v4i32
), "");
158 assert(desc_type
== AC_DESC_IMAGE
|| desc_type
== AC_DESC_FMASK
);
162 rsrc
= ac_build_load_to_sgpr_uint_wraparound(&ctx
->ac
, list
, index
);
164 rsrc
= ac_build_load_to_sgpr(&ctx
->ac
, list
, index
);
166 if (desc_type
== AC_DESC_IMAGE
&& uses_store
)
167 rsrc
= force_dcc_off(ctx
, rsrc
);
172 * Load an image view, fmask view. or sampler state descriptor.
174 static LLVMValueRef
si_load_sampler_desc(struct si_shader_context
*ctx
, LLVMValueRef list
,
175 LLVMValueRef index
, enum ac_descriptor_type type
)
177 LLVMBuilderRef builder
= ctx
->ac
.builder
;
181 /* The image is at [0:7]. */
182 index
= LLVMBuildMul(builder
, index
, LLVMConstInt(ctx
->ac
.i32
, 2, 0), "");
185 /* The buffer is in [4:7]. */
186 index
= ac_build_imad(&ctx
->ac
, index
, LLVMConstInt(ctx
->ac
.i32
, 4, 0), ctx
->ac
.i32_1
);
187 list
= LLVMBuildPointerCast(builder
, list
, ac_array_in_const32_addr_space(ctx
->ac
.v4i32
), "");
190 /* The FMASK is at [8:15]. */
191 index
= ac_build_imad(&ctx
->ac
, index
, LLVMConstInt(ctx
->ac
.i32
, 2, 0), ctx
->ac
.i32_1
);
193 case AC_DESC_SAMPLER
:
194 /* The sampler state is at [12:15]. */
195 index
= ac_build_imad(&ctx
->ac
, index
, LLVMConstInt(ctx
->ac
.i32
, 4, 0),
196 LLVMConstInt(ctx
->ac
.i32
, 3, 0));
197 list
= LLVMBuildPointerCast(builder
, list
, ac_array_in_const32_addr_space(ctx
->ac
.v4i32
), "");
199 case AC_DESC_PLANE_0
:
200 case AC_DESC_PLANE_1
:
201 case AC_DESC_PLANE_2
:
202 /* Only used for the multiplane image support for Vulkan. Should
203 * never be reached in radeonsi.
205 unreachable("Plane descriptor requested in radeonsi.");
208 return ac_build_load_to_sgpr(&ctx
->ac
, list
, index
);
211 static LLVMValueRef
si_nir_load_sampler_desc(struct ac_shader_abi
*abi
, unsigned descriptor_set
,
212 unsigned base_index
, unsigned constant_index
,
213 LLVMValueRef dynamic_index
,
214 enum ac_descriptor_type desc_type
, bool image
,
215 bool write
, bool bindless
)
217 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
218 LLVMBuilderRef builder
= ctx
->ac
.builder
;
219 unsigned const_index
= base_index
+ constant_index
;
221 assert(!descriptor_set
);
222 assert(desc_type
<= AC_DESC_BUFFER
);
225 LLVMValueRef list
= ac_get_arg(&ctx
->ac
, ctx
->bindless_samplers_and_images
);
227 /* dynamic_index is the bindless handle */
229 /* Bindless image descriptors use 16-dword slots. */
231 LLVMBuildMul(ctx
->ac
.builder
, dynamic_index
, LLVMConstInt(ctx
->ac
.i64
, 2, 0), "");
232 /* FMASK is right after the image. */
233 if (desc_type
== AC_DESC_FMASK
) {
234 dynamic_index
= LLVMBuildAdd(ctx
->ac
.builder
, dynamic_index
, ctx
->ac
.i32_1
, "");
237 return si_load_image_desc(ctx
, list
, dynamic_index
, desc_type
, write
, true);
240 /* Since bindless handle arithmetic can contain an unsigned integer
241 * wraparound and si_load_sampler_desc assumes there isn't any,
242 * use GEP without "inbounds" (inside ac_build_pointer_add)
243 * to prevent incorrect code generation and hangs.
246 LLVMBuildMul(ctx
->ac
.builder
, dynamic_index
, LLVMConstInt(ctx
->ac
.i64
, 2, 0), "");
247 list
= ac_build_pointer_add(&ctx
->ac
, list
, dynamic_index
);
248 return si_load_sampler_desc(ctx
, list
, ctx
->ac
.i32_0
, desc_type
);
251 unsigned num_slots
= image
? ctx
->num_images
: ctx
->num_samplers
;
252 assert(const_index
< num_slots
|| dynamic_index
);
254 LLVMValueRef list
= ac_get_arg(&ctx
->ac
, ctx
->samplers_and_images
);
255 LLVMValueRef index
= LLVMConstInt(ctx
->ac
.i32
, const_index
, false);
258 index
= LLVMBuildAdd(builder
, index
, dynamic_index
, "");
260 /* From the GL_ARB_shader_image_load_store extension spec:
262 * If a shader performs an image load, store, or atomic
263 * operation using an image variable declared as an array,
264 * and if the index used to select an individual element is
265 * negative or greater than or equal to the size of the
266 * array, the results of the operation are undefined but may
267 * not lead to termination.
269 index
= si_llvm_bound_index(ctx
, index
, num_slots
);
273 /* FMASKs are separate from images. */
274 if (desc_type
== AC_DESC_FMASK
) {
276 LLVMBuildAdd(ctx
->ac
.builder
, index
, LLVMConstInt(ctx
->ac
.i32
, SI_NUM_IMAGES
, 0), "");
278 index
= LLVMBuildSub(ctx
->ac
.builder
, LLVMConstInt(ctx
->ac
.i32
, SI_NUM_IMAGE_SLOTS
- 1, 0),
280 return si_load_image_desc(ctx
, list
, index
, desc_type
, write
, false);
283 index
= LLVMBuildAdd(ctx
->ac
.builder
, index
,
284 LLVMConstInt(ctx
->ac
.i32
, SI_NUM_IMAGE_SLOTS
/ 2, 0), "");
285 return si_load_sampler_desc(ctx
, list
, index
, desc_type
);
288 void si_llvm_init_resource_callbacks(struct si_shader_context
*ctx
)
290 ctx
->abi
.load_ubo
= load_ubo
;
291 ctx
->abi
.load_ssbo
= load_ssbo
;
292 ctx
->abi
.load_sampler_desc
= si_nir_load_sampler_desc
;