2 * Copyright 2020 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 #include "si_shader_internal.h"
29 LLVMValueRef
si_get_sample_id(struct si_shader_context
*ctx
)
31 return si_unpack_param(ctx
, ctx
->args
.ancillary
, 8, 4);
34 static LLVMValueRef
load_sample_mask_in(struct ac_shader_abi
*abi
)
36 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
37 return ac_to_integer(&ctx
->ac
, ac_get_arg(&ctx
->ac
, ctx
->args
.sample_coverage
));
40 static LLVMValueRef
load_sample_position(struct ac_shader_abi
*abi
, LLVMValueRef sample_id
)
42 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
43 LLVMValueRef desc
= ac_get_arg(&ctx
->ac
, ctx
->rw_buffers
);
44 LLVMValueRef buf_index
= LLVMConstInt(ctx
->ac
.i32
, SI_PS_CONST_SAMPLE_POSITIONS
, 0);
45 LLVMValueRef resource
= ac_build_load_to_sgpr(&ctx
->ac
, desc
, buf_index
);
47 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
48 LLVMValueRef offset0
=
49 LLVMBuildMul(ctx
->ac
.builder
, sample_id
, LLVMConstInt(ctx
->ac
.i32
, 8, 0), "");
50 LLVMValueRef offset1
=
51 LLVMBuildAdd(ctx
->ac
.builder
, offset0
, LLVMConstInt(ctx
->ac
.i32
, 4, 0), "");
53 LLVMValueRef pos
[4] = {si_buffer_load_const(ctx
, resource
, offset0
),
54 si_buffer_load_const(ctx
, resource
, offset1
),
55 LLVMConstReal(ctx
->ac
.f32
, 0), LLVMConstReal(ctx
->ac
.f32
, 0)};
57 return ac_build_gather_values(&ctx
->ac
, pos
, 4);
60 static LLVMValueRef
si_nir_emit_fbfetch(struct ac_shader_abi
*abi
)
62 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
63 struct ac_image_args args
= {};
64 LLVMValueRef ptr
, image
, fmask
;
66 /* Ignore src0, because KHR_blend_func_extended disallows multiple render
70 /* Load the image descriptor. */
71 STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0
% 2 == 0);
72 ptr
= ac_get_arg(&ctx
->ac
, ctx
->rw_buffers
);
74 LLVMBuildPointerCast(ctx
->ac
.builder
, ptr
, ac_array_in_const32_addr_space(ctx
->ac
.v8i32
), "");
76 ac_build_load_to_sgpr(&ctx
->ac
, ptr
, LLVMConstInt(ctx
->ac
.i32
, SI_PS_IMAGE_COLORBUF0
/ 2, 0));
80 args
.coords
[chan
++] = si_unpack_param(ctx
, ctx
->pos_fixed_pt
, 0, 16);
82 if (!ctx
->shader
->key
.mono
.u
.ps
.fbfetch_is_1D
)
83 args
.coords
[chan
++] = si_unpack_param(ctx
, ctx
->pos_fixed_pt
, 16, 16);
85 /* Get the current render target layer index. */
86 if (ctx
->shader
->key
.mono
.u
.ps
.fbfetch_layered
)
87 args
.coords
[chan
++] = si_unpack_param(ctx
, ctx
->args
.ancillary
, 16, 11);
89 if (ctx
->shader
->key
.mono
.u
.ps
.fbfetch_msaa
)
90 args
.coords
[chan
++] = si_get_sample_id(ctx
);
92 if (ctx
->shader
->key
.mono
.u
.ps
.fbfetch_msaa
&& !(ctx
->screen
->debug_flags
& DBG(NO_FMASK
))) {
93 fmask
= ac_build_load_to_sgpr(&ctx
->ac
, ptr
,
94 LLVMConstInt(ctx
->ac
.i32
, SI_PS_IMAGE_COLORBUF0_FMASK
/ 2, 0));
96 ac_apply_fmask_to_sample(&ctx
->ac
, fmask
, args
.coords
,
97 ctx
->shader
->key
.mono
.u
.ps
.fbfetch_layered
);
100 args
.opcode
= ac_image_load
;
101 args
.resource
= image
;
103 args
.attributes
= AC_FUNC_ATTR_READNONE
;
105 if (ctx
->shader
->key
.mono
.u
.ps
.fbfetch_msaa
)
107 ctx
->shader
->key
.mono
.u
.ps
.fbfetch_layered
? ac_image_2darraymsaa
: ac_image_2dmsaa
;
108 else if (ctx
->shader
->key
.mono
.u
.ps
.fbfetch_is_1D
)
109 args
.dim
= ctx
->shader
->key
.mono
.u
.ps
.fbfetch_layered
? ac_image_1darray
: ac_image_1d
;
111 args
.dim
= ctx
->shader
->key
.mono
.u
.ps
.fbfetch_layered
? ac_image_2darray
: ac_image_2d
;
113 return ac_build_image_opcode(&ctx
->ac
, &args
);
116 static LLVMValueRef
si_build_fs_interp(struct si_shader_context
*ctx
, unsigned attr_index
,
117 unsigned chan
, LLVMValueRef prim_mask
, LLVMValueRef i
,
121 return ac_build_fs_interp(&ctx
->ac
, LLVMConstInt(ctx
->ac
.i32
, chan
, 0),
122 LLVMConstInt(ctx
->ac
.i32
, attr_index
, 0), prim_mask
, i
, j
);
124 return ac_build_fs_interp_mov(&ctx
->ac
, LLVMConstInt(ctx
->ac
.i32
, 2, 0), /* P0 */
125 LLVMConstInt(ctx
->ac
.i32
, chan
, 0),
126 LLVMConstInt(ctx
->ac
.i32
, attr_index
, 0), prim_mask
);
130 * Interpolate a fragment shader input.
133 * @param input_index index of the input in hardware
134 * @param semantic_index semantic index
135 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
136 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
137 * @param interp_param interpolation weights (i,j)
138 * @param prim_mask SI_PARAM_PRIM_MASK
139 * @param face SI_PARAM_FRONT_FACE
140 * @param result the return value (4 components)
142 static void interp_fs_color(struct si_shader_context
*ctx
, unsigned input_index
,
143 unsigned semantic_index
, unsigned num_interp_inputs
,
144 unsigned colors_read_mask
, LLVMValueRef interp_param
,
145 LLVMValueRef prim_mask
, LLVMValueRef face
, LLVMValueRef result
[4])
147 LLVMValueRef i
= NULL
, j
= NULL
;
150 /* fs.constant returns the param from the middle vertex, so it's not
151 * really useful for flat shading. It's meant to be used for custom
152 * interpolation (but the intrinsic can't fetch from the other two
155 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
156 * to do the right thing. The only reason we use fs.constant is that
157 * fs.interp cannot be used on integers, because they can be equal
160 * When interp is false we will use fs.constant or for newer llvm,
163 bool interp
= interp_param
!= NULL
;
167 LLVMBuildBitCast(ctx
->ac
.builder
, interp_param
, ctx
->ac
.v2f32
, "");
169 i
= LLVMBuildExtractElement(ctx
->ac
.builder
, interp_param
, ctx
->ac
.i32_0
, "");
170 j
= LLVMBuildExtractElement(ctx
->ac
.builder
, interp_param
, ctx
->ac
.i32_1
, "");
173 if (ctx
->shader
->key
.part
.ps
.prolog
.color_two_side
) {
174 LLVMValueRef is_face_positive
;
176 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
177 * otherwise it's at offset "num_inputs".
179 unsigned back_attr_offset
= num_interp_inputs
;
180 if (semantic_index
== 1 && colors_read_mask
& 0xf)
181 back_attr_offset
+= 1;
183 is_face_positive
= LLVMBuildICmp(ctx
->ac
.builder
, LLVMIntNE
, face
, ctx
->ac
.i32_0
, "");
185 for (chan
= 0; chan
< 4; chan
++) {
186 LLVMValueRef front
, back
;
188 front
= si_build_fs_interp(ctx
, input_index
, chan
, prim_mask
, i
, j
);
189 back
= si_build_fs_interp(ctx
, back_attr_offset
, chan
, prim_mask
, i
, j
);
191 result
[chan
] = LLVMBuildSelect(ctx
->ac
.builder
, is_face_positive
, front
, back
, "");
194 for (chan
= 0; chan
< 4; chan
++) {
195 result
[chan
] = si_build_fs_interp(ctx
, input_index
, chan
, prim_mask
, i
, j
);
200 static void si_alpha_test(struct si_shader_context
*ctx
, LLVMValueRef alpha
)
202 if (ctx
->shader
->key
.part
.ps
.epilog
.alpha_func
!= PIPE_FUNC_NEVER
) {
203 static LLVMRealPredicate cond_map
[PIPE_FUNC_ALWAYS
+ 1] = {
204 [PIPE_FUNC_LESS
] = LLVMRealOLT
, [PIPE_FUNC_EQUAL
] = LLVMRealOEQ
,
205 [PIPE_FUNC_LEQUAL
] = LLVMRealOLE
, [PIPE_FUNC_GREATER
] = LLVMRealOGT
,
206 [PIPE_FUNC_NOTEQUAL
] = LLVMRealONE
, [PIPE_FUNC_GEQUAL
] = LLVMRealOGE
,
208 LLVMRealPredicate cond
= cond_map
[ctx
->shader
->key
.part
.ps
.epilog
.alpha_func
];
211 LLVMValueRef alpha_ref
= LLVMGetParam(ctx
->main_fn
, SI_PARAM_ALPHA_REF
);
212 LLVMValueRef alpha_pass
= LLVMBuildFCmp(ctx
->ac
.builder
, cond
, alpha
, alpha_ref
, "");
213 ac_build_kill_if_false(&ctx
->ac
, alpha_pass
);
215 ac_build_kill_if_false(&ctx
->ac
, ctx
->ac
.i1false
);
219 static LLVMValueRef
si_scale_alpha_by_sample_mask(struct si_shader_context
*ctx
, LLVMValueRef alpha
,
220 unsigned samplemask_param
)
222 LLVMValueRef coverage
;
224 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
225 coverage
= LLVMGetParam(ctx
->main_fn
, samplemask_param
);
226 coverage
= ac_to_integer(&ctx
->ac
, coverage
);
228 coverage
= ac_build_intrinsic(&ctx
->ac
, "llvm.ctpop.i32", ctx
->ac
.i32
, &coverage
, 1,
229 AC_FUNC_ATTR_READNONE
);
231 coverage
= LLVMBuildUIToFP(ctx
->ac
.builder
, coverage
, ctx
->ac
.f32
, "");
233 coverage
= LLVMBuildFMul(ctx
->ac
.builder
, coverage
,
234 LLVMConstReal(ctx
->ac
.f32
, 1.0 / SI_NUM_SMOOTH_AA_SAMPLES
), "");
236 return LLVMBuildFMul(ctx
->ac
.builder
, alpha
, coverage
, "");
239 struct si_ps_exports
{
241 struct ac_export_args args
[10];
244 static void si_export_mrt_z(struct si_shader_context
*ctx
, LLVMValueRef depth
, LLVMValueRef stencil
,
245 LLVMValueRef samplemask
, struct si_ps_exports
*exp
)
247 struct ac_export_args args
;
249 ac_export_mrt_z(&ctx
->ac
, depth
, stencil
, samplemask
, &args
);
251 memcpy(&exp
->args
[exp
->num
++], &args
, sizeof(args
));
254 /* Initialize arguments for the shader export intrinsic */
255 static void si_llvm_init_ps_export_args(struct si_shader_context
*ctx
, LLVMValueRef
*values
,
256 unsigned cbuf
, unsigned compacted_mrt_index
,
257 struct ac_export_args
*args
)
259 const struct si_shader_key
*key
= &ctx
->shader
->key
;
260 unsigned col_formats
= key
->part
.ps
.epilog
.spi_shader_col_format
;
261 LLVMValueRef f32undef
= LLVMGetUndef(ctx
->ac
.f32
);
262 unsigned spi_shader_col_format
;
264 bool is_int8
, is_int10
;
266 assert(cbuf
>= 0 && cbuf
< 8);
268 spi_shader_col_format
= (col_formats
>> (cbuf
* 4)) & 0xf;
269 is_int8
= (key
->part
.ps
.epilog
.color_is_int8
>> cbuf
) & 0x1;
270 is_int10
= (key
->part
.ps
.epilog
.color_is_int10
>> cbuf
) & 0x1;
272 /* Default is 0xf. Adjusted below depending on the format. */
273 args
->enabled_channels
= 0xf; /* writemask */
275 /* Specify whether the EXEC mask represents the valid mask */
276 args
->valid_mask
= 0;
278 /* Specify whether this is the last export */
281 /* Specify the target we are exporting */
282 args
->target
= V_008DFC_SQ_EXP_MRT
+ compacted_mrt_index
;
285 args
->out
[0] = f32undef
;
286 args
->out
[1] = f32undef
;
287 args
->out
[2] = f32undef
;
288 args
->out
[3] = f32undef
;
290 LLVMValueRef (*packf
)(struct ac_llvm_context
* ctx
, LLVMValueRef args
[2]) = NULL
;
291 LLVMValueRef (*packi
)(struct ac_llvm_context
* ctx
, LLVMValueRef args
[2], unsigned bits
,
294 switch (spi_shader_col_format
) {
295 case V_028714_SPI_SHADER_ZERO
:
296 args
->enabled_channels
= 0; /* writemask */
297 args
->target
= V_008DFC_SQ_EXP_NULL
;
300 case V_028714_SPI_SHADER_32_R
:
301 args
->enabled_channels
= 1; /* writemask */
302 args
->out
[0] = values
[0];
305 case V_028714_SPI_SHADER_32_GR
:
306 args
->enabled_channels
= 0x3; /* writemask */
307 args
->out
[0] = values
[0];
308 args
->out
[1] = values
[1];
311 case V_028714_SPI_SHADER_32_AR
:
312 if (ctx
->screen
->info
.chip_class
>= GFX10
) {
313 args
->enabled_channels
= 0x3; /* writemask */
314 args
->out
[0] = values
[0];
315 args
->out
[1] = values
[3];
317 args
->enabled_channels
= 0x9; /* writemask */
318 args
->out
[0] = values
[0];
319 args
->out
[3] = values
[3];
323 case V_028714_SPI_SHADER_FP16_ABGR
:
324 packf
= ac_build_cvt_pkrtz_f16
;
327 case V_028714_SPI_SHADER_UNORM16_ABGR
:
328 packf
= ac_build_cvt_pknorm_u16
;
331 case V_028714_SPI_SHADER_SNORM16_ABGR
:
332 packf
= ac_build_cvt_pknorm_i16
;
335 case V_028714_SPI_SHADER_UINT16_ABGR
:
336 packi
= ac_build_cvt_pk_u16
;
339 case V_028714_SPI_SHADER_SINT16_ABGR
:
340 packi
= ac_build_cvt_pk_i16
;
343 case V_028714_SPI_SHADER_32_ABGR
:
344 memcpy(&args
->out
[0], values
, sizeof(values
[0]) * 4);
348 /* Pack f16 or norm_i16/u16. */
350 for (chan
= 0; chan
< 2; chan
++) {
351 LLVMValueRef pack_args
[2] = {values
[2 * chan
], values
[2 * chan
+ 1]};
354 packed
= packf(&ctx
->ac
, pack_args
);
355 args
->out
[chan
] = ac_to_float(&ctx
->ac
, packed
);
357 args
->compr
= 1; /* COMPR flag */
361 for (chan
= 0; chan
< 2; chan
++) {
362 LLVMValueRef pack_args
[2] = {ac_to_integer(&ctx
->ac
, values
[2 * chan
]),
363 ac_to_integer(&ctx
->ac
, values
[2 * chan
+ 1])};
366 packed
= packi(&ctx
->ac
, pack_args
, is_int8
? 8 : is_int10
? 10 : 16, chan
== 1);
367 args
->out
[chan
] = ac_to_float(&ctx
->ac
, packed
);
369 args
->compr
= 1; /* COMPR flag */
373 static bool si_export_mrt_color(struct si_shader_context
*ctx
, LLVMValueRef
*color
, unsigned index
,
374 unsigned compacted_mrt_index
, unsigned samplemask_param
,
375 bool is_last
, struct si_ps_exports
*exp
)
380 if (ctx
->shader
->key
.part
.ps
.epilog
.clamp_color
)
381 for (i
= 0; i
< 4; i
++)
382 color
[i
] = ac_build_clamp(&ctx
->ac
, color
[i
]);
385 if (ctx
->shader
->key
.part
.ps
.epilog
.alpha_to_one
)
386 color
[3] = ctx
->ac
.f32_1
;
389 if (index
== 0 && ctx
->shader
->key
.part
.ps
.epilog
.alpha_func
!= PIPE_FUNC_ALWAYS
)
390 si_alpha_test(ctx
, color
[3]);
392 /* Line & polygon smoothing */
393 if (ctx
->shader
->key
.part
.ps
.epilog
.poly_line_smoothing
)
394 color
[3] = si_scale_alpha_by_sample_mask(ctx
, color
[3], samplemask_param
);
396 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
397 if (ctx
->shader
->key
.part
.ps
.epilog
.last_cbuf
> 0) {
398 struct ac_export_args args
[8];
401 assert(compacted_mrt_index
== 0);
403 /* Get the export arguments, also find out what the last one is. */
404 for (c
= 0; c
<= ctx
->shader
->key
.part
.ps
.epilog
.last_cbuf
; c
++) {
405 si_llvm_init_ps_export_args(ctx
, color
, c
, compacted_mrt_index
, &args
[c
]);
406 if (args
[c
].enabled_channels
) {
407 compacted_mrt_index
++;
414 /* Emit all exports. */
415 for (c
= 0; c
<= ctx
->shader
->key
.part
.ps
.epilog
.last_cbuf
; c
++) {
416 if (is_last
&& last
== c
) {
417 args
[c
].valid_mask
= 1; /* whether the EXEC mask is valid */
418 args
[c
].done
= 1; /* DONE bit */
419 } else if (!args
[c
].enabled_channels
)
420 continue; /* unnecessary NULL export */
422 memcpy(&exp
->args
[exp
->num
++], &args
[c
], sizeof(args
[c
]));
425 struct ac_export_args args
;
428 si_llvm_init_ps_export_args(ctx
, color
, index
, compacted_mrt_index
, &args
);
430 args
.valid_mask
= 1; /* whether the EXEC mask is valid */
431 args
.done
= 1; /* DONE bit */
432 } else if (!args
.enabled_channels
)
433 return false; /* unnecessary NULL export */
435 memcpy(&exp
->args
[exp
->num
++], &args
, sizeof(args
));
440 static void si_emit_ps_exports(struct si_shader_context
*ctx
, struct si_ps_exports
*exp
)
442 for (unsigned i
= 0; i
< exp
->num
; i
++)
443 ac_build_export(&ctx
->ac
, &exp
->args
[i
]);
447 * Return PS outputs in this order:
449 * v[0:3] = color0.xyzw
450 * v[4:7] = color1.xyzw
455 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
457 * The alpha-ref SGPR is returned via its original location.
459 static void si_llvm_return_fs_outputs(struct ac_shader_abi
*abi
, unsigned max_outputs
,
462 struct si_shader_context
*ctx
= si_shader_context_from_abi(abi
);
463 struct si_shader
*shader
= ctx
->shader
;
464 struct si_shader_info
*info
= &shader
->selector
->info
;
465 LLVMBuilderRef builder
= ctx
->ac
.builder
;
466 unsigned i
, j
, first_vgpr
, vgpr
;
468 LLVMValueRef color
[8][4] = {};
469 LLVMValueRef depth
= NULL
, stencil
= NULL
, samplemask
= NULL
;
472 if (ctx
->postponed_kill
)
473 ac_build_kill_if_false(&ctx
->ac
, LLVMBuildLoad(builder
, ctx
->postponed_kill
, ""));
475 /* Read the output values. */
476 for (i
= 0; i
< info
->num_outputs
; i
++) {
477 unsigned semantic
= info
->output_semantic
[i
];
480 case FRAG_RESULT_DEPTH
:
481 depth
= LLVMBuildLoad(builder
, addrs
[4 * i
+ 0], "");
483 case FRAG_RESULT_STENCIL
:
484 stencil
= LLVMBuildLoad(builder
, addrs
[4 * i
+ 0], "");
486 case FRAG_RESULT_SAMPLE_MASK
:
487 samplemask
= LLVMBuildLoad(builder
, addrs
[4 * i
+ 0], "");
490 if (semantic
>= FRAG_RESULT_DATA0
&& semantic
<= FRAG_RESULT_DATA7
) {
491 unsigned index
= semantic
- FRAG_RESULT_DATA0
;
493 for (j
= 0; j
< 4; j
++) {
494 LLVMValueRef ptr
= addrs
[4 * i
+ j
];
495 LLVMValueRef result
= LLVMBuildLoad(builder
, ptr
, "");
496 color
[index
][j
] = result
;
499 fprintf(stderr
, "Warning: Unhandled fs output type:%d\n", semantic
);
505 /* Fill the return structure. */
506 ret
= ctx
->return_value
;
509 ret
= LLVMBuildInsertValue(
510 builder
, ret
, ac_to_integer(&ctx
->ac
, LLVMGetParam(ctx
->main_fn
, SI_PARAM_ALPHA_REF
)),
511 SI_SGPR_ALPHA_REF
, "");
514 first_vgpr
= vgpr
= SI_SGPR_ALPHA_REF
+ 1;
515 for (i
= 0; i
< ARRAY_SIZE(color
); i
++) {
519 for (j
= 0; j
< 4; j
++)
520 ret
= LLVMBuildInsertValue(builder
, ret
, color
[i
][j
], vgpr
++, "");
523 ret
= LLVMBuildInsertValue(builder
, ret
, depth
, vgpr
++, "");
525 ret
= LLVMBuildInsertValue(builder
, ret
, stencil
, vgpr
++, "");
527 ret
= LLVMBuildInsertValue(builder
, ret
, samplemask
, vgpr
++, "");
529 /* Add the input sample mask for smoothing at the end. */
530 if (vgpr
< first_vgpr
+ PS_EPILOG_SAMPLEMASK_MIN_LOC
)
531 vgpr
= first_vgpr
+ PS_EPILOG_SAMPLEMASK_MIN_LOC
;
532 ret
= LLVMBuildInsertValue(builder
, ret
, LLVMGetParam(ctx
->main_fn
, SI_PARAM_SAMPLE_COVERAGE
),
535 ctx
->return_value
= ret
;
538 static void si_llvm_emit_polygon_stipple(struct si_shader_context
*ctx
,
539 LLVMValueRef param_rw_buffers
,
540 struct ac_arg param_pos_fixed_pt
)
542 LLVMBuilderRef builder
= ctx
->ac
.builder
;
543 LLVMValueRef slot
, desc
, offset
, row
, bit
, address
[2];
545 /* Use the fixed-point gl_FragCoord input.
546 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
547 * per coordinate to get the repeating effect.
549 address
[0] = si_unpack_param(ctx
, param_pos_fixed_pt
, 0, 5);
550 address
[1] = si_unpack_param(ctx
, param_pos_fixed_pt
, 16, 5);
552 /* Load the buffer descriptor. */
553 slot
= LLVMConstInt(ctx
->ac
.i32
, SI_PS_CONST_POLY_STIPPLE
, 0);
554 desc
= ac_build_load_to_sgpr(&ctx
->ac
, param_rw_buffers
, slot
);
556 /* The stipple pattern is 32x32, each row has 32 bits. */
557 offset
= LLVMBuildMul(builder
, address
[1], LLVMConstInt(ctx
->ac
.i32
, 4, 0), "");
558 row
= si_buffer_load_const(ctx
, desc
, offset
);
559 row
= ac_to_integer(&ctx
->ac
, row
);
560 bit
= LLVMBuildLShr(builder
, row
, address
[0], "");
561 bit
= LLVMBuildTrunc(builder
, bit
, ctx
->ac
.i1
, "");
562 ac_build_kill_if_false(&ctx
->ac
, bit
);
566 * Build the pixel shader prolog function. This handles:
567 * - two-side color selection and interpolation
568 * - overriding interpolation parameters for the API PS
569 * - polygon stippling
571 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
572 * overriden by other states. (e.g. per-sample interpolation)
573 * Interpolated colors are stored after the preloaded VGPRs.
575 void si_llvm_build_ps_prolog(struct si_shader_context
*ctx
, union si_shader_part_key
*key
)
577 LLVMValueRef ret
, func
;
578 int num_returns
, i
, num_color_channels
;
580 memset(&ctx
->args
, 0, sizeof(ctx
->args
));
582 /* Declare inputs. */
583 LLVMTypeRef return_types
[AC_MAX_ARGS
];
585 num_color_channels
= util_bitcount(key
->ps_prolog
.colors_read
);
586 assert(key
->ps_prolog
.num_input_sgprs
+ key
->ps_prolog
.num_input_vgprs
+ num_color_channels
<=
588 for (i
= 0; i
< key
->ps_prolog
.num_input_sgprs
; i
++) {
589 ac_add_arg(&ctx
->args
, AC_ARG_SGPR
, 1, AC_ARG_INT
, NULL
);
590 return_types
[num_returns
++] = ctx
->ac
.i32
;
593 struct ac_arg pos_fixed_pt
;
594 struct ac_arg ancillary
;
595 struct ac_arg param_sample_mask
;
596 for (i
= 0; i
< key
->ps_prolog
.num_input_vgprs
; i
++) {
597 struct ac_arg
*arg
= NULL
;
598 if (i
== key
->ps_prolog
.ancillary_vgpr_index
) {
600 } else if (i
== key
->ps_prolog
.ancillary_vgpr_index
+ 1) {
601 arg
= ¶m_sample_mask
;
602 } else if (i
== key
->ps_prolog
.num_input_vgprs
- 1) {
603 /* POS_FIXED_PT is always last. */
606 ac_add_arg(&ctx
->args
, AC_ARG_VGPR
, 1, AC_ARG_FLOAT
, arg
);
607 return_types
[num_returns
++] = ctx
->ac
.f32
;
610 /* Declare outputs (same as inputs + add colors if needed) */
611 for (i
= 0; i
< num_color_channels
; i
++)
612 return_types
[num_returns
++] = ctx
->ac
.f32
;
614 /* Create the function. */
615 si_llvm_create_func(ctx
, "ps_prolog", return_types
, num_returns
, 0);
618 /* Copy inputs to outputs. This should be no-op, as the registers match,
619 * but it will prevent the compiler from overwriting them unintentionally.
621 ret
= ctx
->return_value
;
622 for (i
= 0; i
< ctx
->args
.arg_count
; i
++) {
623 LLVMValueRef p
= LLVMGetParam(func
, i
);
624 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, p
, i
, "");
627 /* Polygon stippling. */
628 if (key
->ps_prolog
.states
.poly_stipple
) {
629 LLVMValueRef list
= si_prolog_get_rw_buffers(ctx
);
631 si_llvm_emit_polygon_stipple(ctx
, list
, pos_fixed_pt
);
634 if (key
->ps_prolog
.states
.bc_optimize_for_persp
||
635 key
->ps_prolog
.states
.bc_optimize_for_linear
) {
636 unsigned i
, base
= key
->ps_prolog
.num_input_sgprs
;
637 LLVMValueRef center
[2], centroid
[2], tmp
, bc_optimize
;
639 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
640 * The hw doesn't compute CENTROID if the whole wave only
641 * contains fully-covered quads.
643 * PRIM_MASK is after user SGPRs.
645 bc_optimize
= LLVMGetParam(func
, SI_PS_NUM_USER_SGPR
);
647 LLVMBuildLShr(ctx
->ac
.builder
, bc_optimize
, LLVMConstInt(ctx
->ac
.i32
, 31, 0), "");
648 bc_optimize
= LLVMBuildTrunc(ctx
->ac
.builder
, bc_optimize
, ctx
->ac
.i1
, "");
650 if (key
->ps_prolog
.states
.bc_optimize_for_persp
) {
651 /* Read PERSP_CENTER. */
652 for (i
= 0; i
< 2; i
++)
653 center
[i
] = LLVMGetParam(func
, base
+ 2 + i
);
654 /* Read PERSP_CENTROID. */
655 for (i
= 0; i
< 2; i
++)
656 centroid
[i
] = LLVMGetParam(func
, base
+ 4 + i
);
657 /* Select PERSP_CENTROID. */
658 for (i
= 0; i
< 2; i
++) {
659 tmp
= LLVMBuildSelect(ctx
->ac
.builder
, bc_optimize
, center
[i
], centroid
[i
], "");
660 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, tmp
, base
+ 4 + i
, "");
663 if (key
->ps_prolog
.states
.bc_optimize_for_linear
) {
664 /* Read LINEAR_CENTER. */
665 for (i
= 0; i
< 2; i
++)
666 center
[i
] = LLVMGetParam(func
, base
+ 8 + i
);
667 /* Read LINEAR_CENTROID. */
668 for (i
= 0; i
< 2; i
++)
669 centroid
[i
] = LLVMGetParam(func
, base
+ 10 + i
);
670 /* Select LINEAR_CENTROID. */
671 for (i
= 0; i
< 2; i
++) {
672 tmp
= LLVMBuildSelect(ctx
->ac
.builder
, bc_optimize
, center
[i
], centroid
[i
], "");
673 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, tmp
, base
+ 10 + i
, "");
678 /* Force per-sample interpolation. */
679 if (key
->ps_prolog
.states
.force_persp_sample_interp
) {
680 unsigned i
, base
= key
->ps_prolog
.num_input_sgprs
;
681 LLVMValueRef persp_sample
[2];
683 /* Read PERSP_SAMPLE. */
684 for (i
= 0; i
< 2; i
++)
685 persp_sample
[i
] = LLVMGetParam(func
, base
+ i
);
686 /* Overwrite PERSP_CENTER. */
687 for (i
= 0; i
< 2; i
++)
688 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, persp_sample
[i
], base
+ 2 + i
, "");
689 /* Overwrite PERSP_CENTROID. */
690 for (i
= 0; i
< 2; i
++)
691 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, persp_sample
[i
], base
+ 4 + i
, "");
693 if (key
->ps_prolog
.states
.force_linear_sample_interp
) {
694 unsigned i
, base
= key
->ps_prolog
.num_input_sgprs
;
695 LLVMValueRef linear_sample
[2];
697 /* Read LINEAR_SAMPLE. */
698 for (i
= 0; i
< 2; i
++)
699 linear_sample
[i
] = LLVMGetParam(func
, base
+ 6 + i
);
700 /* Overwrite LINEAR_CENTER. */
701 for (i
= 0; i
< 2; i
++)
702 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, linear_sample
[i
], base
+ 8 + i
, "");
703 /* Overwrite LINEAR_CENTROID. */
704 for (i
= 0; i
< 2; i
++)
705 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, linear_sample
[i
], base
+ 10 + i
, "");
708 /* Force center interpolation. */
709 if (key
->ps_prolog
.states
.force_persp_center_interp
) {
710 unsigned i
, base
= key
->ps_prolog
.num_input_sgprs
;
711 LLVMValueRef persp_center
[2];
713 /* Read PERSP_CENTER. */
714 for (i
= 0; i
< 2; i
++)
715 persp_center
[i
] = LLVMGetParam(func
, base
+ 2 + i
);
716 /* Overwrite PERSP_SAMPLE. */
717 for (i
= 0; i
< 2; i
++)
718 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, persp_center
[i
], base
+ i
, "");
719 /* Overwrite PERSP_CENTROID. */
720 for (i
= 0; i
< 2; i
++)
721 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, persp_center
[i
], base
+ 4 + i
, "");
723 if (key
->ps_prolog
.states
.force_linear_center_interp
) {
724 unsigned i
, base
= key
->ps_prolog
.num_input_sgprs
;
725 LLVMValueRef linear_center
[2];
727 /* Read LINEAR_CENTER. */
728 for (i
= 0; i
< 2; i
++)
729 linear_center
[i
] = LLVMGetParam(func
, base
+ 8 + i
);
730 /* Overwrite LINEAR_SAMPLE. */
731 for (i
= 0; i
< 2; i
++)
732 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, linear_center
[i
], base
+ 6 + i
, "");
733 /* Overwrite LINEAR_CENTROID. */
734 for (i
= 0; i
< 2; i
++)
735 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, linear_center
[i
], base
+ 10 + i
, "");
738 /* Interpolate colors. */
739 unsigned color_out_idx
= 0;
740 for (i
= 0; i
< 2; i
++) {
741 unsigned writemask
= (key
->ps_prolog
.colors_read
>> (i
* 4)) & 0xf;
742 unsigned face_vgpr
= key
->ps_prolog
.num_input_sgprs
+ key
->ps_prolog
.face_vgpr_index
;
743 LLVMValueRef interp
[2], color
[4];
744 LLVMValueRef interp_ij
= NULL
, prim_mask
= NULL
, face
= NULL
;
749 /* If the interpolation qualifier is not CONSTANT (-1). */
750 if (key
->ps_prolog
.color_interp_vgpr_index
[i
] != -1) {
751 unsigned interp_vgpr
=
752 key
->ps_prolog
.num_input_sgprs
+ key
->ps_prolog
.color_interp_vgpr_index
[i
];
754 /* Get the (i,j) updated by bc_optimize handling. */
755 interp
[0] = LLVMBuildExtractValue(ctx
->ac
.builder
, ret
, interp_vgpr
, "");
756 interp
[1] = LLVMBuildExtractValue(ctx
->ac
.builder
, ret
, interp_vgpr
+ 1, "");
757 interp_ij
= ac_build_gather_values(&ctx
->ac
, interp
, 2);
760 /* Use the absolute location of the input. */
761 prim_mask
= LLVMGetParam(func
, SI_PS_NUM_USER_SGPR
);
763 if (key
->ps_prolog
.states
.color_two_side
) {
764 face
= LLVMGetParam(func
, face_vgpr
);
765 face
= ac_to_integer(&ctx
->ac
, face
);
768 interp_fs_color(ctx
, key
->ps_prolog
.color_attr_index
[i
], i
, key
->ps_prolog
.num_interp_inputs
,
769 key
->ps_prolog
.colors_read
, interp_ij
, prim_mask
, face
, color
);
772 unsigned chan
= u_bit_scan(&writemask
);
773 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, color
[chan
],
774 ctx
->args
.arg_count
+ color_out_idx
++, "");
778 /* Section 15.2.2 (Shader Inputs) of the OpenGL 4.5 (Core Profile) spec
781 * "When per-sample shading is active due to the use of a fragment
782 * input qualified by sample or due to the use of the gl_SampleID
783 * or gl_SamplePosition variables, only the bit for the current
784 * sample is set in gl_SampleMaskIn. When state specifies multiple
785 * fragment shader invocations for a given fragment, the sample
786 * mask for any single fragment shader invocation may specify a
787 * subset of the covered samples for the fragment. In this case,
788 * the bit corresponding to each covered sample will be set in
789 * exactly one fragment shader invocation."
791 * The samplemask loaded by hardware is always the coverage of the
792 * entire pixel/fragment, so mask bits out based on the sample ID.
794 if (key
->ps_prolog
.states
.samplemask_log_ps_iter
) {
795 /* The bit pattern matches that used by fixed function fragment
797 static const uint16_t ps_iter_masks
[] = {
798 0xffff, /* not used */
799 0x5555, 0x1111, 0x0101, 0x0001,
801 assert(key
->ps_prolog
.states
.samplemask_log_ps_iter
< ARRAY_SIZE(ps_iter_masks
));
803 uint32_t ps_iter_mask
= ps_iter_masks
[key
->ps_prolog
.states
.samplemask_log_ps_iter
];
804 LLVMValueRef sampleid
= si_unpack_param(ctx
, ancillary
, 8, 4);
805 LLVMValueRef samplemask
= ac_get_arg(&ctx
->ac
, param_sample_mask
);
807 samplemask
= ac_to_integer(&ctx
->ac
, samplemask
);
809 LLVMBuildAnd(ctx
->ac
.builder
, samplemask
,
810 LLVMBuildShl(ctx
->ac
.builder
, LLVMConstInt(ctx
->ac
.i32
, ps_iter_mask
, false),
813 samplemask
= ac_to_float(&ctx
->ac
, samplemask
);
815 ret
= LLVMBuildInsertValue(ctx
->ac
.builder
, ret
, samplemask
, param_sample_mask
.arg_index
, "");
818 /* Tell LLVM to insert WQM instruction sequence when needed. */
819 if (key
->ps_prolog
.wqm
) {
820 LLVMAddTargetDependentFunctionAttr(func
, "amdgpu-ps-wqm-outputs", "");
823 si_llvm_build_ret(ctx
, ret
);
827 * Build the pixel shader epilog function. This handles everything that must be
828 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
830 void si_llvm_build_ps_epilog(struct si_shader_context
*ctx
, union si_shader_part_key
*key
)
832 LLVMValueRef depth
= NULL
, stencil
= NULL
, samplemask
= NULL
;
834 struct si_ps_exports exp
= {};
836 memset(&ctx
->args
, 0, sizeof(ctx
->args
));
838 /* Declare input SGPRs. */
839 ac_add_arg(&ctx
->args
, AC_ARG_SGPR
, 1, AC_ARG_INT
, &ctx
->rw_buffers
);
840 ac_add_arg(&ctx
->args
, AC_ARG_SGPR
, 1, AC_ARG_INT
, &ctx
->bindless_samplers_and_images
);
841 ac_add_arg(&ctx
->args
, AC_ARG_SGPR
, 1, AC_ARG_INT
, &ctx
->const_and_shader_buffers
);
842 ac_add_arg(&ctx
->args
, AC_ARG_SGPR
, 1, AC_ARG_INT
, &ctx
->samplers_and_images
);
843 si_add_arg_checked(&ctx
->args
, AC_ARG_SGPR
, 1, AC_ARG_FLOAT
, NULL
, SI_PARAM_ALPHA_REF
);
845 /* Declare input VGPRs. */
846 unsigned required_num_params
=
847 ctx
->args
.num_sgprs_used
+ util_bitcount(key
->ps_epilog
.colors_written
) * 4 +
848 key
->ps_epilog
.writes_z
+ key
->ps_epilog
.writes_stencil
+ key
->ps_epilog
.writes_samplemask
;
850 required_num_params
=
851 MAX2(required_num_params
, ctx
->args
.num_sgprs_used
+ PS_EPILOG_SAMPLEMASK_MIN_LOC
+ 1);
853 while (ctx
->args
.arg_count
< required_num_params
)
854 ac_add_arg(&ctx
->args
, AC_ARG_VGPR
, 1, AC_ARG_FLOAT
, NULL
);
856 /* Create the function. */
857 si_llvm_create_func(ctx
, "ps_epilog", NULL
, 0, 0);
858 /* Disable elimination of unused inputs. */
859 ac_llvm_add_target_dep_function_attr(ctx
->main_fn
, "InitialPSInputAddr", 0xffffff);
861 /* Process colors. */
862 unsigned vgpr
= ctx
->args
.num_sgprs_used
;
863 unsigned colors_written
= key
->ps_epilog
.colors_written
;
864 int last_color_export
= -1;
866 /* Find the last color export. */
867 if (!key
->ps_epilog
.writes_z
&& !key
->ps_epilog
.writes_stencil
&&
868 !key
->ps_epilog
.writes_samplemask
) {
869 unsigned spi_format
= key
->ps_epilog
.states
.spi_shader_col_format
;
871 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
872 if (colors_written
== 0x1 && key
->ps_epilog
.states
.last_cbuf
> 0) {
873 /* Just set this if any of the colorbuffers are enabled. */
874 if (spi_format
& ((1ull << (4 * (key
->ps_epilog
.states
.last_cbuf
+ 1))) - 1))
875 last_color_export
= 0;
877 for (i
= 0; i
< 8; i
++)
878 if (colors_written
& (1 << i
) && (spi_format
>> (i
* 4)) & 0xf)
879 last_color_export
= i
;
883 unsigned num_compacted_mrts
= 0;
884 while (colors_written
) {
885 LLVMValueRef color
[4];
886 int output_index
= u_bit_scan(&colors_written
);
888 for (i
= 0; i
< 4; i
++)
889 color
[i
] = LLVMGetParam(ctx
->main_fn
, vgpr
++);
891 if (si_export_mrt_color(ctx
, color
, output_index
, num_compacted_mrts
,
892 ctx
->args
.arg_count
- 1,
893 output_index
== last_color_export
, &exp
))
894 num_compacted_mrts
++;
897 /* Process depth, stencil, samplemask. */
898 if (key
->ps_epilog
.writes_z
)
899 depth
= LLVMGetParam(ctx
->main_fn
, vgpr
++);
900 if (key
->ps_epilog
.writes_stencil
)
901 stencil
= LLVMGetParam(ctx
->main_fn
, vgpr
++);
902 if (key
->ps_epilog
.writes_samplemask
)
903 samplemask
= LLVMGetParam(ctx
->main_fn
, vgpr
++);
905 if (depth
|| stencil
|| samplemask
)
906 si_export_mrt_z(ctx
, depth
, stencil
, samplemask
, &exp
);
907 else if (last_color_export
== -1)
908 ac_build_export_null(&ctx
->ac
);
911 si_emit_ps_exports(ctx
, &exp
);
914 LLVMBuildRetVoid(ctx
->ac
.builder
);
917 void si_llvm_build_monolithic_ps(struct si_shader_context
*ctx
, struct si_shader
*shader
)
919 LLVMValueRef parts
[3];
920 unsigned num_parts
= 0, main_index
;
921 LLVMValueRef main_fn
= ctx
->main_fn
;
923 union si_shader_part_key prolog_key
;
924 si_get_ps_prolog_key(shader
, &prolog_key
, false);
926 if (si_need_ps_prolog(&prolog_key
)) {
927 si_llvm_build_ps_prolog(ctx
, &prolog_key
);
928 parts
[num_parts
++] = ctx
->main_fn
;
931 main_index
= num_parts
;
932 parts
[num_parts
++] = main_fn
;
934 union si_shader_part_key epilog_key
;
935 si_get_ps_epilog_key(shader
, &epilog_key
);
936 si_llvm_build_ps_epilog(ctx
, &epilog_key
);
937 parts
[num_parts
++] = ctx
->main_fn
;
939 si_build_wrapper_function(ctx
, parts
, num_parts
, main_index
, 0);
942 void si_llvm_init_ps_callbacks(struct si_shader_context
*ctx
)
944 ctx
->abi
.emit_outputs
= si_llvm_return_fs_outputs
;
945 ctx
->abi
.load_sample_position
= load_sample_position
;
946 ctx
->abi
.load_sample_mask_in
= load_sample_mask_in
;
947 ctx
->abi
.emit_fbfetch
= si_nir_emit_fbfetch
;