projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: use ctx->ac. for types and integer constants
[mesa.git] / src / gallium / drivers / radeonsi / si_shader_llvm_ps.c
1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "si_shader_internal.h"
26 #include "si_pipe.h"
27 #include "sid.h"
28
29 LLVMValueRef si_get_sample_id(struct si_shader_context *ctx)
30 {
31 return si_unpack_param(ctx, ctx->args.ancillary, 8, 4);
32 }
33
34 static LLVMValueRef load_sample_mask_in(struct ac_shader_abi *abi)
35 {
36 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
37 return ac_to_integer(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args.sample_coverage));
38 }
39
40 static LLVMValueRef load_sample_position(struct ac_shader_abi *abi, LLVMValueRef sample_id)
41 {
42 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
43 LLVMValueRef desc = ac_get_arg(&ctx->ac, ctx->rw_buffers);
44 LLVMValueRef buf_index = LLVMConstInt(ctx->ac.i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
45 LLVMValueRef resource = ac_build_load_to_sgpr(&ctx->ac, desc, buf_index);
46
47 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
48 LLVMValueRef offset0 = LLVMBuildMul(ctx->ac.builder, sample_id, LLVMConstInt(ctx->ac.i32, 8, 0), "");
49 LLVMValueRef offset1 = LLVMBuildAdd(ctx->ac.builder, offset0, LLVMConstInt(ctx->ac.i32, 4, 0), "");
50
51 LLVMValueRef pos[4] = {
52 si_buffer_load_const(ctx, resource, offset0),
53 si_buffer_load_const(ctx, resource, offset1),
54 LLVMConstReal(ctx->ac.f32, 0),
55 LLVMConstReal(ctx->ac.f32, 0)
56 };
57
58 return ac_build_gather_values(&ctx->ac, pos, 4);
59 }
60
61 static LLVMValueRef si_nir_emit_fbfetch(struct ac_shader_abi *abi)
62 {
63 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
64 struct ac_image_args args = {};
65 LLVMValueRef ptr, image, fmask;
66
67 /* Ignore src0, because KHR_blend_func_extended disallows multiple render
68 * targets.
69 */
70
71 /* Load the image descriptor. */
72 STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0 % 2 == 0);
73 ptr = ac_get_arg(&ctx->ac, ctx->rw_buffers);
74 ptr = LLVMBuildPointerCast(ctx->ac.builder, ptr,
75 ac_array_in_const32_addr_space(ctx->ac.v8i32), "");
76 image = ac_build_load_to_sgpr(&ctx->ac, ptr,
77 LLVMConstInt(ctx->ac.i32, SI_PS_IMAGE_COLORBUF0 / 2, 0));
78
79 unsigned chan = 0;
80
81 args.coords[chan++] = si_unpack_param(ctx, ctx->pos_fixed_pt, 0, 16);
82
83 if (!ctx->shader->key.mono.u.ps.fbfetch_is_1D)
84 args.coords[chan++] = si_unpack_param(ctx, ctx->pos_fixed_pt, 16, 16);
85
86 /* Get the current render target layer index. */
87 if (ctx->shader->key.mono.u.ps.fbfetch_layered)
88 args.coords[chan++] = si_unpack_param(ctx, ctx->args.ancillary, 16, 11);
89
90 if (ctx->shader->key.mono.u.ps.fbfetch_msaa)
91 args.coords[chan++] = si_get_sample_id(ctx);
92
93 if (ctx->shader->key.mono.u.ps.fbfetch_msaa &&
94 !(ctx->screen->debug_flags & DBG(NO_FMASK))) {
95 fmask = ac_build_load_to_sgpr(&ctx->ac, ptr,
96 LLVMConstInt(ctx->ac.i32, SI_PS_IMAGE_COLORBUF0_FMASK / 2, 0));
97
98 ac_apply_fmask_to_sample(&ctx->ac, fmask, args.coords,
99 ctx->shader->key.mono.u.ps.fbfetch_layered);
100 }
101
102 args.opcode = ac_image_load;
103 args.resource = image;
104 args.dmask = 0xf;
105 args.attributes = AC_FUNC_ATTR_READNONE;
106
107 if (ctx->shader->key.mono.u.ps.fbfetch_msaa)
108 args.dim = ctx->shader->key.mono.u.ps.fbfetch_layered ?
109 ac_image_2darraymsaa : ac_image_2dmsaa;
110 else if (ctx->shader->key.mono.u.ps.fbfetch_is_1D)
111 args.dim = ctx->shader->key.mono.u.ps.fbfetch_layered ?
112 ac_image_1darray : ac_image_1d;
113 else
114 args.dim = ctx->shader->key.mono.u.ps.fbfetch_layered ?
115 ac_image_2darray : ac_image_2d;
116
117 return ac_build_image_opcode(&ctx->ac, &args);
118 }
119
120 static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
121 unsigned attr_index, unsigned chan,
122 LLVMValueRef prim_mask,
123 LLVMValueRef i, LLVMValueRef j)
124 {
125 if (i || j) {
126 return ac_build_fs_interp(&ctx->ac,
127 LLVMConstInt(ctx->ac.i32, chan, 0),
128 LLVMConstInt(ctx->ac.i32, attr_index, 0),
129 prim_mask, i, j);
130 }
131 return ac_build_fs_interp_mov(&ctx->ac,
132 LLVMConstInt(ctx->ac.i32, 2, 0), /* P0 */
133 LLVMConstInt(ctx->ac.i32, chan, 0),
134 LLVMConstInt(ctx->ac.i32, attr_index, 0),
135 prim_mask);
136 }
137
138 /**
139 * Interpolate a fragment shader input.
140 *
141 * @param ctx context
142 * @param input_index index of the input in hardware
143 * @param semantic_name TGSI_SEMANTIC_*
144 * @param semantic_index semantic index
145 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
146 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
147 * @param interp_param interpolation weights (i,j)
148 * @param prim_mask SI_PARAM_PRIM_MASK
149 * @param face SI_PARAM_FRONT_FACE
150 * @param result the return value (4 components)
151 */
152 static void interp_fs_color(struct si_shader_context *ctx,
153 unsigned input_index,
154 unsigned semantic_index,
155 unsigned num_interp_inputs,
156 unsigned colors_read_mask,
157 LLVMValueRef interp_param,
158 LLVMValueRef prim_mask,
159 LLVMValueRef face,
160 LLVMValueRef result[4])
161 {
162 LLVMValueRef i = NULL, j = NULL;
163 unsigned chan;
164
165 /* fs.constant returns the param from the middle vertex, so it's not
166 * really useful for flat shading. It's meant to be used for custom
167 * interpolation (but the intrinsic can't fetch from the other two
168 * vertices).
169 *
170 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
171 * to do the right thing. The only reason we use fs.constant is that
172 * fs.interp cannot be used on integers, because they can be equal
173 * to NaN.
174 *
175 * When interp is false we will use fs.constant or for newer llvm,
176 * amdgcn.interp.mov.
177 */
178 bool interp = interp_param != NULL;
179
180 if (interp) {
181 interp_param = LLVMBuildBitCast(ctx->ac.builder, interp_param,
182 LLVMVectorType(ctx->ac.f32, 2), "");
183
184 i = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
185 ctx->ac.i32_0, "");
186 j = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
187 ctx->ac.i32_1, "");
188 }
189
190 if (ctx->shader->key.part.ps.prolog.color_two_side) {
191 LLVMValueRef is_face_positive;
192
193 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
194 * otherwise it's at offset "num_inputs".
195 */
196 unsigned back_attr_offset = num_interp_inputs;
197 if (semantic_index == 1 && colors_read_mask & 0xf)
198 back_attr_offset += 1;
199
200 is_face_positive = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
201 face, ctx->ac.i32_0, "");
202
203 for (chan = 0; chan < 4; chan++) {
204 LLVMValueRef front, back;
205
206 front = si_build_fs_interp(ctx,
207 input_index, chan,
208 prim_mask, i, j);
209 back = si_build_fs_interp(ctx,
210 back_attr_offset, chan,
211 prim_mask, i, j);
212
213 result[chan] = LLVMBuildSelect(ctx->ac.builder,
214 is_face_positive,
215 front,
216 back,
217 "");
218 }
219 } else {
220 for (chan = 0; chan < 4; chan++) {
221 result[chan] = si_build_fs_interp(ctx,
222 input_index, chan,
223 prim_mask, i, j);
224 }
225 }
226 }
227
228 static void si_alpha_test(struct si_shader_context *ctx, LLVMValueRef alpha)
229 {
230 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
231 static LLVMRealPredicate cond_map[PIPE_FUNC_ALWAYS + 1] = {
232 [PIPE_FUNC_LESS] = LLVMRealOLT,
233 [PIPE_FUNC_EQUAL] = LLVMRealOEQ,
234 [PIPE_FUNC_LEQUAL] = LLVMRealOLE,
235 [PIPE_FUNC_GREATER] = LLVMRealOGT,
236 [PIPE_FUNC_NOTEQUAL] = LLVMRealONE,
237 [PIPE_FUNC_GEQUAL] = LLVMRealOGE,
238 };
239 LLVMRealPredicate cond = cond_map[ctx->shader->key.part.ps.epilog.alpha_func];
240 assert(cond);
241
242 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
243 SI_PARAM_ALPHA_REF);
244 LLVMValueRef alpha_pass =
245 LLVMBuildFCmp(ctx->ac.builder, cond, alpha, alpha_ref, "");
246 ac_build_kill_if_false(&ctx->ac, alpha_pass);
247 } else {
248 ac_build_kill_if_false(&ctx->ac, ctx->ac.i1false);
249 }
250 }
251
252 static LLVMValueRef si_scale_alpha_by_sample_mask(struct si_shader_context *ctx,
253 LLVMValueRef alpha,
254 unsigned samplemask_param)
255 {
256 LLVMValueRef coverage;
257
258 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
259 coverage = LLVMGetParam(ctx->main_fn,
260 samplemask_param);
261 coverage = ac_to_integer(&ctx->ac, coverage);
262
263 coverage = ac_build_intrinsic(&ctx->ac, "llvm.ctpop.i32",
264 ctx->ac.i32,
265 &coverage, 1, AC_FUNC_ATTR_READNONE);
266
267 coverage = LLVMBuildUIToFP(ctx->ac.builder, coverage,
268 ctx->ac.f32, "");
269
270 coverage = LLVMBuildFMul(ctx->ac.builder, coverage,
271 LLVMConstReal(ctx->ac.f32,
272 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
273
274 return LLVMBuildFMul(ctx->ac.builder, alpha, coverage, "");
275 }
276
277 struct si_ps_exports {
278 unsigned num;
279 struct ac_export_args args[10];
280 };
281
282 static void si_export_mrt_z(struct si_shader_context *ctx,
283 LLVMValueRef depth, LLVMValueRef stencil,
284 LLVMValueRef samplemask, struct si_ps_exports *exp)
285 {
286 struct ac_export_args args;
287
288 ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, &args);
289
290 memcpy(&exp->args[exp->num++], &args, sizeof(args));
291 }
292
293 /* Initialize arguments for the shader export intrinsic */
294 static void si_llvm_init_ps_export_args(struct si_shader_context *ctx,
295 LLVMValueRef *values,
296 unsigned target,
297 struct ac_export_args *args)
298 {
299 const struct si_shader_key *key = &ctx->shader->key;
300 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
301 LLVMValueRef f32undef = LLVMGetUndef(ctx->ac.f32);
302 unsigned spi_shader_col_format;
303 unsigned chan;
304 bool is_int8, is_int10;
305 int cbuf = target - V_008DFC_SQ_EXP_MRT;
306
307 assert(cbuf >= 0 && cbuf < 8);
308
309 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
310 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
311 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
312
313 /* Default is 0xf. Adjusted below depending on the format. */
314 args->enabled_channels = 0xf; /* writemask */
315
316 /* Specify whether the EXEC mask represents the valid mask */
317 args->valid_mask = 0;
318
319 /* Specify whether this is the last export */
320 args->done = 0;
321
322 /* Specify the target we are exporting */
323 args->target = target;
324
325 args->compr = false;
326 args->out[0] = f32undef;
327 args->out[1] = f32undef;
328 args->out[2] = f32undef;
329 args->out[3] = f32undef;
330
331 LLVMValueRef (*packf)(struct ac_llvm_context *ctx, LLVMValueRef args[2]) = NULL;
332 LLVMValueRef (*packi)(struct ac_llvm_context *ctx, LLVMValueRef args[2],
333 unsigned bits, bool hi) = NULL;
334
335 switch (spi_shader_col_format) {
336 case V_028714_SPI_SHADER_ZERO:
337 args->enabled_channels = 0; /* writemask */
338 args->target = V_008DFC_SQ_EXP_NULL;
339 break;
340
341 case V_028714_SPI_SHADER_32_R:
342 args->enabled_channels = 1; /* writemask */
343 args->out[0] = values[0];
344 break;
345
346 case V_028714_SPI_SHADER_32_GR:
347 args->enabled_channels = 0x3; /* writemask */
348 args->out[0] = values[0];
349 args->out[1] = values[1];
350 break;
351
352 case V_028714_SPI_SHADER_32_AR:
353 if (ctx->screen->info.chip_class >= GFX10) {
354 args->enabled_channels = 0x3; /* writemask */
355 args->out[0] = values[0];
356 args->out[1] = values[3];
357 } else {
358 args->enabled_channels = 0x9; /* writemask */
359 args->out[0] = values[0];
360 args->out[3] = values[3];
361 }
362 break;
363
364 case V_028714_SPI_SHADER_FP16_ABGR:
365 packf = ac_build_cvt_pkrtz_f16;
366 break;
367
368 case V_028714_SPI_SHADER_UNORM16_ABGR:
369 packf = ac_build_cvt_pknorm_u16;
370 break;
371
372 case V_028714_SPI_SHADER_SNORM16_ABGR:
373 packf = ac_build_cvt_pknorm_i16;
374 break;
375
376 case V_028714_SPI_SHADER_UINT16_ABGR:
377 packi = ac_build_cvt_pk_u16;
378 break;
379
380 case V_028714_SPI_SHADER_SINT16_ABGR:
381 packi = ac_build_cvt_pk_i16;
382 break;
383
384 case V_028714_SPI_SHADER_32_ABGR:
385 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
386 break;
387 }
388
389 /* Pack f16 or norm_i16/u16. */
390 if (packf) {
391 for (chan = 0; chan < 2; chan++) {
392 LLVMValueRef pack_args[2] = {
393 values[2 * chan],
394 values[2 * chan + 1]
395 };
396 LLVMValueRef packed;
397
398 packed = packf(&ctx->ac, pack_args);
399 args->out[chan] = ac_to_float(&ctx->ac, packed);
400 }
401 args->compr = 1; /* COMPR flag */
402 }
403 /* Pack i16/u16. */
404 if (packi) {
405 for (chan = 0; chan < 2; chan++) {
406 LLVMValueRef pack_args[2] = {
407 ac_to_integer(&ctx->ac, values[2 * chan]),
408 ac_to_integer(&ctx->ac, values[2 * chan + 1])
409 };
410 LLVMValueRef packed;
411
412 packed = packi(&ctx->ac, pack_args,
413 is_int8 ? 8 : is_int10 ? 10 : 16,
414 chan == 1);
415 args->out[chan] = ac_to_float(&ctx->ac, packed);
416 }
417 args->compr = 1; /* COMPR flag */
418 }
419 }
420
421 static void si_export_mrt_color(struct si_shader_context *ctx,
422 LLVMValueRef *color, unsigned index,
423 unsigned samplemask_param,
424 bool is_last, struct si_ps_exports *exp)
425 {
426 int i;
427
428 /* Clamp color */
429 if (ctx->shader->key.part.ps.epilog.clamp_color)
430 for (i = 0; i < 4; i++)
431 color[i] = ac_build_clamp(&ctx->ac, color[i]);
432
433 /* Alpha to one */
434 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
435 color[3] = ctx->ac.f32_1;
436
437 /* Alpha test */
438 if (index == 0 &&
439 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
440 si_alpha_test(ctx, color[3]);
441
442 /* Line & polygon smoothing */
443 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
444 color[3] = si_scale_alpha_by_sample_mask(ctx, color[3],
445 samplemask_param);
446
447 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
448 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
449 struct ac_export_args args[8];
450 int c, last = -1;
451
452 /* Get the export arguments, also find out what the last one is. */
453 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
454 si_llvm_init_ps_export_args(ctx, color,
455 V_008DFC_SQ_EXP_MRT + c, &args[c]);
456 if (args[c].enabled_channels)
457 last = c;
458 }
459
460 /* Emit all exports. */
461 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
462 if (is_last && last == c) {
463 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
464 args[c].done = 1; /* DONE bit */
465 } else if (!args[c].enabled_channels)
466 continue; /* unnecessary NULL export */
467
468 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
469 }
470 } else {
471 struct ac_export_args args;
472
473 /* Export */
474 si_llvm_init_ps_export_args(ctx, color, V_008DFC_SQ_EXP_MRT + index,
475 &args);
476 if (is_last) {
477 args.valid_mask = 1; /* whether the EXEC mask is valid */
478 args.done = 1; /* DONE bit */
479 } else if (!args.enabled_channels)
480 return; /* unnecessary NULL export */
481
482 memcpy(&exp->args[exp->num++], &args, sizeof(args));
483 }
484 }
485
486 static void si_emit_ps_exports(struct si_shader_context *ctx,
487 struct si_ps_exports *exp)
488 {
489 for (unsigned i = 0; i < exp->num; i++)
490 ac_build_export(&ctx->ac, &exp->args[i]);
491 }
492
493 /**
494 * Return PS outputs in this order:
495 *
496 * v[0:3] = color0.xyzw
497 * v[4:7] = color1.xyzw
498 * ...
499 * vN+0 = Depth
500 * vN+1 = Stencil
501 * vN+2 = SampleMask
502 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
503 *
504 * The alpha-ref SGPR is returned via its original location.
505 */
506 static void si_llvm_return_fs_outputs(struct ac_shader_abi *abi,
507 unsigned max_outputs,
508 LLVMValueRef *addrs)
509 {
510 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
511 struct si_shader *shader = ctx->shader;
512 struct si_shader_info *info = &shader->selector->info;
513 LLVMBuilderRef builder = ctx->ac.builder;
514 unsigned i, j, first_vgpr, vgpr;
515
516 LLVMValueRef color[8][4] = {};
517 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
518 LLVMValueRef ret;
519
520 if (ctx->postponed_kill)
521 ac_build_kill_if_false(&ctx->ac, LLVMBuildLoad(builder, ctx->postponed_kill, ""));
522
523 /* Read the output values. */
524 for (i = 0; i < info->num_outputs; i++) {
525 unsigned semantic_name = info->output_semantic_name[i];
526 unsigned semantic_index = info->output_semantic_index[i];
527
528 switch (semantic_name) {
529 case TGSI_SEMANTIC_COLOR:
530 assert(semantic_index < 8);
531 for (j = 0; j < 4; j++) {
532 LLVMValueRef ptr = addrs[4 * i + j];
533 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
534 color[semantic_index][j] = result;
535 }
536 break;
537 case TGSI_SEMANTIC_POSITION:
538 depth = LLVMBuildLoad(builder,
539 addrs[4 * i + 0], "");
540 break;
541 case TGSI_SEMANTIC_STENCIL:
542 stencil = LLVMBuildLoad(builder,
543 addrs[4 * i + 0], "");
544 break;
545 case TGSI_SEMANTIC_SAMPLEMASK:
546 samplemask = LLVMBuildLoad(builder,
547 addrs[4 * i + 0], "");
548 break;
549 default:
550 fprintf(stderr, "Warning: GFX6 unhandled fs output type:%d\n",
551 semantic_name);
552 }
553 }
554
555 /* Fill the return structure. */
556 ret = ctx->return_value;
557
558 /* Set SGPRs. */
559 ret = LLVMBuildInsertValue(builder, ret,
560 ac_to_integer(&ctx->ac,
561 LLVMGetParam(ctx->main_fn,
562 SI_PARAM_ALPHA_REF)),
563 SI_SGPR_ALPHA_REF, "");
564
565 /* Set VGPRs */
566 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
567 for (i = 0; i < ARRAY_SIZE(color); i++) {
568 if (!color[i][0])
569 continue;
570
571 for (j = 0; j < 4; j++)
572 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
573 }
574 if (depth)
575 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
576 if (stencil)
577 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
578 if (samplemask)
579 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
580
581 /* Add the input sample mask for smoothing at the end. */
582 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
583 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
584 ret = LLVMBuildInsertValue(builder, ret,
585 LLVMGetParam(ctx->main_fn,
586 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
587
588 ctx->return_value = ret;
589 }
590
591 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
592 LLVMValueRef param_rw_buffers,
593 struct ac_arg param_pos_fixed_pt)
594 {
595 LLVMBuilderRef builder = ctx->ac.builder;
596 LLVMValueRef slot, desc, offset, row, bit, address[2];
597
598 /* Use the fixed-point gl_FragCoord input.
599 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
600 * per coordinate to get the repeating effect.
601 */
602 address[0] = si_unpack_param(ctx, param_pos_fixed_pt, 0, 5);
603 address[1] = si_unpack_param(ctx, param_pos_fixed_pt, 16, 5);
604
605 /* Load the buffer descriptor. */
606 slot = LLVMConstInt(ctx->ac.i32, SI_PS_CONST_POLY_STIPPLE, 0);
607 desc = ac_build_load_to_sgpr(&ctx->ac, param_rw_buffers, slot);
608
609 /* The stipple pattern is 32x32, each row has 32 bits. */
610 offset = LLVMBuildMul(builder, address[1],
611 LLVMConstInt(ctx->ac.i32, 4, 0), "");
612 row = si_buffer_load_const(ctx, desc, offset);
613 row = ac_to_integer(&ctx->ac, row);
614 bit = LLVMBuildLShr(builder, row, address[0], "");
615 bit = LLVMBuildTrunc(builder, bit, ctx->ac.i1, "");
616 ac_build_kill_if_false(&ctx->ac, bit);
617 }
618
619 static void si_llvm_emit_kill(struct ac_shader_abi *abi, LLVMValueRef visible)
620 {
621 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
622 LLVMBuilderRef builder = ctx->ac.builder;
623
624 if (ctx->shader->selector->force_correct_derivs_after_kill) {
625 /* Kill immediately while maintaining WQM. */
626 ac_build_kill_if_false(&ctx->ac,
627 ac_build_wqm_vote(&ctx->ac, visible));
628
629 LLVMValueRef mask = LLVMBuildLoad(builder, ctx->postponed_kill, "");
630 mask = LLVMBuildAnd(builder, mask, visible, "");
631 LLVMBuildStore(builder, mask, ctx->postponed_kill);
632 return;
633 }
634
635 ac_build_kill_if_false(&ctx->ac, visible);
636 }
637
638 /**
639 * Build the pixel shader prolog function. This handles:
640 * - two-side color selection and interpolation
641 * - overriding interpolation parameters for the API PS
642 * - polygon stippling
643 *
644 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
645 * overriden by other states. (e.g. per-sample interpolation)
646 * Interpolated colors are stored after the preloaded VGPRs.
647 */
648 void si_llvm_build_ps_prolog(struct si_shader_context *ctx,
649 union si_shader_part_key *key)
650 {
651 LLVMValueRef ret, func;
652 int num_returns, i, num_color_channels;
653
654 memset(&ctx->args, 0, sizeof(ctx->args));
655
656 /* Declare inputs. */
657 LLVMTypeRef return_types[AC_MAX_ARGS];
658 num_returns = 0;
659 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
660 assert(key->ps_prolog.num_input_sgprs +
661 key->ps_prolog.num_input_vgprs +
662 num_color_channels <= AC_MAX_ARGS);
663 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++) {
664 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
665 return_types[num_returns++] = ctx->ac.i32;
666
667 }
668
669 struct ac_arg pos_fixed_pt;
670 struct ac_arg ancillary;
671 struct ac_arg param_sample_mask;
672 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++) {
673 struct ac_arg *arg = NULL;
674 if (i == key->ps_prolog.ancillary_vgpr_index) {
675 arg = &ancillary;
676 } else if (i == key->ps_prolog.ancillary_vgpr_index + 1) {
677 arg = &param_sample_mask;
678 } else if (i == key->ps_prolog.num_input_vgprs - 1) {
679 /* POS_FIXED_PT is always last. */
680 arg = &pos_fixed_pt;
681 }
682 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, arg);
683 return_types[num_returns++] = ctx->ac.f32;
684 }
685
686 /* Declare outputs (same as inputs + add colors if needed) */
687 for (i = 0; i < num_color_channels; i++)
688 return_types[num_returns++] = ctx->ac.f32;
689
690 /* Create the function. */
691 si_llvm_create_func(ctx, "ps_prolog", return_types, num_returns, 0);
692 func = ctx->main_fn;
693
694 /* Copy inputs to outputs. This should be no-op, as the registers match,
695 * but it will prevent the compiler from overwriting them unintentionally.
696 */
697 ret = ctx->return_value;
698 for (i = 0; i < ctx->args.arg_count; i++) {
699 LLVMValueRef p = LLVMGetParam(func, i);
700 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
701 }
702
703 /* Polygon stippling. */
704 if (key->ps_prolog.states.poly_stipple) {
705 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
706
707 si_llvm_emit_polygon_stipple(ctx, list, pos_fixed_pt);
708 }
709
710 if (key->ps_prolog.states.bc_optimize_for_persp ||
711 key->ps_prolog.states.bc_optimize_for_linear) {
712 unsigned i, base = key->ps_prolog.num_input_sgprs;
713 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
714
715 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
716 * The hw doesn't compute CENTROID if the whole wave only
717 * contains fully-covered quads.
718 *
719 * PRIM_MASK is after user SGPRs.
720 */
721 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
722 bc_optimize = LLVMBuildLShr(ctx->ac.builder, bc_optimize,
723 LLVMConstInt(ctx->ac.i32, 31, 0), "");
724 bc_optimize = LLVMBuildTrunc(ctx->ac.builder, bc_optimize,
725 ctx->ac.i1, "");
726
727 if (key->ps_prolog.states.bc_optimize_for_persp) {
728 /* Read PERSP_CENTER. */
729 for (i = 0; i < 2; i++)
730 center[i] = LLVMGetParam(func, base + 2 + i);
731 /* Read PERSP_CENTROID. */
732 for (i = 0; i < 2; i++)
733 centroid[i] = LLVMGetParam(func, base + 4 + i);
734 /* Select PERSP_CENTROID. */
735 for (i = 0; i < 2; i++) {
736 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
737 center[i], centroid[i], "");
738 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
739 tmp, base + 4 + i, "");
740 }
741 }
742 if (key->ps_prolog.states.bc_optimize_for_linear) {
743 /* Read LINEAR_CENTER. */
744 for (i = 0; i < 2; i++)
745 center[i] = LLVMGetParam(func, base + 8 + i);
746 /* Read LINEAR_CENTROID. */
747 for (i = 0; i < 2; i++)
748 centroid[i] = LLVMGetParam(func, base + 10 + i);
749 /* Select LINEAR_CENTROID. */
750 for (i = 0; i < 2; i++) {
751 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
752 center[i], centroid[i], "");
753 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
754 tmp, base + 10 + i, "");
755 }
756 }
757 }
758
759 /* Force per-sample interpolation. */
760 if (key->ps_prolog.states.force_persp_sample_interp) {
761 unsigned i, base = key->ps_prolog.num_input_sgprs;
762 LLVMValueRef persp_sample[2];
763
764 /* Read PERSP_SAMPLE. */
765 for (i = 0; i < 2; i++)
766 persp_sample[i] = LLVMGetParam(func, base + i);
767 /* Overwrite PERSP_CENTER. */
768 for (i = 0; i < 2; i++)
769 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
770 persp_sample[i], base + 2 + i, "");
771 /* Overwrite PERSP_CENTROID. */
772 for (i = 0; i < 2; i++)
773 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
774 persp_sample[i], base + 4 + i, "");
775 }
776 if (key->ps_prolog.states.force_linear_sample_interp) {
777 unsigned i, base = key->ps_prolog.num_input_sgprs;
778 LLVMValueRef linear_sample[2];
779
780 /* Read LINEAR_SAMPLE. */
781 for (i = 0; i < 2; i++)
782 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
783 /* Overwrite LINEAR_CENTER. */
784 for (i = 0; i < 2; i++)
785 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
786 linear_sample[i], base + 8 + i, "");
787 /* Overwrite LINEAR_CENTROID. */
788 for (i = 0; i < 2; i++)
789 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
790 linear_sample[i], base + 10 + i, "");
791 }
792
793 /* Force center interpolation. */
794 if (key->ps_prolog.states.force_persp_center_interp) {
795 unsigned i, base = key->ps_prolog.num_input_sgprs;
796 LLVMValueRef persp_center[2];
797
798 /* Read PERSP_CENTER. */
799 for (i = 0; i < 2; i++)
800 persp_center[i] = LLVMGetParam(func, base + 2 + i);
801 /* Overwrite PERSP_SAMPLE. */
802 for (i = 0; i < 2; i++)
803 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
804 persp_center[i], base + i, "");
805 /* Overwrite PERSP_CENTROID. */
806 for (i = 0; i < 2; i++)
807 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
808 persp_center[i], base + 4 + i, "");
809 }
810 if (key->ps_prolog.states.force_linear_center_interp) {
811 unsigned i, base = key->ps_prolog.num_input_sgprs;
812 LLVMValueRef linear_center[2];
813
814 /* Read LINEAR_CENTER. */
815 for (i = 0; i < 2; i++)
816 linear_center[i] = LLVMGetParam(func, base + 8 + i);
817 /* Overwrite LINEAR_SAMPLE. */
818 for (i = 0; i < 2; i++)
819 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
820 linear_center[i], base + 6 + i, "");
821 /* Overwrite LINEAR_CENTROID. */
822 for (i = 0; i < 2; i++)
823 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
824 linear_center[i], base + 10 + i, "");
825 }
826
827 /* Interpolate colors. */
828 unsigned color_out_idx = 0;
829 for (i = 0; i < 2; i++) {
830 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
831 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
832 key->ps_prolog.face_vgpr_index;
833 LLVMValueRef interp[2], color[4];
834 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
835
836 if (!writemask)
837 continue;
838
839 /* If the interpolation qualifier is not CONSTANT (-1). */
840 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
841 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
842 key->ps_prolog.color_interp_vgpr_index[i];
843
844 /* Get the (i,j) updated by bc_optimize handling. */
845 interp[0] = LLVMBuildExtractValue(ctx->ac.builder, ret,
846 interp_vgpr, "");
847 interp[1] = LLVMBuildExtractValue(ctx->ac.builder, ret,
848 interp_vgpr + 1, "");
849 interp_ij = ac_build_gather_values(&ctx->ac, interp, 2);
850 }
851
852 /* Use the absolute location of the input. */
853 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
854
855 if (key->ps_prolog.states.color_two_side) {
856 face = LLVMGetParam(func, face_vgpr);
857 face = ac_to_integer(&ctx->ac, face);
858 }
859
860 interp_fs_color(ctx,
861 key->ps_prolog.color_attr_index[i], i,
862 key->ps_prolog.num_interp_inputs,
863 key->ps_prolog.colors_read, interp_ij,
864 prim_mask, face, color);
865
866 while (writemask) {
867 unsigned chan = u_bit_scan(&writemask);
868 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, color[chan],
869 ctx->args.arg_count + color_out_idx++, "");
870 }
871 }
872
873 /* Section 15.2.2 (Shader Inputs) of the OpenGL 4.5 (Core Profile) spec
874 * says:
875 *
876 * "When per-sample shading is active due to the use of a fragment
877 * input qualified by sample or due to the use of the gl_SampleID
878 * or gl_SamplePosition variables, only the bit for the current
879 * sample is set in gl_SampleMaskIn. When state specifies multiple
880 * fragment shader invocations for a given fragment, the sample
881 * mask for any single fragment shader invocation may specify a
882 * subset of the covered samples for the fragment. In this case,
883 * the bit corresponding to each covered sample will be set in
884 * exactly one fragment shader invocation."
885 *
886 * The samplemask loaded by hardware is always the coverage of the
887 * entire pixel/fragment, so mask bits out based on the sample ID.
888 */
889 if (key->ps_prolog.states.samplemask_log_ps_iter) {
890 /* The bit pattern matches that used by fixed function fragment
891 * processing. */
892 static const uint16_t ps_iter_masks[] = {
893 0xffff, /* not used */
894 0x5555,
895 0x1111,
896 0x0101,
897 0x0001,
898 };
899 assert(key->ps_prolog.states.samplemask_log_ps_iter < ARRAY_SIZE(ps_iter_masks));
900
901 uint32_t ps_iter_mask = ps_iter_masks[key->ps_prolog.states.samplemask_log_ps_iter];
902 LLVMValueRef sampleid = si_unpack_param(ctx, ancillary, 8, 4);
903 LLVMValueRef samplemask = ac_get_arg(&ctx->ac, param_sample_mask);
904
905 samplemask = ac_to_integer(&ctx->ac, samplemask);
906 samplemask = LLVMBuildAnd(
907 ctx->ac.builder,
908 samplemask,
909 LLVMBuildShl(ctx->ac.builder,
910 LLVMConstInt(ctx->ac.i32, ps_iter_mask, false),
911 sampleid, ""),
912 "");
913 samplemask = ac_to_float(&ctx->ac, samplemask);
914
915 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, samplemask,
916 param_sample_mask.arg_index, "");
917 }
918
919 /* Tell LLVM to insert WQM instruction sequence when needed. */
920 if (key->ps_prolog.wqm) {
921 LLVMAddTargetDependentFunctionAttr(func,
922 "amdgpu-ps-wqm-outputs", "");
923 }
924
925 si_llvm_build_ret(ctx, ret);
926 }
927
928 /**
929 * Build the pixel shader epilog function. This handles everything that must be
930 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
931 */
932 void si_llvm_build_ps_epilog(struct si_shader_context *ctx,
933 union si_shader_part_key *key)
934 {
935 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
936 int i;
937 struct si_ps_exports exp = {};
938
939 memset(&ctx->args, 0, sizeof(ctx->args));
940
941 /* Declare input SGPRs. */
942 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->rw_buffers);
943 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
944 &ctx->bindless_samplers_and_images);
945 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
946 &ctx->const_and_shader_buffers);
947 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
948 &ctx->samplers_and_images);
949 si_add_arg_checked(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT,
950 NULL, SI_PARAM_ALPHA_REF);
951
952 /* Declare input VGPRs. */
953 unsigned required_num_params =
954 ctx->args.num_sgprs_used +
955 util_bitcount(key->ps_epilog.colors_written) * 4 +
956 key->ps_epilog.writes_z +
957 key->ps_epilog.writes_stencil +
958 key->ps_epilog.writes_samplemask;
959
960 required_num_params = MAX2(required_num_params,
961 ctx->args.num_sgprs_used + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
962
963 while (ctx->args.arg_count < required_num_params)
964 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, NULL);
965
966 /* Create the function. */
967 si_llvm_create_func(ctx, "ps_epilog", NULL, 0, 0);
968 /* Disable elimination of unused inputs. */
969 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
970 "InitialPSInputAddr", 0xffffff);
971
972 /* Process colors. */
973 unsigned vgpr = ctx->args.num_sgprs_used;
974 unsigned colors_written = key->ps_epilog.colors_written;
975 int last_color_export = -1;
976
977 /* Find the last color export. */
978 if (!key->ps_epilog.writes_z &&
979 !key->ps_epilog.writes_stencil &&
980 !key->ps_epilog.writes_samplemask) {
981 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
982
983 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
984 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
985 /* Just set this if any of the colorbuffers are enabled. */
986 if (spi_format &
987 ((1ull << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
988 last_color_export = 0;
989 } else {
990 for (i = 0; i < 8; i++)
991 if (colors_written & (1 << i) &&
992 (spi_format >> (i * 4)) & 0xf)
993 last_color_export = i;
994 }
995 }
996
997 while (colors_written) {
998 LLVMValueRef color[4];
999 int mrt = u_bit_scan(&colors_written);
1000
1001 for (i = 0; i < 4; i++)
1002 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
1003
1004 si_export_mrt_color(ctx, color, mrt,
1005 ctx->args.arg_count - 1,
1006 mrt == last_color_export, &exp);
1007 }
1008
1009 /* Process depth, stencil, samplemask. */
1010 if (key->ps_epilog.writes_z)
1011 depth = LLVMGetParam(ctx->main_fn, vgpr++);
1012 if (key->ps_epilog.writes_stencil)
1013 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
1014 if (key->ps_epilog.writes_samplemask)
1015 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
1016
1017 if (depth || stencil || samplemask)
1018 si_export_mrt_z(ctx, depth, stencil, samplemask, &exp);
1019 else if (last_color_export == -1)
1020 ac_build_export_null(&ctx->ac);
1021
1022 if (exp.num)
1023 si_emit_ps_exports(ctx, &exp);
1024
1025 /* Compile. */
1026 LLVMBuildRetVoid(ctx->ac.builder);
1027 }
1028
1029 void si_llvm_build_monolithic_ps(struct si_shader_context *ctx,
1030 struct si_shader *shader)
1031 {
1032 LLVMValueRef parts[3];
1033 unsigned num_parts = 0, main_index;
1034
1035 union si_shader_part_key prolog_key;
1036 si_get_ps_prolog_key(shader, &prolog_key, false);
1037
1038 if (si_need_ps_prolog(&prolog_key)) {
1039 si_llvm_build_ps_prolog(ctx, &prolog_key);
1040 parts[num_parts++] = ctx->main_fn;
1041 }
1042
1043 main_index = num_parts;
1044 parts[num_parts++] = ctx->main_fn;
1045
1046 union si_shader_part_key epilog_key;
1047 si_get_ps_epilog_key(shader, &epilog_key);
1048 si_llvm_build_ps_epilog(ctx, &epilog_key);
1049 parts[num_parts++] = ctx->main_fn;
1050
1051 si_build_wrapper_function(ctx, parts, num_parts, main_index, 0);
1052 }
1053
1054 void si_llvm_init_ps_callbacks(struct si_shader_context *ctx)
1055 {
1056 ctx->abi.emit_outputs = si_llvm_return_fs_outputs;
1057 ctx->abi.load_sample_position = load_sample_position;
1058 ctx->abi.load_sample_mask_in = load_sample_mask_in;
1059 ctx->abi.emit_fbfetch = si_nir_emit_fbfetch;
1060 ctx->abi.emit_kill = si_llvm_emit_kill;
1061 }