projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
v3d: rename vertex shader key (num)_fs_inputs fields
[mesa.git] / src / broadcom / compiler / nir_to_vir.c
1 /*
2 * Copyright © 2016 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <inttypes.h>
25 #include "util/u_format.h"
26 #include "util/u_math.h"
27 #include "util/u_memory.h"
28 #include "util/ralloc.h"
29 #include "util/hash_table.h"
30 #include "compiler/nir/nir.h"
31 #include "compiler/nir/nir_builder.h"
32 #include "common/v3d_device_info.h"
33 #include "v3d_compiler.h"
34
35 /* We don't do any address packing. */
36 #define __gen_user_data void
37 #define __gen_address_type uint32_t
38 #define __gen_address_offset(reloc) (*reloc)
39 #define __gen_emit_reloc(cl, reloc)
40 #include "cle/v3d_packet_v41_pack.h"
41
42 #define GENERAL_TMU_LOOKUP_PER_QUAD (0 << 7)
43 #define GENERAL_TMU_LOOKUP_PER_PIXEL (1 << 7)
44 #define GENERAL_TMU_LOOKUP_TYPE_8BIT_I (0 << 0)
45 #define GENERAL_TMU_LOOKUP_TYPE_16BIT_I (1 << 0)
46 #define GENERAL_TMU_LOOKUP_TYPE_VEC2 (2 << 0)
47 #define GENERAL_TMU_LOOKUP_TYPE_VEC3 (3 << 0)
48 #define GENERAL_TMU_LOOKUP_TYPE_VEC4 (4 << 0)
49 #define GENERAL_TMU_LOOKUP_TYPE_8BIT_UI (5 << 0)
50 #define GENERAL_TMU_LOOKUP_TYPE_16BIT_UI (6 << 0)
51 #define GENERAL_TMU_LOOKUP_TYPE_32BIT_UI (7 << 0)
52
53 #define V3D_TSY_SET_QUORUM 0
54 #define V3D_TSY_INC_WAITERS 1
55 #define V3D_TSY_DEC_WAITERS 2
56 #define V3D_TSY_INC_QUORUM 3
57 #define V3D_TSY_DEC_QUORUM 4
58 #define V3D_TSY_FREE_ALL 5
59 #define V3D_TSY_RELEASE 6
60 #define V3D_TSY_ACQUIRE 7
61 #define V3D_TSY_WAIT 8
62 #define V3D_TSY_WAIT_INC 9
63 #define V3D_TSY_WAIT_CHECK 10
64 #define V3D_TSY_WAIT_INC_CHECK 11
65 #define V3D_TSY_WAIT_CV 12
66 #define V3D_TSY_INC_SEMAPHORE 13
67 #define V3D_TSY_DEC_SEMAPHORE 14
68 #define V3D_TSY_SET_QUORUM_FREE_ALL 15
69
70 static void
71 ntq_emit_cf_list(struct v3d_compile *c, struct exec_list *list);
72
73 static void
74 resize_qreg_array(struct v3d_compile *c,
75 struct qreg **regs,
76 uint32_t *size,
77 uint32_t decl_size)
78 {
79 if (*size >= decl_size)
80 return;
81
82 uint32_t old_size = *size;
83 *size = MAX2(*size * 2, decl_size);
84 *regs = reralloc(c, *regs, struct qreg, *size);
85 if (!*regs) {
86 fprintf(stderr, "Malloc failure\n");
87 abort();
88 }
89
90 for (uint32_t i = old_size; i < *size; i++)
91 (*regs)[i] = c->undef;
92 }
93
94 void
95 vir_emit_thrsw(struct v3d_compile *c)
96 {
97 if (c->threads == 1)
98 return;
99
100 /* Always thread switch after each texture operation for now.
101 *
102 * We could do better by batching a bunch of texture fetches up and
103 * then doing one thread switch and collecting all their results
104 * afterward.
105 */
106 c->last_thrsw = vir_NOP(c);
107 c->last_thrsw->qpu.sig.thrsw = true;
108 c->last_thrsw_at_top_level = !c->in_control_flow;
109
110 /* We need to lock the scoreboard before any tlb acess happens. If this
111 * thread switch comes after we have emitted a tlb load, then it means
112 * that we can't lock on the last thread switch any more.
113 */
114 if (c->emitted_tlb_load)
115 c->lock_scoreboard_on_first_thrsw = true;
116 }
117
118 uint32_t
119 v3d_get_op_for_atomic_add(nir_intrinsic_instr *instr, unsigned src)
120 {
121 if (nir_src_is_const(instr->src[src])) {
122 int64_t add_val = nir_src_as_int(instr->src[src]);
123 if (add_val == 1)
124 return V3D_TMU_OP_WRITE_AND_READ_INC;
125 else if (add_val == -1)
126 return V3D_TMU_OP_WRITE_OR_READ_DEC;
127 }
128
129 return V3D_TMU_OP_WRITE_ADD_READ_PREFETCH;
130 }
131
132 static uint32_t
133 v3d_general_tmu_op(nir_intrinsic_instr *instr)
134 {
135 switch (instr->intrinsic) {
136 case nir_intrinsic_load_ssbo:
137 case nir_intrinsic_load_ubo:
138 case nir_intrinsic_load_uniform:
139 case nir_intrinsic_load_shared:
140 case nir_intrinsic_load_scratch:
141 case nir_intrinsic_store_ssbo:
142 case nir_intrinsic_store_shared:
143 case nir_intrinsic_store_scratch:
144 return V3D_TMU_OP_REGULAR;
145 case nir_intrinsic_ssbo_atomic_add:
146 return v3d_get_op_for_atomic_add(instr, 2);
147 case nir_intrinsic_shared_atomic_add:
148 return v3d_get_op_for_atomic_add(instr, 1);
149 case nir_intrinsic_ssbo_atomic_imin:
150 case nir_intrinsic_shared_atomic_imin:
151 return V3D_TMU_OP_WRITE_SMIN;
152 case nir_intrinsic_ssbo_atomic_umin:
153 case nir_intrinsic_shared_atomic_umin:
154 return V3D_TMU_OP_WRITE_UMIN_FULL_L1_CLEAR;
155 case nir_intrinsic_ssbo_atomic_imax:
156 case nir_intrinsic_shared_atomic_imax:
157 return V3D_TMU_OP_WRITE_SMAX;
158 case nir_intrinsic_ssbo_atomic_umax:
159 case nir_intrinsic_shared_atomic_umax:
160 return V3D_TMU_OP_WRITE_UMAX;
161 case nir_intrinsic_ssbo_atomic_and:
162 case nir_intrinsic_shared_atomic_and:
163 return V3D_TMU_OP_WRITE_AND_READ_INC;
164 case nir_intrinsic_ssbo_atomic_or:
165 case nir_intrinsic_shared_atomic_or:
166 return V3D_TMU_OP_WRITE_OR_READ_DEC;
167 case nir_intrinsic_ssbo_atomic_xor:
168 case nir_intrinsic_shared_atomic_xor:
169 return V3D_TMU_OP_WRITE_XOR_READ_NOT;
170 case nir_intrinsic_ssbo_atomic_exchange:
171 case nir_intrinsic_shared_atomic_exchange:
172 return V3D_TMU_OP_WRITE_XCHG_READ_FLUSH;
173 case nir_intrinsic_ssbo_atomic_comp_swap:
174 case nir_intrinsic_shared_atomic_comp_swap:
175 return V3D_TMU_OP_WRITE_CMPXCHG_READ_FLUSH;
176 default:
177 unreachable("unknown intrinsic op");
178 }
179 }
180
181 /**
182 * Implements indirect uniform loads and SSBO accesses through the TMU general
183 * memory access interface.
184 */
185 static void
186 ntq_emit_tmu_general(struct v3d_compile *c, nir_intrinsic_instr *instr,
187 bool is_shared_or_scratch)
188 {
189 uint32_t tmu_op = v3d_general_tmu_op(instr);
190
191 /* If we were able to replace atomic_add for an inc/dec, then we
192 * need/can to do things slightly different, like not loading the
193 * amount to add/sub, as that is implicit.
194 */
195 bool atomic_add_replaced =
196 ((instr->intrinsic == nir_intrinsic_ssbo_atomic_add ||
197 instr->intrinsic == nir_intrinsic_shared_atomic_add) &&
198 (tmu_op == V3D_TMU_OP_WRITE_AND_READ_INC ||
199 tmu_op == V3D_TMU_OP_WRITE_OR_READ_DEC));
200
201 bool is_store = (instr->intrinsic == nir_intrinsic_store_ssbo ||
202 instr->intrinsic == nir_intrinsic_store_scratch ||
203 instr->intrinsic == nir_intrinsic_store_shared);
204
205 bool is_load = (instr->intrinsic == nir_intrinsic_load_uniform ||
206 instr->intrinsic == nir_intrinsic_load_ubo ||
207 instr->intrinsic == nir_intrinsic_load_ssbo ||
208 instr->intrinsic == nir_intrinsic_load_scratch ||
209 instr->intrinsic == nir_intrinsic_load_shared);
210
211 if (!is_load)
212 c->tmu_dirty_rcl = true;
213
214 bool has_index = !is_shared_or_scratch;
215
216 int offset_src;
217 if (instr->intrinsic == nir_intrinsic_load_uniform) {
218 offset_src = 0;
219 } else if (instr->intrinsic == nir_intrinsic_load_ssbo ||
220 instr->intrinsic == nir_intrinsic_load_ubo ||
221 instr->intrinsic == nir_intrinsic_load_scratch ||
222 instr->intrinsic == nir_intrinsic_load_shared ||
223 atomic_add_replaced) {
224 offset_src = 0 + has_index;
225 } else if (is_store) {
226 offset_src = 1 + has_index;
227 } else {
228 offset_src = 0 + has_index;
229 }
230
231 bool dynamic_src = !nir_src_is_const(instr->src[offset_src]);
232 uint32_t const_offset = 0;
233 if (!dynamic_src)
234 const_offset = nir_src_as_uint(instr->src[offset_src]);
235
236 struct qreg base_offset;
237 if (instr->intrinsic == nir_intrinsic_load_uniform) {
238 const_offset += nir_intrinsic_base(instr);
239 base_offset = vir_uniform(c, QUNIFORM_UBO_ADDR,
240 v3d_unit_data_create(0, const_offset));
241 const_offset = 0;
242 } else if (instr->intrinsic == nir_intrinsic_load_ubo) {
243 uint32_t index = nir_src_as_uint(instr->src[0]) + 1;
244 /* Note that QUNIFORM_UBO_ADDR takes a UBO index shifted up by
245 * 1 (0 is gallium's constant buffer 0).
246 */
247 base_offset =
248 vir_uniform(c, QUNIFORM_UBO_ADDR,
249 v3d_unit_data_create(index, const_offset));
250 const_offset = 0;
251 } else if (is_shared_or_scratch) {
252 /* Shared and scratch variables have no buffer index, and all
253 * start from a common base that we set up at the start of
254 * dispatch.
255 */
256 if (instr->intrinsic == nir_intrinsic_load_scratch ||
257 instr->intrinsic == nir_intrinsic_store_scratch) {
258 base_offset = c->spill_base;
259 } else {
260 base_offset = c->cs_shared_offset;
261 const_offset += nir_intrinsic_base(instr);
262 }
263 } else {
264 base_offset = vir_uniform(c, QUNIFORM_SSBO_OFFSET,
265 nir_src_as_uint(instr->src[is_store ?
266 1 : 0]));
267 }
268
269 struct qreg tmud = vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_TMUD);
270 unsigned writemask = is_store ? nir_intrinsic_write_mask(instr) : 0;
271 uint32_t base_const_offset = const_offset;
272 int first_component = -1;
273 int last_component = -1;
274 do {
275 int tmu_writes = 1; /* address */
276
277 if (is_store) {
278 /* Find the first set of consecutive components that
279 * are enabled in the writemask and emit the TMUD
280 * instructions for them.
281 */
282 first_component = ffs(writemask) - 1;
283 last_component = first_component;
284 while (writemask & BITFIELD_BIT(last_component + 1))
285 last_component++;
286
287 assert(first_component >= 0 &&
288 first_component <= last_component &&
289 last_component < instr->num_components);
290
291 struct qreg tmud = vir_reg(QFILE_MAGIC,
292 V3D_QPU_WADDR_TMUD);
293 for (int i = first_component; i <= last_component; i++) {
294 struct qreg data =
295 ntq_get_src(c, instr->src[0], i);
296 vir_MOV_dest(c, tmud, data);
297 tmu_writes++;
298 }
299
300 /* Update the offset for the TMU write based on the
301 * the first component we are writing.
302 */
303 const_offset = base_const_offset + first_component * 4;
304
305 /* Clear these components from the writemask */
306 uint32_t written_mask =
307 BITFIELD_RANGE(first_component, tmu_writes - 1);
308 writemask &= ~written_mask;
309 } else if (!is_load && !atomic_add_replaced) {
310 struct qreg data =
311 ntq_get_src(c, instr->src[1 + has_index], 0);
312 vir_MOV_dest(c, tmud, data);
313 tmu_writes++;
314 if (tmu_op == V3D_TMU_OP_WRITE_CMPXCHG_READ_FLUSH) {
315 data = ntq_get_src(c, instr->src[2 + has_index],
316 0);
317 vir_MOV_dest(c, tmud, data);
318 tmu_writes++;
319 }
320 }
321
322 /* Make sure we won't exceed the 16-entry TMU fifo if each
323 * thread is storing at the same time.
324 */
325 while (tmu_writes > 16 / c->threads)
326 c->threads /= 2;
327
328 /* The spec says that for atomics, the TYPE field is ignored,
329 * but that doesn't seem to be the case for CMPXCHG. Just use
330 * the number of tmud writes we did to decide the type (or
331 * choose "32bit" for atomic reads, which has been fine).
332 */
333 uint32_t num_components;
334 if (is_load || atomic_add_replaced) {
335 num_components = instr->num_components;
336 } else {
337 assert(tmu_writes > 1);
338 num_components = tmu_writes - 1;
339 }
340
341 uint32_t config = (0xffffff00 |
342 tmu_op << 3|
343 GENERAL_TMU_LOOKUP_PER_PIXEL);
344 if (num_components == 1) {
345 config |= GENERAL_TMU_LOOKUP_TYPE_32BIT_UI;
346 } else {
347 config |= GENERAL_TMU_LOOKUP_TYPE_VEC2 +
348 num_components - 2;
349 }
350
351 if (vir_in_nonuniform_control_flow(c)) {
352 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute),
353 V3D_QPU_PF_PUSHZ);
354 }
355
356 struct qreg tmua;
357 if (config == ~0)
358 tmua = vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_TMUA);
359 else
360 tmua = vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_TMUAU);
361
362 struct qinst *tmu;
363 if (dynamic_src) {
364 struct qreg offset = base_offset;
365 if (const_offset != 0) {
366 offset = vir_ADD(c, offset,
367 vir_uniform_ui(c, const_offset));
368 }
369 struct qreg data =
370 ntq_get_src(c, instr->src[offset_src], 0);
371 tmu = vir_ADD_dest(c, tmua, offset, data);
372 } else {
373 if (const_offset != 0) {
374 tmu = vir_ADD_dest(c, tmua, base_offset,
375 vir_uniform_ui(c, const_offset));
376 } else {
377 tmu = vir_MOV_dest(c, tmua, base_offset);
378 }
379 }
380
381 if (config != ~0) {
382 tmu->uniform =
383 vir_get_uniform_index(c, QUNIFORM_CONSTANT,
384 config);
385 }
386
387 if (vir_in_nonuniform_control_flow(c))
388 vir_set_cond(tmu, V3D_QPU_COND_IFA);
389
390 vir_emit_thrsw(c);
391
392 /* Read the result, or wait for the TMU op to complete. */
393 for (int i = 0; i < nir_intrinsic_dest_components(instr); i++) {
394 ntq_store_dest(c, &instr->dest, i,
395 vir_MOV(c, vir_LDTMU(c)));
396 }
397
398 if (nir_intrinsic_dest_components(instr) == 0)
399 vir_TMUWT(c);
400 } while (is_store && writemask != 0);
401 }
402
403 static struct qreg *
404 ntq_init_ssa_def(struct v3d_compile *c, nir_ssa_def *def)
405 {
406 struct qreg *qregs = ralloc_array(c->def_ht, struct qreg,
407 def->num_components);
408 _mesa_hash_table_insert(c->def_ht, def, qregs);
409 return qregs;
410 }
411
412 /**
413 * This function is responsible for getting VIR results into the associated
414 * storage for a NIR instruction.
415 *
416 * If it's a NIR SSA def, then we just set the associated hash table entry to
417 * the new result.
418 *
419 * If it's a NIR reg, then we need to update the existing qreg assigned to the
420 * NIR destination with the incoming value. To do that without introducing
421 * new MOVs, we require that the incoming qreg either be a uniform, or be
422 * SSA-defined by the previous VIR instruction in the block and rewritable by
423 * this function. That lets us sneak ahead and insert the SF flag beforehand
424 * (knowing that the previous instruction doesn't depend on flags) and rewrite
425 * its destination to be the NIR reg's destination
426 */
427 void
428 ntq_store_dest(struct v3d_compile *c, nir_dest *dest, int chan,
429 struct qreg result)
430 {
431 struct qinst *last_inst = NULL;
432 if (!list_is_empty(&c->cur_block->instructions))
433 last_inst = (struct qinst *)c->cur_block->instructions.prev;
434
435 assert((result.file == QFILE_TEMP &&
436 last_inst && last_inst == c->defs[result.index]));
437
438 if (dest->is_ssa) {
439 assert(chan < dest->ssa.num_components);
440
441 struct qreg *qregs;
442 struct hash_entry *entry =
443 _mesa_hash_table_search(c->def_ht, &dest->ssa);
444
445 if (entry)
446 qregs = entry->data;
447 else
448 qregs = ntq_init_ssa_def(c, &dest->ssa);
449
450 qregs[chan] = result;
451 } else {
452 nir_register *reg = dest->reg.reg;
453 assert(dest->reg.base_offset == 0);
454 assert(reg->num_array_elems == 0);
455 struct hash_entry *entry =
456 _mesa_hash_table_search(c->def_ht, reg);
457 struct qreg *qregs = entry->data;
458
459 /* Insert a MOV if the source wasn't an SSA def in the
460 * previous instruction.
461 */
462 if ((vir_in_nonuniform_control_flow(c) &&
463 c->defs[last_inst->dst.index]->qpu.sig.ldunif)) {
464 result = vir_MOV(c, result);
465 last_inst = c->defs[result.index];
466 }
467
468 /* We know they're both temps, so just rewrite index. */
469 c->defs[last_inst->dst.index] = NULL;
470 last_inst->dst.index = qregs[chan].index;
471
472 /* If we're in control flow, then make this update of the reg
473 * conditional on the execution mask.
474 */
475 if (vir_in_nonuniform_control_flow(c)) {
476 last_inst->dst.index = qregs[chan].index;
477
478 /* Set the flags to the current exec mask.
479 */
480 c->cursor = vir_before_inst(last_inst);
481 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute),
482 V3D_QPU_PF_PUSHZ);
483 c->cursor = vir_after_inst(last_inst);
484
485 vir_set_cond(last_inst, V3D_QPU_COND_IFA);
486 }
487 }
488 }
489
490 struct qreg
491 ntq_get_src(struct v3d_compile *c, nir_src src, int i)
492 {
493 struct hash_entry *entry;
494 if (src.is_ssa) {
495 entry = _mesa_hash_table_search(c->def_ht, src.ssa);
496 assert(i < src.ssa->num_components);
497 } else {
498 nir_register *reg = src.reg.reg;
499 entry = _mesa_hash_table_search(c->def_ht, reg);
500 assert(reg->num_array_elems == 0);
501 assert(src.reg.base_offset == 0);
502 assert(i < reg->num_components);
503 }
504
505 struct qreg *qregs = entry->data;
506 return qregs[i];
507 }
508
509 static struct qreg
510 ntq_get_alu_src(struct v3d_compile *c, nir_alu_instr *instr,
511 unsigned src)
512 {
513 assert(util_is_power_of_two_or_zero(instr->dest.write_mask));
514 unsigned chan = ffs(instr->dest.write_mask) - 1;
515 struct qreg r = ntq_get_src(c, instr->src[src].src,
516 instr->src[src].swizzle[chan]);
517
518 assert(!instr->src[src].abs);
519 assert(!instr->src[src].negate);
520
521 return r;
522 };
523
524 static struct qreg
525 ntq_minify(struct v3d_compile *c, struct qreg size, struct qreg level)
526 {
527 return vir_MAX(c, vir_SHR(c, size, level), vir_uniform_ui(c, 1));
528 }
529
530 static void
531 ntq_emit_txs(struct v3d_compile *c, nir_tex_instr *instr)
532 {
533 unsigned unit = instr->texture_index;
534 int lod_index = nir_tex_instr_src_index(instr, nir_tex_src_lod);
535 int dest_size = nir_tex_instr_dest_size(instr);
536
537 struct qreg lod = c->undef;
538 if (lod_index != -1)
539 lod = ntq_get_src(c, instr->src[lod_index].src, 0);
540
541 for (int i = 0; i < dest_size; i++) {
542 assert(i < 3);
543 enum quniform_contents contents;
544
545 if (instr->is_array && i == dest_size - 1)
546 contents = QUNIFORM_TEXTURE_ARRAY_SIZE;
547 else
548 contents = QUNIFORM_TEXTURE_WIDTH + i;
549
550 struct qreg size = vir_uniform(c, contents, unit);
551
552 switch (instr->sampler_dim) {
553 case GLSL_SAMPLER_DIM_1D:
554 case GLSL_SAMPLER_DIM_2D:
555 case GLSL_SAMPLER_DIM_MS:
556 case GLSL_SAMPLER_DIM_3D:
557 case GLSL_SAMPLER_DIM_CUBE:
558 /* Don't minify the array size. */
559 if (!(instr->is_array && i == dest_size - 1)) {
560 size = ntq_minify(c, size, lod);
561 }
562 break;
563
564 case GLSL_SAMPLER_DIM_RECT:
565 /* There's no LOD field for rects */
566 break;
567
568 default:
569 unreachable("Bad sampler type");
570 }
571
572 ntq_store_dest(c, &instr->dest, i, size);
573 }
574 }
575
576 static void
577 ntq_emit_tex(struct v3d_compile *c, nir_tex_instr *instr)
578 {
579 unsigned unit = instr->texture_index;
580
581 /* Since each texture sampling op requires uploading uniforms to
582 * reference the texture, there's no HW support for texture size and
583 * you just upload uniforms containing the size.
584 */
585 switch (instr->op) {
586 case nir_texop_query_levels:
587 ntq_store_dest(c, &instr->dest, 0,
588 vir_uniform(c, QUNIFORM_TEXTURE_LEVELS, unit));
589 return;
590 case nir_texop_txs:
591 ntq_emit_txs(c, instr);
592 return;
593 default:
594 break;
595 }
596
597 if (c->devinfo->ver >= 40)
598 v3d40_vir_emit_tex(c, instr);
599 else
600 v3d33_vir_emit_tex(c, instr);
601 }
602
603 static struct qreg
604 ntq_fsincos(struct v3d_compile *c, struct qreg src, bool is_cos)
605 {
606 struct qreg input = vir_FMUL(c, src, vir_uniform_f(c, 1.0f / M_PI));
607 if (is_cos)
608 input = vir_FADD(c, input, vir_uniform_f(c, 0.5));
609
610 struct qreg periods = vir_FROUND(c, input);
611 struct qreg sin_output = vir_SIN(c, vir_FSUB(c, input, periods));
612 return vir_XOR(c, sin_output, vir_SHL(c,
613 vir_FTOIN(c, periods),
614 vir_uniform_ui(c, -1)));
615 }
616
617 static struct qreg
618 ntq_fsign(struct v3d_compile *c, struct qreg src)
619 {
620 struct qreg t = vir_get_temp(c);
621
622 vir_MOV_dest(c, t, vir_uniform_f(c, 0.0));
623 vir_set_pf(vir_FMOV_dest(c, vir_nop_reg(), src), V3D_QPU_PF_PUSHZ);
624 vir_MOV_cond(c, V3D_QPU_COND_IFNA, t, vir_uniform_f(c, 1.0));
625 vir_set_pf(vir_FMOV_dest(c, vir_nop_reg(), src), V3D_QPU_PF_PUSHN);
626 vir_MOV_cond(c, V3D_QPU_COND_IFA, t, vir_uniform_f(c, -1.0));
627 return vir_MOV(c, t);
628 }
629
630 static void
631 emit_fragcoord_input(struct v3d_compile *c, int attr)
632 {
633 c->inputs[attr * 4 + 0] = vir_FXCD(c);
634 c->inputs[attr * 4 + 1] = vir_FYCD(c);
635 c->inputs[attr * 4 + 2] = c->payload_z;
636 c->inputs[attr * 4 + 3] = vir_RECIP(c, c->payload_w);
637 }
638
639 static struct qreg
640 emit_fragment_varying(struct v3d_compile *c, nir_variable *var,
641 uint8_t swizzle, int array_index)
642 {
643 struct qreg r3 = vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R3);
644 struct qreg r5 = vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R5);
645
646 struct qreg vary;
647 if (c->devinfo->ver >= 41) {
648 struct qinst *ldvary = vir_add_inst(V3D_QPU_A_NOP, c->undef,
649 c->undef, c->undef);
650 ldvary->qpu.sig.ldvary = true;
651 vary = vir_emit_def(c, ldvary);
652 } else {
653 vir_NOP(c)->qpu.sig.ldvary = true;
654 vary = r3;
655 }
656
657 /* For gl_PointCoord input or distance along a line, we'll be called
658 * with no nir_variable, and we don't count toward VPM size so we
659 * don't track an input slot.
660 */
661 if (!var) {
662 return vir_FADD(c, vir_FMUL(c, vary, c->payload_w), r5);
663 }
664
665 int i = c->num_inputs++;
666 c->input_slots[i] =
667 v3d_slot_from_slot_and_component(var->data.location +
668 array_index, swizzle);
669
670 switch (var->data.interpolation) {
671 case INTERP_MODE_NONE:
672 /* If a gl_FrontColor or gl_BackColor input has no interp
673 * qualifier, then if we're using glShadeModel(GL_FLAT) it
674 * needs to be flat shaded.
675 */
676 switch (var->data.location + array_index) {
677 case VARYING_SLOT_COL0:
678 case VARYING_SLOT_COL1:
679 case VARYING_SLOT_BFC0:
680 case VARYING_SLOT_BFC1:
681 if (c->fs_key->shade_model_flat) {
682 BITSET_SET(c->flat_shade_flags, i);
683 vir_MOV_dest(c, c->undef, vary);
684 return vir_MOV(c, r5);
685 } else {
686 return vir_FADD(c, vir_FMUL(c, vary,
687 c->payload_w), r5);
688 }
689 default:
690 break;
691 }
692 /* FALLTHROUGH */
693 case INTERP_MODE_SMOOTH:
694 if (var->data.centroid) {
695 BITSET_SET(c->centroid_flags, i);
696 return vir_FADD(c, vir_FMUL(c, vary,
697 c->payload_w_centroid), r5);
698 } else {
699 return vir_FADD(c, vir_FMUL(c, vary, c->payload_w), r5);
700 }
701 case INTERP_MODE_NOPERSPECTIVE:
702 BITSET_SET(c->noperspective_flags, i);
703 return vir_FADD(c, vir_MOV(c, vary), r5);
704 case INTERP_MODE_FLAT:
705 BITSET_SET(c->flat_shade_flags, i);
706 vir_MOV_dest(c, c->undef, vary);
707 return vir_MOV(c, r5);
708 default:
709 unreachable("Bad interp mode");
710 }
711 }
712
713 static void
714 emit_fragment_input(struct v3d_compile *c, int attr, nir_variable *var,
715 int array_index)
716 {
717 for (int i = 0; i < glsl_get_vector_elements(var->type); i++) {
718 int chan = var->data.location_frac + i;
719 c->inputs[attr * 4 + chan] =
720 emit_fragment_varying(c, var, chan, array_index);
721 }
722 }
723
724 static void
725 add_output(struct v3d_compile *c,
726 uint32_t decl_offset,
727 uint8_t slot,
728 uint8_t swizzle)
729 {
730 uint32_t old_array_size = c->outputs_array_size;
731 resize_qreg_array(c, &c->outputs, &c->outputs_array_size,
732 decl_offset + 1);
733
734 if (old_array_size != c->outputs_array_size) {
735 c->output_slots = reralloc(c,
736 c->output_slots,
737 struct v3d_varying_slot,
738 c->outputs_array_size);
739 }
740
741 c->output_slots[decl_offset] =
742 v3d_slot_from_slot_and_component(slot, swizzle);
743 }
744
745 /**
746 * If compare_instr is a valid comparison instruction, emits the
747 * compare_instr's comparison and returns the sel_instr's return value based
748 * on the compare_instr's result.
749 */
750 static bool
751 ntq_emit_comparison(struct v3d_compile *c,
752 nir_alu_instr *compare_instr,
753 enum v3d_qpu_cond *out_cond)
754 {
755 struct qreg src0 = ntq_get_alu_src(c, compare_instr, 0);
756 struct qreg src1;
757 if (nir_op_infos[compare_instr->op].num_inputs > 1)
758 src1 = ntq_get_alu_src(c, compare_instr, 1);
759 bool cond_invert = false;
760 struct qreg nop = vir_nop_reg();
761
762 switch (compare_instr->op) {
763 case nir_op_feq32:
764 case nir_op_seq:
765 vir_set_pf(vir_FCMP_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHZ);
766 break;
767 case nir_op_ieq32:
768 vir_set_pf(vir_XOR_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHZ);
769 break;
770
771 case nir_op_fne32:
772 case nir_op_sne:
773 vir_set_pf(vir_FCMP_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHZ);
774 cond_invert = true;
775 break;
776 case nir_op_ine32:
777 vir_set_pf(vir_XOR_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHZ);
778 cond_invert = true;
779 break;
780
781 case nir_op_fge32:
782 case nir_op_sge:
783 vir_set_pf(vir_FCMP_dest(c, nop, src1, src0), V3D_QPU_PF_PUSHC);
784 break;
785 case nir_op_ige32:
786 vir_set_pf(vir_MIN_dest(c, nop, src1, src0), V3D_QPU_PF_PUSHC);
787 cond_invert = true;
788 break;
789 case nir_op_uge32:
790 vir_set_pf(vir_SUB_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHC);
791 cond_invert = true;
792 break;
793
794 case nir_op_slt:
795 case nir_op_flt32:
796 vir_set_pf(vir_FCMP_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHN);
797 break;
798 case nir_op_ilt32:
799 vir_set_pf(vir_MIN_dest(c, nop, src1, src0), V3D_QPU_PF_PUSHC);
800 break;
801 case nir_op_ult32:
802 vir_set_pf(vir_SUB_dest(c, nop, src0, src1), V3D_QPU_PF_PUSHC);
803 break;
804
805 case nir_op_i2b32:
806 vir_set_pf(vir_MOV_dest(c, nop, src0), V3D_QPU_PF_PUSHZ);
807 cond_invert = true;
808 break;
809
810 case nir_op_f2b32:
811 vir_set_pf(vir_FMOV_dest(c, nop, src0), V3D_QPU_PF_PUSHZ);
812 cond_invert = true;
813 break;
814
815 default:
816 return false;
817 }
818
819 *out_cond = cond_invert ? V3D_QPU_COND_IFNA : V3D_QPU_COND_IFA;
820
821 return true;
822 }
823
824 /* Finds an ALU instruction that generates our src value that could
825 * (potentially) be greedily emitted in the consuming instruction.
826 */
827 static struct nir_alu_instr *
828 ntq_get_alu_parent(nir_src src)
829 {
830 if (!src.is_ssa || src.ssa->parent_instr->type != nir_instr_type_alu)
831 return NULL;
832 nir_alu_instr *instr = nir_instr_as_alu(src.ssa->parent_instr);
833 if (!instr)
834 return NULL;
835
836 /* If the ALU instr's srcs are non-SSA, then we would have to avoid
837 * moving emission of the ALU instr down past another write of the
838 * src.
839 */
840 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
841 if (!instr->src[i].src.is_ssa)
842 return NULL;
843 }
844
845 return instr;
846 }
847
848 /* Turns a NIR bool into a condition code to predicate on. */
849 static enum v3d_qpu_cond
850 ntq_emit_bool_to_cond(struct v3d_compile *c, nir_src src)
851 {
852 nir_alu_instr *compare = ntq_get_alu_parent(src);
853 if (!compare)
854 goto out;
855
856 enum v3d_qpu_cond cond;
857 if (ntq_emit_comparison(c, compare, &cond))
858 return cond;
859
860 out:
861 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), ntq_get_src(c, src, 0)),
862 V3D_QPU_PF_PUSHZ);
863 return V3D_QPU_COND_IFNA;
864 }
865
866 static void
867 ntq_emit_alu(struct v3d_compile *c, nir_alu_instr *instr)
868 {
869 /* This should always be lowered to ALU operations for V3D. */
870 assert(!instr->dest.saturate);
871
872 /* Vectors are special in that they have non-scalarized writemasks,
873 * and just take the first swizzle channel for each argument in order
874 * into each writemask channel.
875 */
876 if (instr->op == nir_op_vec2 ||
877 instr->op == nir_op_vec3 ||
878 instr->op == nir_op_vec4) {
879 struct qreg srcs[4];
880 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
881 srcs[i] = ntq_get_src(c, instr->src[i].src,
882 instr->src[i].swizzle[0]);
883 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
884 ntq_store_dest(c, &instr->dest.dest, i,
885 vir_MOV(c, srcs[i]));
886 return;
887 }
888
889 /* General case: We can just grab the one used channel per src. */
890 struct qreg src[nir_op_infos[instr->op].num_inputs];
891 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
892 src[i] = ntq_get_alu_src(c, instr, i);
893 }
894
895 struct qreg result;
896
897 switch (instr->op) {
898 case nir_op_mov:
899 result = vir_MOV(c, src[0]);
900 break;
901
902 case nir_op_fneg:
903 result = vir_XOR(c, src[0], vir_uniform_ui(c, 1 << 31));
904 break;
905 case nir_op_ineg:
906 result = vir_NEG(c, src[0]);
907 break;
908
909 case nir_op_fmul:
910 result = vir_FMUL(c, src[0], src[1]);
911 break;
912 case nir_op_fadd:
913 result = vir_FADD(c, src[0], src[1]);
914 break;
915 case nir_op_fsub:
916 result = vir_FSUB(c, src[0], src[1]);
917 break;
918 case nir_op_fmin:
919 result = vir_FMIN(c, src[0], src[1]);
920 break;
921 case nir_op_fmax:
922 result = vir_FMAX(c, src[0], src[1]);
923 break;
924
925 case nir_op_f2i32: {
926 nir_alu_instr *src0_alu = ntq_get_alu_parent(instr->src[0].src);
927 if (src0_alu && src0_alu->op == nir_op_fround_even) {
928 result = vir_FTOIN(c, ntq_get_alu_src(c, src0_alu, 0));
929 } else {
930 result = vir_FTOIZ(c, src[0]);
931 }
932 break;
933 }
934
935 case nir_op_f2u32:
936 result = vir_FTOUZ(c, src[0]);
937 break;
938 case nir_op_i2f32:
939 result = vir_ITOF(c, src[0]);
940 break;
941 case nir_op_u2f32:
942 result = vir_UTOF(c, src[0]);
943 break;
944 case nir_op_b2f32:
945 result = vir_AND(c, src[0], vir_uniform_f(c, 1.0));
946 break;
947 case nir_op_b2i32:
948 result = vir_AND(c, src[0], vir_uniform_ui(c, 1));
949 break;
950
951 case nir_op_iadd:
952 result = vir_ADD(c, src[0], src[1]);
953 break;
954 case nir_op_ushr:
955 result = vir_SHR(c, src[0], src[1]);
956 break;
957 case nir_op_isub:
958 result = vir_SUB(c, src[0], src[1]);
959 break;
960 case nir_op_ishr:
961 result = vir_ASR(c, src[0], src[1]);
962 break;
963 case nir_op_ishl:
964 result = vir_SHL(c, src[0], src[1]);
965 break;
966 case nir_op_imin:
967 result = vir_MIN(c, src[0], src[1]);
968 break;
969 case nir_op_umin:
970 result = vir_UMIN(c, src[0], src[1]);
971 break;
972 case nir_op_imax:
973 result = vir_MAX(c, src[0], src[1]);
974 break;
975 case nir_op_umax:
976 result = vir_UMAX(c, src[0], src[1]);
977 break;
978 case nir_op_iand:
979 result = vir_AND(c, src[0], src[1]);
980 break;
981 case nir_op_ior:
982 result = vir_OR(c, src[0], src[1]);
983 break;
984 case nir_op_ixor:
985 result = vir_XOR(c, src[0], src[1]);
986 break;
987 case nir_op_inot:
988 result = vir_NOT(c, src[0]);
989 break;
990
991 case nir_op_ufind_msb:
992 result = vir_SUB(c, vir_uniform_ui(c, 31), vir_CLZ(c, src[0]));
993 break;
994
995 case nir_op_imul:
996 result = vir_UMUL(c, src[0], src[1]);
997 break;
998
999 case nir_op_seq:
1000 case nir_op_sne:
1001 case nir_op_sge:
1002 case nir_op_slt: {
1003 enum v3d_qpu_cond cond;
1004 ASSERTED bool ok = ntq_emit_comparison(c, instr, &cond);
1005 assert(ok);
1006 result = vir_MOV(c, vir_SEL(c, cond,
1007 vir_uniform_f(c, 1.0),
1008 vir_uniform_f(c, 0.0)));
1009 break;
1010 }
1011
1012 case nir_op_i2b32:
1013 case nir_op_f2b32:
1014 case nir_op_feq32:
1015 case nir_op_fne32:
1016 case nir_op_fge32:
1017 case nir_op_flt32:
1018 case nir_op_ieq32:
1019 case nir_op_ine32:
1020 case nir_op_ige32:
1021 case nir_op_uge32:
1022 case nir_op_ilt32:
1023 case nir_op_ult32: {
1024 enum v3d_qpu_cond cond;
1025 ASSERTED bool ok = ntq_emit_comparison(c, instr, &cond);
1026 assert(ok);
1027 result = vir_MOV(c, vir_SEL(c, cond,
1028 vir_uniform_ui(c, ~0),
1029 vir_uniform_ui(c, 0)));
1030 break;
1031 }
1032
1033 case nir_op_b32csel:
1034 result = vir_MOV(c,
1035 vir_SEL(c,
1036 ntq_emit_bool_to_cond(c, instr->src[0].src),
1037 src[1], src[2]));
1038 break;
1039
1040 case nir_op_fcsel:
1041 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), src[0]),
1042 V3D_QPU_PF_PUSHZ);
1043 result = vir_MOV(c, vir_SEL(c, V3D_QPU_COND_IFNA,
1044 src[1], src[2]));
1045 break;
1046
1047 case nir_op_frcp:
1048 result = vir_RECIP(c, src[0]);
1049 break;
1050 case nir_op_frsq:
1051 result = vir_RSQRT(c, src[0]);
1052 break;
1053 case nir_op_fexp2:
1054 result = vir_EXP(c, src[0]);
1055 break;
1056 case nir_op_flog2:
1057 result = vir_LOG(c, src[0]);
1058 break;
1059
1060 case nir_op_fceil:
1061 result = vir_FCEIL(c, src[0]);
1062 break;
1063 case nir_op_ffloor:
1064 result = vir_FFLOOR(c, src[0]);
1065 break;
1066 case nir_op_fround_even:
1067 result = vir_FROUND(c, src[0]);
1068 break;
1069 case nir_op_ftrunc:
1070 result = vir_FTRUNC(c, src[0]);
1071 break;
1072
1073 case nir_op_fsin:
1074 result = ntq_fsincos(c, src[0], false);
1075 break;
1076 case nir_op_fcos:
1077 result = ntq_fsincos(c, src[0], true);
1078 break;
1079
1080 case nir_op_fsign:
1081 result = ntq_fsign(c, src[0]);
1082 break;
1083
1084 case nir_op_fabs: {
1085 result = vir_FMOV(c, src[0]);
1086 vir_set_unpack(c->defs[result.index], 0, V3D_QPU_UNPACK_ABS);
1087 break;
1088 }
1089
1090 case nir_op_iabs:
1091 result = vir_MAX(c, src[0], vir_NEG(c, src[0]));
1092 break;
1093
1094 case nir_op_fddx:
1095 case nir_op_fddx_coarse:
1096 case nir_op_fddx_fine:
1097 result = vir_FDX(c, src[0]);
1098 break;
1099
1100 case nir_op_fddy:
1101 case nir_op_fddy_coarse:
1102 case nir_op_fddy_fine:
1103 result = vir_FDY(c, src[0]);
1104 break;
1105
1106 case nir_op_uadd_carry:
1107 vir_set_pf(vir_ADD_dest(c, vir_nop_reg(), src[0], src[1]),
1108 V3D_QPU_PF_PUSHC);
1109 result = vir_MOV(c, vir_SEL(c, V3D_QPU_COND_IFA,
1110 vir_uniform_ui(c, ~0),
1111 vir_uniform_ui(c, 0)));
1112 break;
1113
1114 case nir_op_pack_half_2x16_split:
1115 result = vir_VFPACK(c, src[0], src[1]);
1116 break;
1117
1118 case nir_op_unpack_half_2x16_split_x:
1119 result = vir_FMOV(c, src[0]);
1120 vir_set_unpack(c->defs[result.index], 0, V3D_QPU_UNPACK_L);
1121 break;
1122
1123 case nir_op_unpack_half_2x16_split_y:
1124 result = vir_FMOV(c, src[0]);
1125 vir_set_unpack(c->defs[result.index], 0, V3D_QPU_UNPACK_H);
1126 break;
1127
1128 default:
1129 fprintf(stderr, "unknown NIR ALU inst: ");
1130 nir_print_instr(&instr->instr, stderr);
1131 fprintf(stderr, "\n");
1132 abort();
1133 }
1134
1135 /* We have a scalar result, so the instruction should only have a
1136 * single channel written to.
1137 */
1138 assert(util_is_power_of_two_or_zero(instr->dest.write_mask));
1139 ntq_store_dest(c, &instr->dest.dest,
1140 ffs(instr->dest.write_mask) - 1, result);
1141 }
1142
1143 /* Each TLB read/write setup (a render target or depth buffer) takes an 8-bit
1144 * specifier. They come from a register that's preloaded with 0xffffffff
1145 * (0xff gets you normal vec4 f16 RT0 writes), and when one is neaded the low
1146 * 8 bits are shifted off the bottom and 0xff shifted in from the top.
1147 */
1148 #define TLB_TYPE_F16_COLOR (3 << 6)
1149 #define TLB_TYPE_I32_COLOR (1 << 6)
1150 #define TLB_TYPE_F32_COLOR (0 << 6)
1151 #define TLB_RENDER_TARGET_SHIFT 3 /* Reversed! 7 = RT 0, 0 = RT 7. */
1152 #define TLB_SAMPLE_MODE_PER_SAMPLE (0 << 2)
1153 #define TLB_SAMPLE_MODE_PER_PIXEL (1 << 2)
1154 #define TLB_F16_SWAP_HI_LO (1 << 1)
1155 #define TLB_VEC_SIZE_4_F16 (1 << 0)
1156 #define TLB_VEC_SIZE_2_F16 (0 << 0)
1157 #define TLB_VEC_SIZE_MINUS_1_SHIFT 0
1158
1159 /* Triggers Z/Stencil testing, used when the shader state's "FS modifies Z"
1160 * flag is set.
1161 */
1162 #define TLB_TYPE_DEPTH ((2 << 6) | (0 << 4))
1163 #define TLB_DEPTH_TYPE_INVARIANT (0 << 2) /* Unmodified sideband input used */
1164 #define TLB_DEPTH_TYPE_PER_PIXEL (1 << 2) /* QPU result used */
1165 #define TLB_V42_DEPTH_TYPE_INVARIANT (0 << 3) /* Unmodified sideband input used */
1166 #define TLB_V42_DEPTH_TYPE_PER_PIXEL (1 << 3) /* QPU result used */
1167
1168 /* Stencil is a single 32-bit write. */
1169 #define TLB_TYPE_STENCIL_ALPHA ((2 << 6) | (1 << 4))
1170
1171 static void
1172 vir_emit_tlb_color_write(struct v3d_compile *c, unsigned rt)
1173 {
1174 if (!(c->fs_key->cbufs & (1 << rt)) || !c->output_color_var[rt])
1175 return;
1176
1177 struct qreg tlb_reg = vir_magic_reg(V3D_QPU_WADDR_TLB);
1178 struct qreg tlbu_reg = vir_magic_reg(V3D_QPU_WADDR_TLBU);
1179
1180 nir_variable *var = c->output_color_var[rt];
1181 int num_components = glsl_get_vector_elements(var->type);
1182 uint32_t conf = 0xffffff00;
1183 struct qinst *inst;
1184
1185 conf |= c->msaa_per_sample_output ? TLB_SAMPLE_MODE_PER_SAMPLE :
1186 TLB_SAMPLE_MODE_PER_PIXEL;
1187 conf |= (7 - rt) << TLB_RENDER_TARGET_SHIFT;
1188
1189 if (c->fs_key->swap_color_rb & (1 << rt))
1190 num_components = MAX2(num_components, 3);
1191 assert(num_components != 0);
1192
1193 enum glsl_base_type type = glsl_get_base_type(var->type);
1194 bool is_int_format = type == GLSL_TYPE_INT || type == GLSL_TYPE_UINT;
1195 bool is_32b_tlb_format = is_int_format ||
1196 (c->fs_key->f32_color_rb & (1 << rt));
1197
1198 if (is_int_format) {
1199 /* The F32 vs I32 distinction was dropped in 4.2. */
1200 if (c->devinfo->ver < 42)
1201 conf |= TLB_TYPE_I32_COLOR;
1202 else
1203 conf |= TLB_TYPE_F32_COLOR;
1204 conf |= ((num_components - 1) << TLB_VEC_SIZE_MINUS_1_SHIFT);
1205 } else {
1206 if (c->fs_key->f32_color_rb & (1 << rt)) {
1207 conf |= TLB_TYPE_F32_COLOR;
1208 conf |= ((num_components - 1) <<
1209 TLB_VEC_SIZE_MINUS_1_SHIFT);
1210 } else {
1211 conf |= TLB_TYPE_F16_COLOR;
1212 conf |= TLB_F16_SWAP_HI_LO;
1213 if (num_components >= 3)
1214 conf |= TLB_VEC_SIZE_4_F16;
1215 else
1216 conf |= TLB_VEC_SIZE_2_F16;
1217 }
1218 }
1219
1220 int num_samples = c->msaa_per_sample_output ? V3D_MAX_SAMPLES : 1;
1221 for (int i = 0; i < num_samples; i++) {
1222 struct qreg *color = c->msaa_per_sample_output ?
1223 &c->sample_colors[(rt * V3D_MAX_SAMPLES + i) * 4] :
1224 &c->outputs[var->data.driver_location * 4];
1225
1226 struct qreg r = color[0];
1227 struct qreg g = color[1];
1228 struct qreg b = color[2];
1229 struct qreg a = color[3];
1230
1231 if (c->fs_key->swap_color_rb & (1 << rt)) {
1232 r = color[2];
1233 b = color[0];
1234 }
1235
1236 if (c->fs_key->sample_alpha_to_one)
1237 a = vir_uniform_f(c, 1.0);
1238
1239 if (is_32b_tlb_format) {
1240 if (i == 0) {
1241 inst = vir_MOV_dest(c, tlbu_reg, r);
1242 inst->uniform =
1243 vir_get_uniform_index(c,
1244 QUNIFORM_CONSTANT,
1245 conf);
1246 } else {
1247 inst = vir_MOV_dest(c, tlb_reg, r);
1248 }
1249
1250 if (num_components >= 2)
1251 vir_MOV_dest(c, tlb_reg, g);
1252 if (num_components >= 3)
1253 vir_MOV_dest(c, tlb_reg, b);
1254 if (num_components >= 4)
1255 vir_MOV_dest(c, tlb_reg, a);
1256 } else {
1257 inst = vir_VFPACK_dest(c, tlb_reg, r, g);
1258 if (conf != ~0 && i == 0) {
1259 inst->dst = tlbu_reg;
1260 inst->uniform =
1261 vir_get_uniform_index(c,
1262 QUNIFORM_CONSTANT,
1263 conf);
1264 }
1265
1266 if (num_components >= 3)
1267 inst = vir_VFPACK_dest(c, tlb_reg, b, a);
1268 }
1269 }
1270 }
1271
1272 static void
1273 emit_frag_end(struct v3d_compile *c)
1274 {
1275 /* XXX
1276 if (c->output_sample_mask_index != -1) {
1277 vir_MS_MASK(c, c->outputs[c->output_sample_mask_index]);
1278 }
1279 */
1280
1281 bool has_any_tlb_color_write = false;
1282 for (int rt = 0; rt < V3D_MAX_DRAW_BUFFERS; rt++) {
1283 if (c->fs_key->cbufs & (1 << rt) && c->output_color_var[rt])
1284 has_any_tlb_color_write = true;
1285 }
1286
1287 if (c->fs_key->sample_alpha_to_coverage && c->output_color_var[0]) {
1288 struct nir_variable *var = c->output_color_var[0];
1289 struct qreg *color = &c->outputs[var->data.driver_location * 4];
1290
1291 vir_SETMSF_dest(c, vir_nop_reg(),
1292 vir_AND(c,
1293 vir_MSF(c),
1294 vir_FTOC(c, color[3])));
1295 }
1296
1297 struct qreg tlbu_reg = vir_magic_reg(V3D_QPU_WADDR_TLBU);
1298 if (c->output_position_index != -1) {
1299 struct qinst *inst = vir_MOV_dest(c, tlbu_reg,
1300 c->outputs[c->output_position_index]);
1301 uint8_t tlb_specifier = TLB_TYPE_DEPTH;
1302
1303 if (c->devinfo->ver >= 42) {
1304 tlb_specifier |= (TLB_V42_DEPTH_TYPE_PER_PIXEL |
1305 TLB_SAMPLE_MODE_PER_PIXEL);
1306 } else
1307 tlb_specifier |= TLB_DEPTH_TYPE_PER_PIXEL;
1308
1309 inst->uniform = vir_get_uniform_index(c, QUNIFORM_CONSTANT,
1310 tlb_specifier |
1311 0xffffff00);
1312 c->writes_z = true;
1313 } else if (c->s->info.fs.uses_discard ||
1314 !c->s->info.fs.early_fragment_tests ||
1315 c->fs_key->sample_alpha_to_coverage ||
1316 !has_any_tlb_color_write) {
1317 /* Emit passthrough Z if it needed to be delayed until shader
1318 * end due to potential discards.
1319 *
1320 * Since (single-threaded) fragment shaders always need a TLB
1321 * write, emit passthrouh Z if we didn't have any color
1322 * buffers and flag us as potentially discarding, so that we
1323 * can use Z as the TLB write.
1324 */
1325 c->s->info.fs.uses_discard = true;
1326
1327 struct qinst *inst = vir_MOV_dest(c, tlbu_reg,
1328 vir_nop_reg());
1329 uint8_t tlb_specifier = TLB_TYPE_DEPTH;
1330
1331 if (c->devinfo->ver >= 42) {
1332 /* The spec says the PER_PIXEL flag is ignored for
1333 * invariant writes, but the simulator demands it.
1334 */
1335 tlb_specifier |= (TLB_V42_DEPTH_TYPE_INVARIANT |
1336 TLB_SAMPLE_MODE_PER_PIXEL);
1337 } else {
1338 tlb_specifier |= TLB_DEPTH_TYPE_INVARIANT;
1339 }
1340
1341 inst->uniform = vir_get_uniform_index(c,
1342 QUNIFORM_CONSTANT,
1343 tlb_specifier |
1344 0xffffff00);
1345 c->writes_z = true;
1346 }
1347
1348 /* XXX: Performance improvement: Merge Z write and color writes TLB
1349 * uniform setup
1350 */
1351 for (int rt = 0; rt < V3D_MAX_DRAW_BUFFERS; rt++)
1352 vir_emit_tlb_color_write(c, rt);
1353 }
1354
1355 static void
1356 vir_VPM_WRITE(struct v3d_compile *c, struct qreg val, uint32_t vpm_index)
1357 {
1358 if (c->devinfo->ver >= 40) {
1359 vir_STVPMV(c, vir_uniform_ui(c, vpm_index), val);
1360 } else {
1361 /* XXX: v3d33_vir_vpm_write_setup(c); */
1362 vir_MOV_dest(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_VPM), val);
1363 }
1364 }
1365
1366 static void
1367 emit_vert_end(struct v3d_compile *c)
1368 {
1369 /* GFXH-1684: VPM writes need to be complete by the end of the shader.
1370 */
1371 if (c->devinfo->ver >= 40 && c->devinfo->ver <= 42)
1372 vir_VPMWT(c);
1373 }
1374
1375 void
1376 v3d_optimize_nir(struct nir_shader *s)
1377 {
1378 bool progress;
1379 unsigned lower_flrp =
1380 (s->options->lower_flrp16 ? 16 : 0) |
1381 (s->options->lower_flrp32 ? 32 : 0) |
1382 (s->options->lower_flrp64 ? 64 : 0);
1383
1384 do {
1385 progress = false;
1386
1387 NIR_PASS_V(s, nir_lower_vars_to_ssa);
1388 NIR_PASS(progress, s, nir_lower_alu_to_scalar, NULL, NULL);
1389 NIR_PASS(progress, s, nir_lower_phis_to_scalar);
1390 NIR_PASS(progress, s, nir_copy_prop);
1391 NIR_PASS(progress, s, nir_opt_remove_phis);
1392 NIR_PASS(progress, s, nir_opt_dce);
1393 NIR_PASS(progress, s, nir_opt_dead_cf);
1394 NIR_PASS(progress, s, nir_opt_cse);
1395 NIR_PASS(progress, s, nir_opt_peephole_select, 8, true, true);
1396 NIR_PASS(progress, s, nir_opt_algebraic);
1397 NIR_PASS(progress, s, nir_opt_constant_folding);
1398
1399 if (lower_flrp != 0) {
1400 bool lower_flrp_progress = false;
1401
1402 NIR_PASS(lower_flrp_progress, s, nir_lower_flrp,
1403 lower_flrp,
1404 false /* always_precise */,
1405 s->options->lower_ffma);
1406 if (lower_flrp_progress) {
1407 NIR_PASS(progress, s, nir_opt_constant_folding);
1408 progress = true;
1409 }
1410
1411 /* Nothing should rematerialize any flrps, so we only
1412 * need to do this lowering once.
1413 */
1414 lower_flrp = 0;
1415 }
1416
1417 NIR_PASS(progress, s, nir_opt_undef);
1418 } while (progress);
1419
1420 NIR_PASS(progress, s, nir_opt_move, nir_move_load_ubo);
1421 }
1422
1423 static int
1424 driver_location_compare(const void *in_a, const void *in_b)
1425 {
1426 const nir_variable *const *a = in_a;
1427 const nir_variable *const *b = in_b;
1428
1429 return (*a)->data.driver_location - (*b)->data.driver_location;
1430 }
1431
1432 static struct qreg
1433 ntq_emit_vpm_read(struct v3d_compile *c,
1434 uint32_t *num_components_queued,
1435 uint32_t *remaining,
1436 uint32_t vpm_index)
1437 {
1438 struct qreg vpm = vir_reg(QFILE_VPM, vpm_index);
1439
1440 if (c->devinfo->ver >= 40 ) {
1441 return vir_LDVPMV_IN(c,
1442 vir_uniform_ui(c,
1443 (*num_components_queued)++));
1444 }
1445
1446 if (*num_components_queued != 0) {
1447 (*num_components_queued)--;
1448 return vir_MOV(c, vpm);
1449 }
1450
1451 uint32_t num_components = MIN2(*remaining, 32);
1452
1453 v3d33_vir_vpm_read_setup(c, num_components);
1454
1455 *num_components_queued = num_components - 1;
1456 *remaining -= num_components;
1457
1458 return vir_MOV(c, vpm);
1459 }
1460
1461 static void
1462 ntq_setup_vpm_inputs(struct v3d_compile *c)
1463 {
1464 /* Figure out how many components of each vertex attribute the shader
1465 * uses. Each variable should have been split to individual
1466 * components and unused ones DCEed. The vertex fetcher will load
1467 * from the start of the attribute to the number of components we
1468 * declare we need in c->vattr_sizes[].
1469 */
1470 nir_foreach_variable(var, &c->s->inputs) {
1471 /* No VS attribute array support. */
1472 assert(MAX2(glsl_get_length(var->type), 1) == 1);
1473
1474 unsigned loc = var->data.driver_location;
1475 int start_component = var->data.location_frac;
1476 int num_components = glsl_get_components(var->type);
1477
1478 c->vattr_sizes[loc] = MAX2(c->vattr_sizes[loc],
1479 start_component + num_components);
1480 }
1481
1482 unsigned num_components = 0;
1483 uint32_t vpm_components_queued = 0;
1484 bool uses_iid = c->s->info.system_values_read &
1485 (1ull << SYSTEM_VALUE_INSTANCE_ID);
1486 bool uses_vid = c->s->info.system_values_read &
1487 (1ull << SYSTEM_VALUE_VERTEX_ID);
1488 num_components += uses_iid;
1489 num_components += uses_vid;
1490
1491 for (int i = 0; i < ARRAY_SIZE(c->vattr_sizes); i++)
1492 num_components += c->vattr_sizes[i];
1493
1494 if (uses_iid) {
1495 c->iid = ntq_emit_vpm_read(c, &vpm_components_queued,
1496 &num_components, ~0);
1497 }
1498
1499 if (uses_vid) {
1500 c->vid = ntq_emit_vpm_read(c, &vpm_components_queued,
1501 &num_components, ~0);
1502 }
1503
1504 /* The actual loads will happen directly in nir_intrinsic_load_input
1505 * on newer versions.
1506 */
1507 if (c->devinfo->ver >= 40)
1508 return;
1509
1510 for (int loc = 0; loc < ARRAY_SIZE(c->vattr_sizes); loc++) {
1511 resize_qreg_array(c, &c->inputs, &c->inputs_array_size,
1512 (loc + 1) * 4);
1513
1514 for (int i = 0; i < c->vattr_sizes[loc]; i++) {
1515 c->inputs[loc * 4 + i] =
1516 ntq_emit_vpm_read(c,
1517 &vpm_components_queued,
1518 &num_components,
1519 loc * 4 + i);
1520
1521 }
1522 }
1523
1524 if (c->devinfo->ver >= 40) {
1525 assert(vpm_components_queued == num_components);
1526 } else {
1527 assert(vpm_components_queued == 0);
1528 assert(num_components == 0);
1529 }
1530 }
1531
1532 static bool
1533 var_needs_point_coord(struct v3d_compile *c, nir_variable *var)
1534 {
1535 return (var->data.location == VARYING_SLOT_PNTC ||
1536 (var->data.location >= VARYING_SLOT_VAR0 &&
1537 (c->fs_key->point_sprite_mask &
1538 (1 << (var->data.location - VARYING_SLOT_VAR0)))));
1539 }
1540
1541 static bool
1542 program_reads_point_coord(struct v3d_compile *c)
1543 {
1544 nir_foreach_variable(var, &c->s->inputs) {
1545 if (var_needs_point_coord(c, var))
1546 return true;
1547 }
1548
1549 return false;
1550 }
1551
1552 static void
1553 ntq_setup_fs_inputs(struct v3d_compile *c)
1554 {
1555 unsigned num_entries = 0;
1556 unsigned num_components = 0;
1557 nir_foreach_variable(var, &c->s->inputs) {
1558 num_entries++;
1559 num_components += glsl_get_components(var->type);
1560 }
1561
1562 nir_variable *vars[num_entries];
1563
1564 unsigned i = 0;
1565 nir_foreach_variable(var, &c->s->inputs)
1566 vars[i++] = var;
1567
1568 /* Sort the variables so that we emit the input setup in
1569 * driver_location order. This is required for VPM reads, whose data
1570 * is fetched into the VPM in driver_location (TGSI register index)
1571 * order.
1572 */
1573 qsort(&vars, num_entries, sizeof(*vars), driver_location_compare);
1574
1575 for (unsigned i = 0; i < num_entries; i++) {
1576 nir_variable *var = vars[i];
1577 unsigned array_len = MAX2(glsl_get_length(var->type), 1);
1578 unsigned loc = var->data.driver_location;
1579
1580 resize_qreg_array(c, &c->inputs, &c->inputs_array_size,
1581 (loc + array_len) * 4);
1582
1583 if (var->data.location == VARYING_SLOT_POS) {
1584 emit_fragcoord_input(c, loc);
1585 } else if (var_needs_point_coord(c, var)) {
1586 c->inputs[loc * 4 + 0] = c->point_x;
1587 c->inputs[loc * 4 + 1] = c->point_y;
1588 } else {
1589 for (int j = 0; j < array_len; j++)
1590 emit_fragment_input(c, loc + j, var, j);
1591 }
1592 }
1593 }
1594
1595 static void
1596 ntq_setup_outputs(struct v3d_compile *c)
1597 {
1598 if (c->s->info.stage != MESA_SHADER_FRAGMENT)
1599 return;
1600
1601 nir_foreach_variable(var, &c->s->outputs) {
1602 unsigned array_len = MAX2(glsl_get_length(var->type), 1);
1603 unsigned loc = var->data.driver_location * 4;
1604
1605 assert(array_len == 1);
1606 (void)array_len;
1607
1608 for (int i = 0; i < 4 - var->data.location_frac; i++) {
1609 add_output(c, loc + var->data.location_frac + i,
1610 var->data.location,
1611 var->data.location_frac + i);
1612 }
1613
1614 switch (var->data.location) {
1615 case FRAG_RESULT_COLOR:
1616 c->output_color_var[0] = var;
1617 c->output_color_var[1] = var;
1618 c->output_color_var[2] = var;
1619 c->output_color_var[3] = var;
1620 break;
1621 case FRAG_RESULT_DATA0:
1622 case FRAG_RESULT_DATA1:
1623 case FRAG_RESULT_DATA2:
1624 case FRAG_RESULT_DATA3:
1625 c->output_color_var[var->data.location -
1626 FRAG_RESULT_DATA0] = var;
1627 break;
1628 case FRAG_RESULT_DEPTH:
1629 c->output_position_index = loc;
1630 break;
1631 case FRAG_RESULT_SAMPLE_MASK:
1632 c->output_sample_mask_index = loc;
1633 break;
1634 }
1635 }
1636 }
1637
1638 /**
1639 * Sets up the mapping from nir_register to struct qreg *.
1640 *
1641 * Each nir_register gets a struct qreg per 32-bit component being stored.
1642 */
1643 static void
1644 ntq_setup_registers(struct v3d_compile *c, struct exec_list *list)
1645 {
1646 foreach_list_typed(nir_register, nir_reg, node, list) {
1647 unsigned array_len = MAX2(nir_reg->num_array_elems, 1);
1648 struct qreg *qregs = ralloc_array(c->def_ht, struct qreg,
1649 array_len *
1650 nir_reg->num_components);
1651
1652 _mesa_hash_table_insert(c->def_ht, nir_reg, qregs);
1653
1654 for (int i = 0; i < array_len * nir_reg->num_components; i++)
1655 qregs[i] = vir_get_temp(c);
1656 }
1657 }
1658
1659 static void
1660 ntq_emit_load_const(struct v3d_compile *c, nir_load_const_instr *instr)
1661 {
1662 /* XXX perf: Experiment with using immediate loads to avoid having
1663 * these end up in the uniform stream. Watch out for breaking the
1664 * small immediates optimization in the process!
1665 */
1666 struct qreg *qregs = ntq_init_ssa_def(c, &instr->def);
1667 for (int i = 0; i < instr->def.num_components; i++)
1668 qregs[i] = vir_uniform_ui(c, instr->value[i].u32);
1669
1670 _mesa_hash_table_insert(c->def_ht, &instr->def, qregs);
1671 }
1672
1673 static void
1674 ntq_emit_ssa_undef(struct v3d_compile *c, nir_ssa_undef_instr *instr)
1675 {
1676 struct qreg *qregs = ntq_init_ssa_def(c, &instr->def);
1677
1678 /* VIR needs there to be *some* value, so pick 0 (same as for
1679 * ntq_setup_registers().
1680 */
1681 for (int i = 0; i < instr->def.num_components; i++)
1682 qregs[i] = vir_uniform_ui(c, 0);
1683 }
1684
1685 static void
1686 ntq_emit_image_size(struct v3d_compile *c, nir_intrinsic_instr *instr)
1687 {
1688 assert(instr->intrinsic == nir_intrinsic_image_deref_size);
1689 nir_variable *var = nir_intrinsic_get_var(instr, 0);
1690 unsigned image_index = var->data.driver_location;
1691 const struct glsl_type *sampler_type = glsl_without_array(var->type);
1692 bool is_array = glsl_sampler_type_is_array(sampler_type);
1693
1694 ntq_store_dest(c, &instr->dest, 0,
1695 vir_uniform(c, QUNIFORM_IMAGE_WIDTH, image_index));
1696 if (instr->num_components > 1) {
1697 ntq_store_dest(c, &instr->dest, 1,
1698 vir_uniform(c, QUNIFORM_IMAGE_HEIGHT,
1699 image_index));
1700 }
1701 if (instr->num_components > 2) {
1702 ntq_store_dest(c, &instr->dest, 2,
1703 vir_uniform(c,
1704 is_array ?
1705 QUNIFORM_IMAGE_ARRAY_SIZE :
1706 QUNIFORM_IMAGE_DEPTH,
1707 image_index));
1708 }
1709 }
1710
1711 static void
1712 vir_emit_tlb_color_read(struct v3d_compile *c, nir_intrinsic_instr *instr)
1713 {
1714 assert(c->s->info.stage == MESA_SHADER_FRAGMENT);
1715
1716 int rt = nir_src_as_uint(instr->src[0]);
1717 assert(rt < V3D_MAX_DRAW_BUFFERS);
1718
1719 int sample_index = nir_intrinsic_base(instr) ;
1720 assert(sample_index < V3D_MAX_SAMPLES);
1721
1722 int component = nir_intrinsic_component(instr);
1723 assert(component < 4);
1724
1725 /* We need to emit our TLB reads after we have acquired the scoreboard
1726 * lock, or the GPU will hang. Usually, we do our scoreboard locking on
1727 * the last thread switch to improve parallelism, however, that is only
1728 * guaranteed to happen before the tlb color writes.
1729 *
1730 * To fix that, we make sure we always emit a thread switch before the
1731 * first tlb color read. If that happens to be the last thread switch
1732 * we emit, then everything is fine, but otherwsie, if any code after
1733 * this point needs to emit additional thread switches, then we will
1734 * switch the strategy to locking the scoreboard on the first thread
1735 * switch instead -- see vir_emit_thrsw().
1736 */
1737 if (!c->emitted_tlb_load) {
1738 if (!c->last_thrsw_at_top_level) {
1739 assert(c->devinfo->ver >= 41);
1740 vir_emit_thrsw(c);
1741 }
1742
1743 c->emitted_tlb_load = true;
1744 }
1745
1746 struct qreg *color_reads_for_sample =
1747 &c->color_reads[(rt * V3D_MAX_SAMPLES + sample_index) * 4];
1748
1749 if (color_reads_for_sample[component].file == QFILE_NULL) {
1750 enum pipe_format rt_format = c->fs_key->color_fmt[rt].format;
1751 int num_components =
1752 util_format_get_nr_components(rt_format);
1753
1754 const bool swap_rb = c->fs_key->swap_color_rb & (1 << rt);
1755 if (swap_rb)
1756 num_components = MAX2(num_components, 3);
1757
1758 nir_variable *var = c->output_color_var[rt];
1759 enum glsl_base_type type = glsl_get_base_type(var->type);
1760
1761 bool is_int_format = type == GLSL_TYPE_INT ||
1762 type == GLSL_TYPE_UINT;
1763
1764 bool is_32b_tlb_format = is_int_format ||
1765 (c->fs_key->f32_color_rb & (1 << rt));
1766
1767 int num_samples = c->fs_key->msaa ? V3D_MAX_SAMPLES : 1;
1768
1769 uint32_t conf = 0xffffff00;
1770 conf |= c->fs_key->msaa ? TLB_SAMPLE_MODE_PER_SAMPLE :
1771 TLB_SAMPLE_MODE_PER_PIXEL;
1772 conf |= (7 - rt) << TLB_RENDER_TARGET_SHIFT;
1773
1774 if (is_32b_tlb_format) {
1775 /* The F32 vs I32 distinction was dropped in 4.2. */
1776 conf |= (c->devinfo->ver < 42 && is_int_format) ?
1777 TLB_TYPE_I32_COLOR : TLB_TYPE_F32_COLOR;
1778
1779 conf |= ((num_components - 1) <<
1780 TLB_VEC_SIZE_MINUS_1_SHIFT);
1781 } else {
1782 conf |= TLB_TYPE_F16_COLOR;
1783 conf |= TLB_F16_SWAP_HI_LO;
1784
1785 if (num_components >= 3)
1786 conf |= TLB_VEC_SIZE_4_F16;
1787 else
1788 conf |= TLB_VEC_SIZE_2_F16;
1789 }
1790
1791
1792 for (int i = 0; i < num_samples; i++) {
1793 struct qreg r, g, b, a;
1794 if (is_32b_tlb_format) {
1795 r = conf != 0xffffffff && i == 0?
1796 vir_TLBU_COLOR_READ(c, conf) :
1797 vir_TLB_COLOR_READ(c);
1798 if (num_components >= 2)
1799 g = vir_TLB_COLOR_READ(c);
1800 if (num_components >= 3)
1801 b = vir_TLB_COLOR_READ(c);
1802 if (num_components >= 4)
1803 a = vir_TLB_COLOR_READ(c);
1804 } else {
1805 struct qreg rg = conf != 0xffffffff && i == 0 ?
1806 vir_TLBU_COLOR_READ(c, conf) :
1807 vir_TLB_COLOR_READ(c);
1808 r = vir_FMOV(c, rg);
1809 vir_set_unpack(c->defs[r.index], 0,
1810 V3D_QPU_UNPACK_L);
1811 g = vir_FMOV(c, rg);
1812 vir_set_unpack(c->defs[g.index], 0,
1813 V3D_QPU_UNPACK_H);
1814
1815 if (num_components > 2) {
1816 struct qreg ba = vir_TLB_COLOR_READ(c);
1817 b = vir_FMOV(c, ba);
1818 vir_set_unpack(c->defs[b.index], 0,
1819 V3D_QPU_UNPACK_L);
1820 a = vir_FMOV(c, ba);
1821 vir_set_unpack(c->defs[a.index], 0,
1822 V3D_QPU_UNPACK_H);
1823 }
1824 }
1825
1826 struct qreg *color_reads =
1827 &c->color_reads[(rt * V3D_MAX_SAMPLES + i) * 4];
1828
1829 color_reads[0] = swap_rb ? b : r;
1830 if (num_components >= 2)
1831 color_reads[1] = g;
1832 if (num_components >= 3)
1833 color_reads[2] = swap_rb ? r : b;
1834 if (num_components >= 4)
1835 color_reads[3] = a;
1836 }
1837 }
1838
1839 assert(color_reads_for_sample[component].file != QFILE_NULL);
1840 ntq_store_dest(c, &instr->dest, 0,
1841 vir_MOV(c, color_reads_for_sample[component]));
1842 }
1843
1844 static void
1845 ntq_emit_load_uniform(struct v3d_compile *c, nir_intrinsic_instr *instr)
1846 {
1847 if (nir_src_is_const(instr->src[0])) {
1848 int offset = (nir_intrinsic_base(instr) +
1849 nir_src_as_uint(instr->src[0]));
1850 assert(offset % 4 == 0);
1851 /* We need dwords */
1852 offset = offset / 4;
1853 for (int i = 0; i < instr->num_components; i++) {
1854 ntq_store_dest(c, &instr->dest, i,
1855 vir_uniform(c, QUNIFORM_UNIFORM,
1856 offset + i));
1857 }
1858 } else {
1859 ntq_emit_tmu_general(c, instr, false);
1860 }
1861 }
1862
1863 static void
1864 ntq_emit_load_input(struct v3d_compile *c, nir_intrinsic_instr *instr)
1865 {
1866 /* XXX: Use ldvpmv (uniform offset) or ldvpmd (non-uniform offset)
1867 * and enable PIPE_SHADER_CAP_INDIRECT_INPUT_ADDR.
1868 */
1869 unsigned offset =
1870 nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[0]);
1871
1872 if (c->s->info.stage != MESA_SHADER_FRAGMENT && c->devinfo->ver >= 40) {
1873 /* Emit the LDVPM directly now, rather than at the top
1874 * of the shader like we did for V3D 3.x (which needs
1875 * vpmsetup when not just taking the next offset).
1876 *
1877 * Note that delaying like this may introduce stalls,
1878 * as LDVPMV takes a minimum of 1 instruction but may
1879 * be slower if the VPM unit is busy with another QPU.
1880 */
1881 int index = 0;
1882 if (c->s->info.system_values_read &
1883 (1ull << SYSTEM_VALUE_INSTANCE_ID)) {
1884 index++;
1885 }
1886 if (c->s->info.system_values_read &
1887 (1ull << SYSTEM_VALUE_VERTEX_ID)) {
1888 index++;
1889 }
1890 for (int i = 0; i < offset; i++)
1891 index += c->vattr_sizes[i];
1892 index += nir_intrinsic_component(instr);
1893 for (int i = 0; i < instr->num_components; i++) {
1894 struct qreg vpm_offset = vir_uniform_ui(c, index++);
1895 ntq_store_dest(c, &instr->dest, i,
1896 vir_LDVPMV_IN(c, vpm_offset));
1897 }
1898 } else {
1899 for (int i = 0; i < instr->num_components; i++) {
1900 int comp = nir_intrinsic_component(instr) + i;
1901 ntq_store_dest(c, &instr->dest, i,
1902 vir_MOV(c, c->inputs[offset * 4 + comp]));
1903 }
1904 }
1905 }
1906
1907 static void
1908 ntq_emit_per_sample_color_write(struct v3d_compile *c,
1909 nir_intrinsic_instr *instr)
1910 {
1911 assert(instr->intrinsic == nir_intrinsic_store_tlb_sample_color_v3d);
1912
1913 unsigned rt = nir_src_as_uint(instr->src[1]);
1914 assert(rt < V3D_MAX_DRAW_BUFFERS);
1915
1916 unsigned sample_idx = nir_intrinsic_base(instr);
1917 assert(sample_idx < V3D_MAX_SAMPLES);
1918
1919 unsigned offset = (rt * V3D_MAX_SAMPLES + sample_idx) * 4;
1920 for (int i = 0; i < instr->num_components; i++) {
1921 c->sample_colors[offset + i] =
1922 vir_MOV(c, ntq_get_src(c, instr->src[0], i));
1923 }
1924 }
1925
1926 static void
1927 ntq_emit_color_write(struct v3d_compile *c,
1928 nir_intrinsic_instr *instr)
1929 {
1930 unsigned offset = (nir_intrinsic_base(instr) +
1931 nir_src_as_uint(instr->src[1])) * 4 +
1932 nir_intrinsic_component(instr);
1933 for (int i = 0; i < instr->num_components; i++) {
1934 c->outputs[offset + i] =
1935 vir_MOV(c, ntq_get_src(c, instr->src[0], i));
1936 }
1937 }
1938
1939 static void
1940 ntq_emit_intrinsic(struct v3d_compile *c, nir_intrinsic_instr *instr)
1941 {
1942 switch (instr->intrinsic) {
1943 case nir_intrinsic_load_uniform:
1944 ntq_emit_load_uniform(c, instr);
1945 break;
1946
1947 case nir_intrinsic_load_ubo:
1948 ntq_emit_tmu_general(c, instr, false);
1949 break;
1950
1951 case nir_intrinsic_ssbo_atomic_add:
1952 case nir_intrinsic_ssbo_atomic_imin:
1953 case nir_intrinsic_ssbo_atomic_umin:
1954 case nir_intrinsic_ssbo_atomic_imax:
1955 case nir_intrinsic_ssbo_atomic_umax:
1956 case nir_intrinsic_ssbo_atomic_and:
1957 case nir_intrinsic_ssbo_atomic_or:
1958 case nir_intrinsic_ssbo_atomic_xor:
1959 case nir_intrinsic_ssbo_atomic_exchange:
1960 case nir_intrinsic_ssbo_atomic_comp_swap:
1961 case nir_intrinsic_load_ssbo:
1962 case nir_intrinsic_store_ssbo:
1963 ntq_emit_tmu_general(c, instr, false);
1964 break;
1965
1966 case nir_intrinsic_shared_atomic_add:
1967 case nir_intrinsic_shared_atomic_imin:
1968 case nir_intrinsic_shared_atomic_umin:
1969 case nir_intrinsic_shared_atomic_imax:
1970 case nir_intrinsic_shared_atomic_umax:
1971 case nir_intrinsic_shared_atomic_and:
1972 case nir_intrinsic_shared_atomic_or:
1973 case nir_intrinsic_shared_atomic_xor:
1974 case nir_intrinsic_shared_atomic_exchange:
1975 case nir_intrinsic_shared_atomic_comp_swap:
1976 case nir_intrinsic_load_shared:
1977 case nir_intrinsic_store_shared:
1978 case nir_intrinsic_load_scratch:
1979 case nir_intrinsic_store_scratch:
1980 ntq_emit_tmu_general(c, instr, true);
1981 break;
1982
1983 case nir_intrinsic_image_deref_load:
1984 case nir_intrinsic_image_deref_store:
1985 case nir_intrinsic_image_deref_atomic_add:
1986 case nir_intrinsic_image_deref_atomic_imin:
1987 case nir_intrinsic_image_deref_atomic_umin:
1988 case nir_intrinsic_image_deref_atomic_imax:
1989 case nir_intrinsic_image_deref_atomic_umax:
1990 case nir_intrinsic_image_deref_atomic_and:
1991 case nir_intrinsic_image_deref_atomic_or:
1992 case nir_intrinsic_image_deref_atomic_xor:
1993 case nir_intrinsic_image_deref_atomic_exchange:
1994 case nir_intrinsic_image_deref_atomic_comp_swap:
1995 v3d40_vir_emit_image_load_store(c, instr);
1996 break;
1997
1998 case nir_intrinsic_get_buffer_size:
1999 ntq_store_dest(c, &instr->dest, 0,
2000 vir_uniform(c, QUNIFORM_GET_BUFFER_SIZE,
2001 nir_src_as_uint(instr->src[0])));
2002 break;
2003
2004 case nir_intrinsic_load_user_clip_plane:
2005 for (int i = 0; i < instr->num_components; i++) {
2006 ntq_store_dest(c, &instr->dest, i,
2007 vir_uniform(c, QUNIFORM_USER_CLIP_PLANE,
2008 nir_intrinsic_ucp_id(instr) *
2009 4 + i));
2010 }
2011 break;
2012
2013 case nir_intrinsic_load_viewport_x_scale:
2014 ntq_store_dest(c, &instr->dest, 0,
2015 vir_uniform(c, QUNIFORM_VIEWPORT_X_SCALE, 0));
2016 break;
2017
2018 case nir_intrinsic_load_viewport_y_scale:
2019 ntq_store_dest(c, &instr->dest, 0,
2020 vir_uniform(c, QUNIFORM_VIEWPORT_Y_SCALE, 0));
2021 break;
2022
2023 case nir_intrinsic_load_viewport_z_scale:
2024 ntq_store_dest(c, &instr->dest, 0,
2025 vir_uniform(c, QUNIFORM_VIEWPORT_Z_SCALE, 0));
2026 break;
2027
2028 case nir_intrinsic_load_viewport_z_offset:
2029 ntq_store_dest(c, &instr->dest, 0,
2030 vir_uniform(c, QUNIFORM_VIEWPORT_Z_OFFSET, 0));
2031 break;
2032
2033 case nir_intrinsic_load_alpha_ref_float:
2034 ntq_store_dest(c, &instr->dest, 0,
2035 vir_uniform(c, QUNIFORM_ALPHA_REF, 0));
2036 break;
2037
2038 case nir_intrinsic_load_sample_mask_in:
2039 ntq_store_dest(c, &instr->dest, 0, vir_MSF(c));
2040 break;
2041
2042 case nir_intrinsic_load_helper_invocation:
2043 vir_set_pf(vir_MSF_dest(c, vir_nop_reg()), V3D_QPU_PF_PUSHZ);
2044 ntq_store_dest(c, &instr->dest, 0,
2045 vir_MOV(c, vir_SEL(c, V3D_QPU_COND_IFA,
2046 vir_uniform_ui(c, ~0),
2047 vir_uniform_ui(c, 0))));
2048 break;
2049
2050 case nir_intrinsic_load_front_face:
2051 /* The register contains 0 (front) or 1 (back), and we need to
2052 * turn it into a NIR bool where true means front.
2053 */
2054 ntq_store_dest(c, &instr->dest, 0,
2055 vir_ADD(c,
2056 vir_uniform_ui(c, -1),
2057 vir_REVF(c)));
2058 break;
2059
2060 case nir_intrinsic_load_instance_id:
2061 ntq_store_dest(c, &instr->dest, 0, vir_MOV(c, c->iid));
2062 break;
2063
2064 case nir_intrinsic_load_vertex_id:
2065 ntq_store_dest(c, &instr->dest, 0, vir_MOV(c, c->vid));
2066 break;
2067
2068 case nir_intrinsic_load_tlb_color_v3d:
2069 vir_emit_tlb_color_read(c, instr);
2070 break;
2071
2072 case nir_intrinsic_load_input:
2073 ntq_emit_load_input(c, instr);
2074 break;
2075
2076 case nir_intrinsic_store_tlb_sample_color_v3d:
2077 ntq_emit_per_sample_color_write(c, instr);
2078 break;
2079
2080 case nir_intrinsic_store_output:
2081 /* XXX perf: Use stvpmv with uniform non-constant offsets and
2082 * stvpmd with non-uniform offsets and enable
2083 * PIPE_SHADER_CAP_INDIRECT_OUTPUT_ADDR.
2084 */
2085 if (c->s->info.stage == MESA_SHADER_FRAGMENT) {
2086 ntq_emit_color_write(c, instr);
2087 } else {
2088 assert(instr->num_components == 1);
2089
2090 vir_VPM_WRITE(c,
2091 ntq_get_src(c, instr->src[0], 0),
2092 nir_intrinsic_base(instr));
2093 }
2094 break;
2095
2096 case nir_intrinsic_image_deref_size:
2097 ntq_emit_image_size(c, instr);
2098 break;
2099
2100 case nir_intrinsic_discard:
2101 if (vir_in_nonuniform_control_flow(c)) {
2102 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute),
2103 V3D_QPU_PF_PUSHZ);
2104 vir_set_cond(vir_SETMSF_dest(c, vir_nop_reg(),
2105 vir_uniform_ui(c, 0)),
2106 V3D_QPU_COND_IFA);
2107 } else {
2108 vir_SETMSF_dest(c, vir_nop_reg(),
2109 vir_uniform_ui(c, 0));
2110 }
2111 break;
2112
2113 case nir_intrinsic_discard_if: {
2114 enum v3d_qpu_cond cond = ntq_emit_bool_to_cond(c, instr->src[0]);
2115
2116 if (vir_in_nonuniform_control_flow(c)) {
2117 struct qinst *exec_flag = vir_MOV_dest(c, vir_nop_reg(),
2118 c->execute);
2119 if (cond == V3D_QPU_COND_IFA) {
2120 vir_set_uf(exec_flag, V3D_QPU_UF_ANDZ);
2121 } else {
2122 vir_set_uf(exec_flag, V3D_QPU_UF_NORNZ);
2123 cond = V3D_QPU_COND_IFA;
2124 }
2125 }
2126
2127 vir_set_cond(vir_SETMSF_dest(c, vir_nop_reg(),
2128 vir_uniform_ui(c, 0)), cond);
2129
2130 break;
2131 }
2132
2133 case nir_intrinsic_memory_barrier:
2134 case nir_intrinsic_memory_barrier_atomic_counter:
2135 case nir_intrinsic_memory_barrier_buffer:
2136 case nir_intrinsic_memory_barrier_image:
2137 case nir_intrinsic_memory_barrier_shared:
2138 case nir_intrinsic_group_memory_barrier:
2139 /* We don't do any instruction scheduling of these NIR
2140 * instructions between each other, so we just need to make
2141 * sure that the TMU operations before the barrier are flushed
2142 * before the ones after the barrier. That is currently
2143 * handled by having a THRSW in each of them and a LDTMU
2144 * series or a TMUWT after.
2145 */
2146 break;
2147
2148 case nir_intrinsic_barrier:
2149 /* Emit a TSY op to get all invocations in the workgroup
2150 * (actually supergroup) to block until the last invocation
2151 * reaches the TSY op.
2152 */
2153 if (c->devinfo->ver >= 42) {
2154 vir_BARRIERID_dest(c, vir_reg(QFILE_MAGIC,
2155 V3D_QPU_WADDR_SYNCB));
2156 } else {
2157 struct qinst *sync =
2158 vir_BARRIERID_dest(c,
2159 vir_reg(QFILE_MAGIC,
2160 V3D_QPU_WADDR_SYNCU));
2161 sync->uniform =
2162 vir_get_uniform_index(c, QUNIFORM_CONSTANT,
2163 0xffffff00 |
2164 V3D_TSY_WAIT_INC_CHECK);
2165
2166 }
2167
2168 /* The blocking of a TSY op only happens at the next thread
2169 * switch. No texturing may be outstanding at the time of a
2170 * TSY blocking operation.
2171 */
2172 vir_emit_thrsw(c);
2173 break;
2174
2175 case nir_intrinsic_load_num_work_groups:
2176 for (int i = 0; i < 3; i++) {
2177 ntq_store_dest(c, &instr->dest, i,
2178 vir_uniform(c, QUNIFORM_NUM_WORK_GROUPS,
2179 i));
2180 }
2181 break;
2182
2183 case nir_intrinsic_load_local_invocation_index:
2184 ntq_store_dest(c, &instr->dest, 0,
2185 vir_SHR(c, c->cs_payload[1],
2186 vir_uniform_ui(c, 32 - c->local_invocation_index_bits)));
2187 break;
2188
2189 case nir_intrinsic_load_work_group_id:
2190 ntq_store_dest(c, &instr->dest, 0,
2191 vir_AND(c, c->cs_payload[0],
2192 vir_uniform_ui(c, 0xffff)));
2193 ntq_store_dest(c, &instr->dest, 1,
2194 vir_SHR(c, c->cs_payload[0],
2195 vir_uniform_ui(c, 16)));
2196 ntq_store_dest(c, &instr->dest, 2,
2197 vir_AND(c, c->cs_payload[1],
2198 vir_uniform_ui(c, 0xffff)));
2199 break;
2200
2201 case nir_intrinsic_load_subgroup_id:
2202 ntq_store_dest(c, &instr->dest, 0, vir_EIDX(c));
2203 break;
2204
2205 default:
2206 fprintf(stderr, "Unknown intrinsic: ");
2207 nir_print_instr(&instr->instr, stderr);
2208 fprintf(stderr, "\n");
2209 break;
2210 }
2211 }
2212
2213 /* Clears (activates) the execute flags for any channels whose jump target
2214 * matches this block.
2215 *
2216 * XXX perf: Could we be using flpush/flpop somehow for our execution channel
2217 * enabling?
2218 *
2219 * XXX perf: For uniform control flow, we should be able to skip c->execute
2220 * handling entirely.
2221 */
2222 static void
2223 ntq_activate_execute_for_block(struct v3d_compile *c)
2224 {
2225 vir_set_pf(vir_XOR_dest(c, vir_nop_reg(),
2226 c->execute, vir_uniform_ui(c, c->cur_block->index)),
2227 V3D_QPU_PF_PUSHZ);
2228
2229 vir_MOV_cond(c, V3D_QPU_COND_IFA, c->execute, vir_uniform_ui(c, 0));
2230 }
2231
2232 static void
2233 ntq_emit_uniform_if(struct v3d_compile *c, nir_if *if_stmt)
2234 {
2235 nir_block *nir_else_block = nir_if_first_else_block(if_stmt);
2236 bool empty_else_block =
2237 (nir_else_block == nir_if_last_else_block(if_stmt) &&
2238 exec_list_is_empty(&nir_else_block->instr_list));
2239
2240 struct qblock *then_block = vir_new_block(c);
2241 struct qblock *after_block = vir_new_block(c);
2242 struct qblock *else_block;
2243 if (empty_else_block)
2244 else_block = after_block;
2245 else
2246 else_block = vir_new_block(c);
2247
2248 /* Set up the flags for the IF condition (taking the THEN branch). */
2249 enum v3d_qpu_cond cond = ntq_emit_bool_to_cond(c, if_stmt->condition);
2250
2251 /* Jump to ELSE. */
2252 vir_BRANCH(c, cond == V3D_QPU_COND_IFA ?
2253 V3D_QPU_BRANCH_COND_ALLNA :
2254 V3D_QPU_BRANCH_COND_ALLA);
2255 vir_link_blocks(c->cur_block, else_block);
2256 vir_link_blocks(c->cur_block, then_block);
2257
2258 /* Process the THEN block. */
2259 vir_set_emit_block(c, then_block);
2260 ntq_emit_cf_list(c, &if_stmt->then_list);
2261
2262 if (!empty_else_block) {
2263 /* At the end of the THEN block, jump to ENDIF */
2264 vir_BRANCH(c, V3D_QPU_BRANCH_COND_ALWAYS);
2265 vir_link_blocks(c->cur_block, after_block);
2266
2267 /* Emit the else block. */
2268 vir_set_emit_block(c, else_block);
2269 ntq_emit_cf_list(c, &if_stmt->else_list);
2270 }
2271
2272 vir_link_blocks(c->cur_block, after_block);
2273
2274 vir_set_emit_block(c, after_block);
2275 }
2276
2277 static void
2278 ntq_emit_nonuniform_if(struct v3d_compile *c, nir_if *if_stmt)
2279 {
2280 nir_block *nir_else_block = nir_if_first_else_block(if_stmt);
2281 bool empty_else_block =
2282 (nir_else_block == nir_if_last_else_block(if_stmt) &&
2283 exec_list_is_empty(&nir_else_block->instr_list));
2284
2285 struct qblock *then_block = vir_new_block(c);
2286 struct qblock *after_block = vir_new_block(c);
2287 struct qblock *else_block;
2288 if (empty_else_block)
2289 else_block = after_block;
2290 else
2291 else_block = vir_new_block(c);
2292
2293 bool was_uniform_control_flow = false;
2294 if (!vir_in_nonuniform_control_flow(c)) {
2295 c->execute = vir_MOV(c, vir_uniform_ui(c, 0));
2296 was_uniform_control_flow = true;
2297 }
2298
2299 /* Set up the flags for the IF condition (taking the THEN branch). */
2300 enum v3d_qpu_cond cond = ntq_emit_bool_to_cond(c, if_stmt->condition);
2301
2302 /* Update the flags+cond to mean "Taking the ELSE branch (!cond) and
2303 * was previously active (execute Z) for updating the exec flags.
2304 */
2305 if (was_uniform_control_flow) {
2306 cond = v3d_qpu_cond_invert(cond);
2307 } else {
2308 struct qinst *inst = vir_MOV_dest(c, vir_nop_reg(), c->execute);
2309 if (cond == V3D_QPU_COND_IFA) {
2310 vir_set_uf(inst, V3D_QPU_UF_NORNZ);
2311 } else {
2312 vir_set_uf(inst, V3D_QPU_UF_ANDZ);
2313 cond = V3D_QPU_COND_IFA;
2314 }
2315 }
2316
2317 vir_MOV_cond(c, cond,
2318 c->execute,
2319 vir_uniform_ui(c, else_block->index));
2320
2321 /* Jump to ELSE if nothing is active for THEN, otherwise fall
2322 * through.
2323 */
2324 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute), V3D_QPU_PF_PUSHZ);
2325 vir_BRANCH(c, V3D_QPU_BRANCH_COND_ALLNA);
2326 vir_link_blocks(c->cur_block, else_block);
2327 vir_link_blocks(c->cur_block, then_block);
2328
2329 /* Process the THEN block. */
2330 vir_set_emit_block(c, then_block);
2331 ntq_emit_cf_list(c, &if_stmt->then_list);
2332
2333 if (!empty_else_block) {
2334 /* Handle the end of the THEN block. First, all currently
2335 * active channels update their execute flags to point to
2336 * ENDIF
2337 */
2338 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute),
2339 V3D_QPU_PF_PUSHZ);
2340 vir_MOV_cond(c, V3D_QPU_COND_IFA, c->execute,
2341 vir_uniform_ui(c, after_block->index));
2342
2343 /* If everything points at ENDIF, then jump there immediately. */
2344 vir_set_pf(vir_XOR_dest(c, vir_nop_reg(),
2345 c->execute,
2346 vir_uniform_ui(c, after_block->index)),
2347 V3D_QPU_PF_PUSHZ);
2348 vir_BRANCH(c, V3D_QPU_BRANCH_COND_ALLA);
2349 vir_link_blocks(c->cur_block, after_block);
2350 vir_link_blocks(c->cur_block, else_block);
2351
2352 vir_set_emit_block(c, else_block);
2353 ntq_activate_execute_for_block(c);
2354 ntq_emit_cf_list(c, &if_stmt->else_list);
2355 }
2356
2357 vir_link_blocks(c->cur_block, after_block);
2358
2359 vir_set_emit_block(c, after_block);
2360 if (was_uniform_control_flow)
2361 c->execute = c->undef;
2362 else
2363 ntq_activate_execute_for_block(c);
2364 }
2365
2366 static void
2367 ntq_emit_if(struct v3d_compile *c, nir_if *nif)
2368 {
2369 bool was_in_control_flow = c->in_control_flow;
2370 c->in_control_flow = true;
2371 if (!vir_in_nonuniform_control_flow(c) &&
2372 nir_src_is_dynamically_uniform(nif->condition)) {
2373 ntq_emit_uniform_if(c, nif);
2374 } else {
2375 ntq_emit_nonuniform_if(c, nif);
2376 }
2377 c->in_control_flow = was_in_control_flow;
2378 }
2379
2380 static void
2381 ntq_emit_jump(struct v3d_compile *c, nir_jump_instr *jump)
2382 {
2383 switch (jump->type) {
2384 case nir_jump_break:
2385 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute),
2386 V3D_QPU_PF_PUSHZ);
2387 vir_MOV_cond(c, V3D_QPU_COND_IFA, c->execute,
2388 vir_uniform_ui(c, c->loop_break_block->index));
2389 break;
2390
2391 case nir_jump_continue:
2392 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute),
2393 V3D_QPU_PF_PUSHZ);
2394 vir_MOV_cond(c, V3D_QPU_COND_IFA, c->execute,
2395 vir_uniform_ui(c, c->loop_cont_block->index));
2396 break;
2397
2398 case nir_jump_return:
2399 unreachable("All returns shouold be lowered\n");
2400 }
2401 }
2402
2403 static void
2404 ntq_emit_instr(struct v3d_compile *c, nir_instr *instr)
2405 {
2406 switch (instr->type) {
2407 case nir_instr_type_deref:
2408 /* ignored, will be walked by the intrinsic using it. */
2409 break;
2410
2411 case nir_instr_type_alu:
2412 ntq_emit_alu(c, nir_instr_as_alu(instr));
2413 break;
2414
2415 case nir_instr_type_intrinsic:
2416 ntq_emit_intrinsic(c, nir_instr_as_intrinsic(instr));
2417 break;
2418
2419 case nir_instr_type_load_const:
2420 ntq_emit_load_const(c, nir_instr_as_load_const(instr));
2421 break;
2422
2423 case nir_instr_type_ssa_undef:
2424 ntq_emit_ssa_undef(c, nir_instr_as_ssa_undef(instr));
2425 break;
2426
2427 case nir_instr_type_tex:
2428 ntq_emit_tex(c, nir_instr_as_tex(instr));
2429 break;
2430
2431 case nir_instr_type_jump:
2432 ntq_emit_jump(c, nir_instr_as_jump(instr));
2433 break;
2434
2435 default:
2436 fprintf(stderr, "Unknown NIR instr type: ");
2437 nir_print_instr(instr, stderr);
2438 fprintf(stderr, "\n");
2439 abort();
2440 }
2441 }
2442
2443 static void
2444 ntq_emit_block(struct v3d_compile *c, nir_block *block)
2445 {
2446 nir_foreach_instr(instr, block) {
2447 ntq_emit_instr(c, instr);
2448 }
2449 }
2450
2451 static void ntq_emit_cf_list(struct v3d_compile *c, struct exec_list *list);
2452
2453 static void
2454 ntq_emit_loop(struct v3d_compile *c, nir_loop *loop)
2455 {
2456 bool was_in_control_flow = c->in_control_flow;
2457 c->in_control_flow = true;
2458
2459 bool was_uniform_control_flow = false;
2460 if (!vir_in_nonuniform_control_flow(c)) {
2461 c->execute = vir_MOV(c, vir_uniform_ui(c, 0));
2462 was_uniform_control_flow = true;
2463 }
2464
2465 struct qblock *save_loop_cont_block = c->loop_cont_block;
2466 struct qblock *save_loop_break_block = c->loop_break_block;
2467
2468 c->loop_cont_block = vir_new_block(c);
2469 c->loop_break_block = vir_new_block(c);
2470
2471 vir_link_blocks(c->cur_block, c->loop_cont_block);
2472 vir_set_emit_block(c, c->loop_cont_block);
2473 ntq_activate_execute_for_block(c);
2474
2475 ntq_emit_cf_list(c, &loop->body);
2476
2477 /* Re-enable any previous continues now, so our ANYA check below
2478 * works.
2479 *
2480 * XXX: Use the .ORZ flags update, instead.
2481 */
2482 vir_set_pf(vir_XOR_dest(c,
2483 vir_nop_reg(),
2484 c->execute,
2485 vir_uniform_ui(c, c->loop_cont_block->index)),
2486 V3D_QPU_PF_PUSHZ);
2487 vir_MOV_cond(c, V3D_QPU_COND_IFA, c->execute, vir_uniform_ui(c, 0));
2488
2489 vir_set_pf(vir_MOV_dest(c, vir_nop_reg(), c->execute), V3D_QPU_PF_PUSHZ);
2490
2491 struct qinst *branch = vir_BRANCH(c, V3D_QPU_BRANCH_COND_ANYA);
2492 /* Pixels that were not dispatched or have been discarded should not
2493 * contribute to looping again.
2494 */
2495 branch->qpu.branch.msfign = V3D_QPU_MSFIGN_P;
2496 vir_link_blocks(c->cur_block, c->loop_cont_block);
2497 vir_link_blocks(c->cur_block, c->loop_break_block);
2498
2499 vir_set_emit_block(c, c->loop_break_block);
2500 if (was_uniform_control_flow)
2501 c->execute = c->undef;
2502 else
2503 ntq_activate_execute_for_block(c);
2504
2505 c->loop_break_block = save_loop_break_block;
2506 c->loop_cont_block = save_loop_cont_block;
2507
2508 c->loops++;
2509
2510 c->in_control_flow = was_in_control_flow;
2511 }
2512
2513 static void
2514 ntq_emit_function(struct v3d_compile *c, nir_function_impl *func)
2515 {
2516 fprintf(stderr, "FUNCTIONS not handled.\n");
2517 abort();
2518 }
2519
2520 static void
2521 ntq_emit_cf_list(struct v3d_compile *c, struct exec_list *list)
2522 {
2523 foreach_list_typed(nir_cf_node, node, node, list) {
2524 switch (node->type) {
2525 case nir_cf_node_block:
2526 ntq_emit_block(c, nir_cf_node_as_block(node));
2527 break;
2528
2529 case nir_cf_node_if:
2530 ntq_emit_if(c, nir_cf_node_as_if(node));
2531 break;
2532
2533 case nir_cf_node_loop:
2534 ntq_emit_loop(c, nir_cf_node_as_loop(node));
2535 break;
2536
2537 case nir_cf_node_function:
2538 ntq_emit_function(c, nir_cf_node_as_function(node));
2539 break;
2540
2541 default:
2542 fprintf(stderr, "Unknown NIR node type\n");
2543 abort();
2544 }
2545 }
2546 }
2547
2548 static void
2549 ntq_emit_impl(struct v3d_compile *c, nir_function_impl *impl)
2550 {
2551 ntq_setup_registers(c, &impl->registers);
2552 ntq_emit_cf_list(c, &impl->body);
2553 }
2554
2555 static void
2556 nir_to_vir(struct v3d_compile *c)
2557 {
2558 switch (c->s->info.stage) {
2559 case MESA_SHADER_FRAGMENT:
2560 c->payload_w = vir_MOV(c, vir_reg(QFILE_REG, 0));
2561 c->payload_w_centroid = vir_MOV(c, vir_reg(QFILE_REG, 1));
2562 c->payload_z = vir_MOV(c, vir_reg(QFILE_REG, 2));
2563
2564 /* V3D 4.x can disable implicit point coordinate varyings if
2565 * they are not used.
2566 */
2567 if (c->fs_key->is_points &&
2568 (c->devinfo->ver < 40 || program_reads_point_coord(c))) {
2569 c->point_x = emit_fragment_varying(c, NULL, 0, 0);
2570 c->point_y = emit_fragment_varying(c, NULL, 0, 0);
2571 c->uses_implicit_point_line_varyings = true;
2572 } else if (c->fs_key->is_lines && c->devinfo->ver < 40) {
2573 c->line_x = emit_fragment_varying(c, NULL, 0, 0);
2574 c->uses_implicit_point_line_varyings = true;
2575 }
2576 break;
2577 case MESA_SHADER_COMPUTE:
2578 /* Set up the TSO for barriers, assuming we do some. */
2579 if (c->devinfo->ver < 42) {
2580 vir_BARRIERID_dest(c, vir_reg(QFILE_MAGIC,
2581 V3D_QPU_WADDR_SYNC));
2582 }
2583
2584 c->cs_payload[0] = vir_MOV(c, vir_reg(QFILE_REG, 0));
2585 c->cs_payload[1] = vir_MOV(c, vir_reg(QFILE_REG, 2));
2586
2587 /* Set up the division between gl_LocalInvocationIndex and
2588 * wg_in_mem in the payload reg.
2589 */
2590 int wg_size = (c->s->info.cs.local_size[0] *
2591 c->s->info.cs.local_size[1] *
2592 c->s->info.cs.local_size[2]);
2593 c->local_invocation_index_bits =
2594 ffs(util_next_power_of_two(MAX2(wg_size, 64))) - 1;
2595 assert(c->local_invocation_index_bits <= 8);
2596
2597 if (c->s->info.cs.shared_size) {
2598 struct qreg wg_in_mem = vir_SHR(c, c->cs_payload[1],
2599 vir_uniform_ui(c, 16));
2600 if (c->s->info.cs.local_size[0] != 1 ||
2601 c->s->info.cs.local_size[1] != 1 ||
2602 c->s->info.cs.local_size[2] != 1) {
2603 int wg_bits = (16 -
2604 c->local_invocation_index_bits);
2605 int wg_mask = (1 << wg_bits) - 1;
2606 wg_in_mem = vir_AND(c, wg_in_mem,
2607 vir_uniform_ui(c, wg_mask));
2608 }
2609 struct qreg shared_per_wg =
2610 vir_uniform_ui(c, c->s->info.cs.shared_size);
2611
2612 c->cs_shared_offset =
2613 vir_ADD(c,
2614 vir_uniform(c, QUNIFORM_SHARED_OFFSET,0),
2615 vir_UMUL(c, wg_in_mem, shared_per_wg));
2616 }
2617 break;
2618 default:
2619 break;
2620 }
2621
2622 if (c->s->scratch_size) {
2623 v3d_setup_spill_base(c);
2624 c->spill_size += V3D_CHANNELS * c->s->scratch_size;
2625 }
2626
2627 if (c->s->info.stage == MESA_SHADER_FRAGMENT)
2628 ntq_setup_fs_inputs(c);
2629 else
2630 ntq_setup_vpm_inputs(c);
2631
2632 ntq_setup_outputs(c);
2633
2634 /* Find the main function and emit the body. */
2635 nir_foreach_function(function, c->s) {
2636 assert(strcmp(function->name, "main") == 0);
2637 assert(function->impl);
2638 ntq_emit_impl(c, function->impl);
2639 }
2640 }
2641
2642 const nir_shader_compiler_options v3d_nir_options = {
2643 .lower_all_io_to_temps = true,
2644 .lower_extract_byte = true,
2645 .lower_extract_word = true,
2646 .lower_bitfield_insert_to_shifts = true,
2647 .lower_bitfield_extract_to_shifts = true,
2648 .lower_bitfield_reverse = true,
2649 .lower_bit_count = true,
2650 .lower_cs_local_id_from_index = true,
2651 .lower_ffract = true,
2652 .lower_fmod = true,
2653 .lower_pack_unorm_2x16 = true,
2654 .lower_pack_snorm_2x16 = true,
2655 .lower_pack_unorm_4x8 = true,
2656 .lower_pack_snorm_4x8 = true,
2657 .lower_unpack_unorm_4x8 = true,
2658 .lower_unpack_snorm_4x8 = true,
2659 .lower_pack_half_2x16 = true,
2660 .lower_unpack_half_2x16 = true,
2661 .lower_fdiv = true,
2662 .lower_find_lsb = true,
2663 .lower_ffma = true,
2664 .lower_flrp32 = true,
2665 .lower_fpow = true,
2666 .lower_fsat = true,
2667 .lower_fsqrt = true,
2668 .lower_ifind_msb = true,
2669 .lower_isign = true,
2670 .lower_ldexp = true,
2671 .lower_mul_high = true,
2672 .lower_wpos_pntc = true,
2673 .lower_rotate = true,
2674 .lower_to_scalar = true,
2675 };
2676
2677 /**
2678 * When demoting a shader down to single-threaded, removes the THRSW
2679 * instructions (one will still be inserted at v3d_vir_to_qpu() for the
2680 * program end).
2681 */
2682 static void
2683 vir_remove_thrsw(struct v3d_compile *c)
2684 {
2685 vir_for_each_block(block, c) {
2686 vir_for_each_inst_safe(inst, block) {
2687 if (inst->qpu.sig.thrsw)
2688 vir_remove_instruction(c, inst);
2689 }
2690 }
2691
2692 c->last_thrsw = NULL;
2693 }
2694
2695 void
2696 vir_emit_last_thrsw(struct v3d_compile *c)
2697 {
2698 /* On V3D before 4.1, we need a TMU op to be outstanding when thread
2699 * switching, so disable threads if we didn't do any TMU ops (each of
2700 * which would have emitted a THRSW).
2701 */
2702 if (!c->last_thrsw_at_top_level && c->devinfo->ver < 41) {
2703 c->threads = 1;
2704 if (c->last_thrsw)
2705 vir_remove_thrsw(c);
2706 return;
2707 }
2708
2709 /* If we're threaded and the last THRSW was in conditional code, then
2710 * we need to emit another one so that we can flag it as the last
2711 * thrsw.
2712 */
2713 if (c->last_thrsw && !c->last_thrsw_at_top_level) {
2714 assert(c->devinfo->ver >= 41);
2715 vir_emit_thrsw(c);
2716 }
2717
2718 /* If we're threaded, then we need to mark the last THRSW instruction
2719 * so we can emit a pair of them at QPU emit time.
2720 *
2721 * For V3D 4.x, we can spawn the non-fragment shaders already in the
2722 * post-last-THRSW state, so we can skip this.
2723 */
2724 if (!c->last_thrsw && c->s->info.stage == MESA_SHADER_FRAGMENT) {
2725 assert(c->devinfo->ver >= 41);
2726 vir_emit_thrsw(c);
2727 }
2728
2729 if (c->last_thrsw)
2730 c->last_thrsw->is_last_thrsw = true;
2731 }
2732
2733 /* There's a flag in the shader for "center W is needed for reasons other than
2734 * non-centroid varyings", so we just walk the program after VIR optimization
2735 * to see if it's used. It should be harmless to set even if we only use
2736 * center W for varyings.
2737 */
2738 static void
2739 vir_check_payload_w(struct v3d_compile *c)
2740 {
2741 if (c->s->info.stage != MESA_SHADER_FRAGMENT)
2742 return;
2743
2744 vir_for_each_inst_inorder(inst, c) {
2745 for (int i = 0; i < vir_get_nsrc(inst); i++) {
2746 if (inst->src[i].file == QFILE_REG &&
2747 inst->src[i].index == 0) {
2748 c->uses_center_w = true;
2749 return;
2750 }
2751 }
2752 }
2753
2754 }
2755
2756 void
2757 v3d_nir_to_vir(struct v3d_compile *c)
2758 {
2759 if (V3D_DEBUG & (V3D_DEBUG_NIR |
2760 v3d_debug_flag_for_shader_stage(c->s->info.stage))) {
2761 fprintf(stderr, "%s prog %d/%d NIR:\n",
2762 vir_get_stage_name(c),
2763 c->program_id, c->variant_id);
2764 nir_print_shader(c->s, stderr);
2765 }
2766
2767 nir_to_vir(c);
2768
2769 /* Emit the last THRSW before STVPM and TLB writes. */
2770 vir_emit_last_thrsw(c);
2771
2772 switch (c->s->info.stage) {
2773 case MESA_SHADER_FRAGMENT:
2774 emit_frag_end(c);
2775 break;
2776 case MESA_SHADER_VERTEX:
2777 emit_vert_end(c);
2778 break;
2779 case MESA_SHADER_COMPUTE:
2780 break;
2781 default:
2782 unreachable("bad stage");
2783 }
2784
2785 if (V3D_DEBUG & (V3D_DEBUG_VIR |
2786 v3d_debug_flag_for_shader_stage(c->s->info.stage))) {
2787 fprintf(stderr, "%s prog %d/%d pre-opt VIR:\n",
2788 vir_get_stage_name(c),
2789 c->program_id, c->variant_id);
2790 vir_dump(c);
2791 fprintf(stderr, "\n");
2792 }
2793
2794 vir_optimize(c);
2795
2796 vir_check_payload_w(c);
2797
2798 /* XXX perf: On VC4, we do a VIR-level instruction scheduling here.
2799 * We used that on that platform to pipeline TMU writes and reduce the
2800 * number of thread switches, as well as try (mostly successfully) to
2801 * reduce maximum register pressure to allow more threads. We should
2802 * do something of that sort for V3D -- either instruction scheduling
2803 * here, or delay the the THRSW and LDTMUs from our texture
2804 * instructions until the results are needed.
2805 */
2806
2807 if (V3D_DEBUG & (V3D_DEBUG_VIR |
2808 v3d_debug_flag_for_shader_stage(c->s->info.stage))) {
2809 fprintf(stderr, "%s prog %d/%d VIR:\n",
2810 vir_get_stage_name(c),
2811 c->program_id, c->variant_id);
2812 vir_dump(c);
2813 fprintf(stderr, "\n");
2814 }
2815
2816 /* Attempt to allocate registers for the temporaries. If we fail,
2817 * reduce thread count and try again.
2818 */
2819 int min_threads = (c->devinfo->ver >= 41) ? 2 : 1;
2820 struct qpu_reg *temp_registers;
2821 while (true) {
2822 bool spilled;
2823 temp_registers = v3d_register_allocate(c, &spilled);
2824 if (spilled)
2825 continue;
2826
2827 if (temp_registers)
2828 break;
2829
2830 if (c->threads == min_threads) {
2831 fprintf(stderr, "Failed to register allocate at %d threads:\n",
2832 c->threads);
2833 vir_dump(c);
2834 c->failed = true;
2835 return;
2836 }
2837
2838 c->threads /= 2;
2839
2840 if (c->threads == 1)
2841 vir_remove_thrsw(c);
2842 }
2843
2844 if (c->spills &&
2845 (V3D_DEBUG & (V3D_DEBUG_VIR |
2846 v3d_debug_flag_for_shader_stage(c->s->info.stage)))) {
2847 fprintf(stderr, "%s prog %d/%d spilled VIR:\n",
2848 vir_get_stage_name(c),
2849 c->program_id, c->variant_id);
2850 vir_dump(c);
2851 fprintf(stderr, "\n");
2852 }
2853
2854 v3d_vir_to_qpu(c, temp_registers);
2855 }