projects / mesa.git / blob
? search:


573a557f63e5d67e4dfe617fe5db943edd3cbfda
[mesa.git] / src / gallium / drivers / vc4 / vc4_qpu_emit.c
1 /*
2 * Copyright © 2014 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <inttypes.h>
25
26 #include "vc4_context.h"
27 #include "vc4_qir.h"
28 #include "vc4_qpu.h"
29 #include "util/ralloc.h"
30
31 static void
32 vc4_dump_program(struct vc4_compile *c)
33 {
34 fprintf(stderr, "%s prog %d/%d QPU:\n",
35 qir_get_stage_name(c->stage),
36 c->program_id, c->variant_id);
37
38 for (int i = 0; i < c->qpu_inst_count; i++) {
39 fprintf(stderr, "0x%016"PRIx64" ", c->qpu_insts[i]);
40 vc4_qpu_disasm(&c->qpu_insts[i], 1);
41 fprintf(stderr, "\n");
42 }
43 }
44
45 static void
46 queue(struct vc4_compile *c, uint64_t inst)
47 {
48 struct queued_qpu_inst *q = rzalloc(c, struct queued_qpu_inst);
49 q->inst = inst;
50 list_addtail(&q->link, &c->qpu_inst_list);
51 }
52
53 static uint64_t *
54 last_inst(struct vc4_compile *c)
55 {
56 struct queued_qpu_inst *q =
57 (struct queued_qpu_inst *)c->qpu_inst_list.prev;
58 return &q->inst;
59 }
60
61 static void
62 set_last_cond_add(struct vc4_compile *c, uint32_t cond)
63 {
64 *last_inst(c) = qpu_set_cond_add(*last_inst(c), cond);
65 }
66
67 /**
68 * Some special registers can be read from either file, which lets us resolve
69 * raddr conflicts without extra MOVs.
70 */
71 static bool
72 swap_file(struct qpu_reg *src)
73 {
74 switch (src->addr) {
75 case QPU_R_UNIF:
76 case QPU_R_VARY:
77 if (src->mux == QPU_MUX_SMALL_IMM) {
78 return false;
79 } else {
80 if (src->mux == QPU_MUX_A)
81 src->mux = QPU_MUX_B;
82 else
83 src->mux = QPU_MUX_A;
84 return true;
85 }
86
87 default:
88 return false;
89 }
90 }
91
92 /**
93 * This is used to resolve the fact that we might register-allocate two
94 * different operands of an instruction to the same physical register file
95 * even though instructions have only one field for the register file source
96 * address.
97 *
98 * In that case, we need to move one to a temporary that can be used in the
99 * instruction, instead. We reserve ra31/rb31 for this purpose.
100 */
101 static void
102 fixup_raddr_conflict(struct vc4_compile *c,
103 struct qpu_reg dst,
104 struct qpu_reg *src0, struct qpu_reg *src1)
105 {
106 uint32_t mux0 = src0->mux == QPU_MUX_SMALL_IMM ? QPU_MUX_B : src0->mux;
107 uint32_t mux1 = src1->mux == QPU_MUX_SMALL_IMM ? QPU_MUX_B : src1->mux;
108
109 if (mux0 <= QPU_MUX_R5 ||
110 mux0 != mux1 ||
111 (src0->addr == src1->addr &&
112 src0->mux == src1->mux)) {
113 return;
114 }
115
116 if (swap_file(src0) || swap_file(src1))
117 return;
118
119 if (mux0 == QPU_MUX_A) {
120 queue(c, qpu_a_MOV(qpu_rb(31), *src0));
121 *src0 = qpu_rb(31);
122 } else {
123 queue(c, qpu_a_MOV(qpu_ra(31), *src0));
124 *src0 = qpu_ra(31);
125 }
126 }
127
128 void
129 vc4_generate_code(struct vc4_context *vc4, struct vc4_compile *c)
130 {
131 struct qpu_reg *temp_registers = vc4_register_allocate(vc4, c);
132 bool discard = false;
133 uint32_t inputs_remaining = c->num_inputs;
134 uint32_t vpm_read_fifo_count = 0;
135 uint32_t vpm_read_offset = 0;
136 int last_vpm_read_index = -1;
137 /* Map from the QIR ops enum order to QPU unpack bits. */
138 static const uint32_t unpack_map[] = {
139 QPU_UNPACK_8A,
140 QPU_UNPACK_8B,
141 QPU_UNPACK_8C,
142 QPU_UNPACK_8D,
143 QPU_UNPACK_16A_TO_F32,
144 QPU_UNPACK_16B_TO_F32,
145 };
146
147 list_inithead(&c->qpu_inst_list);
148
149 switch (c->stage) {
150 case QSTAGE_VERT:
151 case QSTAGE_COORD:
152 /* There's a 4-entry FIFO for VPMVCD reads, each of which can
153 * load up to 16 dwords (4 vec4s) per vertex.
154 */
155 while (inputs_remaining) {
156 uint32_t num_entries = MIN2(inputs_remaining, 16);
157 queue(c, qpu_load_imm_ui(qpu_vrsetup(),
158 vpm_read_offset |
159 0x00001a00 |
160 ((num_entries & 0xf) << 20)));
161 inputs_remaining -= num_entries;
162 vpm_read_offset += num_entries;
163 vpm_read_fifo_count++;
164 }
165 assert(vpm_read_fifo_count <= 4);
166
167 queue(c, qpu_load_imm_ui(qpu_vwsetup(), 0x00001a00));
168 break;
169 case QSTAGE_FRAG:
170 break;
171 }
172
173 list_for_each_entry(struct qinst, qinst, &c->instructions, link) {
174 #if 0
175 fprintf(stderr, "translating qinst to qpu: ");
176 qir_dump_inst(qinst);
177 fprintf(stderr, "\n");
178 #endif
179
180 static const struct {
181 uint32_t op;
182 bool is_mul;
183 } translate[] = {
184 #define A(name) [QOP_##name] = {QPU_A_##name, false}
185 #define M(name) [QOP_##name] = {QPU_M_##name, true}
186 A(FADD),
187 A(FSUB),
188 A(FMIN),
189 A(FMAX),
190 A(FMINABS),
191 A(FMAXABS),
192 A(FTOI),
193 A(ITOF),
194 A(ADD),
195 A(SUB),
196 A(SHL),
197 A(SHR),
198 A(ASR),
199 A(MIN),
200 A(MAX),
201 A(AND),
202 A(OR),
203 A(XOR),
204 A(NOT),
205
206 M(FMUL),
207 M(MUL24),
208 };
209
210 struct qpu_reg src[4];
211 for (int i = 0; i < qir_get_op_nsrc(qinst->op); i++) {
212 int index = qinst->src[i].index;
213 switch (qinst->src[i].file) {
214 case QFILE_NULL:
215 src[i] = qpu_rn(0);
216 break;
217 case QFILE_TEMP:
218 src[i] = temp_registers[index];
219 break;
220 case QFILE_UNIF:
221 src[i] = qpu_unif();
222 break;
223 case QFILE_VARY:
224 src[i] = qpu_vary();
225 break;
226 case QFILE_SMALL_IMM:
227 src[i].mux = QPU_MUX_SMALL_IMM;
228 src[i].addr = qpu_encode_small_immediate(qinst->src[i].index);
229 /* This should only have returned a valid
230 * small immediate field, not ~0 for failure.
231 */
232 assert(src[i].addr <= 47);
233 break;
234 case QFILE_VPM:
235 assert((int)qinst->src[i].index >=
236 last_vpm_read_index);
237 (void)last_vpm_read_index;
238 last_vpm_read_index = qinst->src[i].index;
239 src[i] = qpu_ra(QPU_R_VPM);
240 break;
241 }
242 }
243
244 struct qpu_reg dst;
245 switch (qinst->dst.file) {
246 case QFILE_NULL:
247 dst = qpu_ra(QPU_W_NOP);
248 break;
249 case QFILE_TEMP:
250 dst = temp_registers[qinst->dst.index];
251 break;
252 case QFILE_VPM:
253 dst = qpu_ra(QPU_W_VPM);
254 break;
255 case QFILE_VARY:
256 case QFILE_UNIF:
257 case QFILE_SMALL_IMM:
258 assert(!"not reached");
259 break;
260 }
261
262 switch (qinst->op) {
263 case QOP_MOV:
264 /* Skip emitting the MOV if it's a no-op. */
265 if (dst.mux == QPU_MUX_A || dst.mux == QPU_MUX_B ||
266 dst.mux != src[0].mux || dst.addr != src[0].addr) {
267 queue(c, qpu_a_MOV(dst, src[0]));
268 }
269 break;
270
271 case QOP_SEL_X_0_ZS:
272 case QOP_SEL_X_0_ZC:
273 case QOP_SEL_X_0_NS:
274 case QOP_SEL_X_0_NC:
275 queue(c, qpu_a_MOV(dst, src[0]));
276 set_last_cond_add(c, qinst->op - QOP_SEL_X_0_ZS +
277 QPU_COND_ZS);
278
279 queue(c, qpu_a_XOR(dst, qpu_r0(), qpu_r0()));
280 set_last_cond_add(c, ((qinst->op - QOP_SEL_X_0_ZS) ^
281 1) + QPU_COND_ZS);
282 break;
283
284 case QOP_SEL_X_Y_ZS:
285 case QOP_SEL_X_Y_ZC:
286 case QOP_SEL_X_Y_NS:
287 case QOP_SEL_X_Y_NC:
288 queue(c, qpu_a_MOV(dst, src[0]));
289 set_last_cond_add(c, qinst->op - QOP_SEL_X_Y_ZS +
290 QPU_COND_ZS);
291
292 queue(c, qpu_a_MOV(dst, src[1]));
293 set_last_cond_add(c, ((qinst->op - QOP_SEL_X_Y_ZS) ^
294 1) + QPU_COND_ZS);
295
296 break;
297
298 case QOP_RCP:
299 case QOP_RSQ:
300 case QOP_EXP2:
301 case QOP_LOG2:
302 switch (qinst->op) {
303 case QOP_RCP:
304 queue(c, qpu_a_MOV(qpu_rb(QPU_W_SFU_RECIP),
305 src[0]));
306 break;
307 case QOP_RSQ:
308 queue(c, qpu_a_MOV(qpu_rb(QPU_W_SFU_RECIPSQRT),
309 src[0]));
310 break;
311 case QOP_EXP2:
312 queue(c, qpu_a_MOV(qpu_rb(QPU_W_SFU_EXP),
313 src[0]));
314 break;
315 case QOP_LOG2:
316 queue(c, qpu_a_MOV(qpu_rb(QPU_W_SFU_LOG),
317 src[0]));
318 break;
319 default:
320 abort();
321 }
322
323 if (dst.mux != QPU_MUX_R4)
324 queue(c, qpu_a_MOV(dst, qpu_r4()));
325
326 break;
327
328 case QOP_PACK_8888_F:
329 queue(c, qpu_m_MOV(dst, src[0]));
330 *last_inst(c) |= QPU_PM;
331 *last_inst(c) |= QPU_SET_FIELD(QPU_PACK_MUL_8888,
332 QPU_PACK);
333 break;
334
335 case QOP_PACK_8A_F:
336 case QOP_PACK_8B_F:
337 case QOP_PACK_8C_F:
338 case QOP_PACK_8D_F:
339 queue(c,
340 qpu_m_MOV(dst, src[0]) |
341 QPU_PM |
342 QPU_SET_FIELD(QPU_PACK_MUL_8A +
343 qinst->op - QOP_PACK_8A_F,
344 QPU_PACK));
345 break;
346
347 case QOP_FRAG_X:
348 queue(c, qpu_a_ITOF(dst,
349 qpu_ra(QPU_R_XY_PIXEL_COORD)));
350 break;
351
352 case QOP_FRAG_Y:
353 queue(c, qpu_a_ITOF(dst,
354 qpu_rb(QPU_R_XY_PIXEL_COORD)));
355 break;
356
357 case QOP_FRAG_REV_FLAG:
358 queue(c, qpu_a_ITOF(dst,
359 qpu_rb(QPU_R_MS_REV_FLAGS)));
360 break;
361
362 case QOP_FRAG_Z:
363 case QOP_FRAG_W:
364 /* QOP_FRAG_Z/W don't emit instructions, just allocate
365 * the register to the Z/W payload.
366 */
367 break;
368
369 case QOP_TLB_DISCARD_SETUP:
370 discard = true;
371 queue(c, qpu_a_MOV(src[0], src[0]));
372 *last_inst(c) |= QPU_SF;
373 break;
374
375 case QOP_TLB_STENCIL_SETUP:
376 queue(c, qpu_a_MOV(qpu_ra(QPU_W_TLB_STENCIL_SETUP), src[0]));
377 break;
378
379 case QOP_TLB_Z_WRITE:
380 queue(c, qpu_a_MOV(qpu_ra(QPU_W_TLB_Z), src[0]));
381 if (discard) {
382 set_last_cond_add(c, QPU_COND_ZS);
383 }
384 break;
385
386 case QOP_TLB_COLOR_READ:
387 queue(c, qpu_NOP());
388 *last_inst(c) = qpu_set_sig(*last_inst(c),
389 QPU_SIG_COLOR_LOAD);
390
391 if (dst.mux != QPU_MUX_R4)
392 queue(c, qpu_a_MOV(dst, qpu_r4()));
393 break;
394
395 case QOP_TLB_COLOR_WRITE:
396 queue(c, qpu_a_MOV(qpu_tlbc(), src[0]));
397 if (discard) {
398 set_last_cond_add(c, QPU_COND_ZS);
399 }
400 break;
401
402 case QOP_VARY_ADD_C:
403 queue(c, qpu_a_FADD(dst, src[0], qpu_r5()));
404 break;
405
406 case QOP_PACK_16A_I:
407 case QOP_PACK_16B_I:
408 queue(c,
409 qpu_a_MOV(dst, src[0]) |
410 QPU_SET_FIELD(qinst->op == QOP_PACK_16A_I ?
411 QPU_PACK_A_16A : QPU_PACK_A_16B,
412 QPU_PACK));
413 break;
414
415 case QOP_TEX_S:
416 case QOP_TEX_T:
417 case QOP_TEX_R:
418 case QOP_TEX_B:
419 queue(c, qpu_a_MOV(qpu_rb(QPU_W_TMU0_S +
420 (qinst->op - QOP_TEX_S)),
421 src[0]));
422 break;
423
424 case QOP_TEX_DIRECT:
425 fixup_raddr_conflict(c, dst, &src[0], &src[1]);
426 queue(c, qpu_a_ADD(qpu_rb(QPU_W_TMU0_S), src[0], src[1]));
427 break;
428
429 case QOP_TEX_RESULT:
430 queue(c, qpu_NOP());
431 *last_inst(c) = qpu_set_sig(*last_inst(c),
432 QPU_SIG_LOAD_TMU0);
433 if (dst.mux != QPU_MUX_R4)
434 queue(c, qpu_a_MOV(dst, qpu_r4()));
435 break;
436
437 case QOP_UNPACK_8A_F:
438 case QOP_UNPACK_8B_F:
439 case QOP_UNPACK_8C_F:
440 case QOP_UNPACK_8D_F:
441 case QOP_UNPACK_16A_F:
442 case QOP_UNPACK_16B_F: {
443 if (src[0].mux == QPU_MUX_R4) {
444 queue(c, qpu_a_MOV(dst, src[0]));
445 *last_inst(c) |= QPU_PM;
446 *last_inst(c) |= QPU_SET_FIELD(QPU_UNPACK_8A +
447 (qinst->op -
448 QOP_UNPACK_8A_F),
449 QPU_UNPACK);
450 } else {
451 assert(src[0].mux == QPU_MUX_A);
452
453 /* Since we're setting the pack bits, if the
454 * destination is in A it would get re-packed.
455 */
456 queue(c, qpu_a_FMAX((dst.mux == QPU_MUX_A ?
457 qpu_rb(31) : dst),
458 src[0], src[0]));
459 *last_inst(c) |=
460 QPU_SET_FIELD(unpack_map[qinst->op -
461 QOP_UNPACK_8A_F],
462 QPU_UNPACK);
463
464 if (dst.mux == QPU_MUX_A) {
465 queue(c, qpu_a_MOV(dst, qpu_rb(31)));
466 }
467 }
468 }
469 break;
470
471 case QOP_UNPACK_8A_I:
472 case QOP_UNPACK_8B_I:
473 case QOP_UNPACK_8C_I:
474 case QOP_UNPACK_8D_I:
475 case QOP_UNPACK_16A_I:
476 case QOP_UNPACK_16B_I: {
477 assert(src[0].mux == QPU_MUX_A);
478
479 /* Since we're setting the pack bits, if the
480 * destination is in A it would get re-packed.
481 */
482 queue(c, qpu_a_MOV((dst.mux == QPU_MUX_A ?
483 qpu_rb(31) : dst), src[0]));
484 *last_inst(c) |= QPU_SET_FIELD(unpack_map[qinst->op -
485 QOP_UNPACK_8A_I],
486 QPU_UNPACK);
487
488 if (dst.mux == QPU_MUX_A) {
489 queue(c, qpu_a_MOV(dst, qpu_rb(31)));
490 }
491 }
492 break;
493
494 default:
495 assert(qinst->op < ARRAY_SIZE(translate));
496 assert(translate[qinst->op].op != 0); /* NOPs */
497
498 /* If we have only one source, put it in the second
499 * argument slot as well so that we don't take up
500 * another raddr just to get unused data.
501 */
502 if (qir_get_op_nsrc(qinst->op) == 1)
503 src[1] = src[0];
504
505 fixup_raddr_conflict(c, dst, &src[0], &src[1]);
506
507 if (translate[qinst->op].is_mul) {
508 queue(c, qpu_m_alu2(translate[qinst->op].op,
509 dst,
510 src[0], src[1]));
511 } else {
512 queue(c, qpu_a_alu2(translate[qinst->op].op,
513 dst,
514 src[0], src[1]));
515 }
516
517 break;
518 }
519
520 if (qinst->sf) {
521 assert(!qir_is_multi_instruction(qinst));
522 *last_inst(c) |= QPU_SF;
523 }
524 }
525
526 qpu_schedule_instructions(c);
527
528 /* thread end can't have VPM write or read */
529 if (QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
530 QPU_WADDR_ADD) == QPU_W_VPM ||
531 QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
532 QPU_WADDR_MUL) == QPU_W_VPM ||
533 QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
534 QPU_RADDR_A) == QPU_R_VPM ||
535 QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
536 QPU_RADDR_B) == QPU_R_VPM) {
537 qpu_serialize_one_inst(c, qpu_NOP());
538 }
539
540 /* thread end can't have uniform read */
541 if (QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
542 QPU_RADDR_A) == QPU_R_UNIF ||
543 QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
544 QPU_RADDR_B) == QPU_R_UNIF) {
545 qpu_serialize_one_inst(c, qpu_NOP());
546 }
547
548 /* thread end can't have TLB operations */
549 if (qpu_inst_is_tlb(c->qpu_insts[c->qpu_inst_count - 1]))
550 qpu_serialize_one_inst(c, qpu_NOP());
551
552 c->qpu_insts[c->qpu_inst_count - 1] =
553 qpu_set_sig(c->qpu_insts[c->qpu_inst_count - 1],
554 QPU_SIG_PROG_END);
555 qpu_serialize_one_inst(c, qpu_NOP());
556 qpu_serialize_one_inst(c, qpu_NOP());
557
558 switch (c->stage) {
559 case QSTAGE_VERT:
560 case QSTAGE_COORD:
561 break;
562 case QSTAGE_FRAG:
563 c->qpu_insts[c->qpu_inst_count - 1] =
564 qpu_set_sig(c->qpu_insts[c->qpu_inst_count - 1],
565 QPU_SIG_SCOREBOARD_UNLOCK);
566 break;
567 }
568
569 if (vc4_debug & VC4_DEBUG_QPU)
570 vc4_dump_program(c);
571
572 vc4_qpu_validate(c->qpu_insts, c->qpu_inst_count);
573
574 free(temp_registers);
575 }