+static void
+set_last_cond_add(struct vc4_compile *c, uint32_t cond)
+{
+ *last_inst(c) = qpu_set_cond_add(*last_inst(c), cond);
+}
+
+/**
+ * Some special registers can be read from either file, which lets us resolve
+ * raddr conflicts without extra MOVs.
+ */
+static bool
+swap_file(struct qpu_reg *src)
+{
+ switch (src->addr) {
+ case QPU_R_UNIF:
+ case QPU_R_VARY:
+ if (src->mux == QPU_MUX_A)
+ src->mux = QPU_MUX_B;
+ else
+ src->mux = QPU_MUX_A;
+ return true;
+
+ default:
+ return false;
+ }
+}
+
+/**
+ * This is used to resolve the fact that we might register-allocate two
+ * different operands of an instruction to the same physical register file
+ * even though instructions have only one field for the register file source
+ * address.
+ *
+ * In that case, we need to move one to a temporary that can be used in the
+ * instruction, instead.
+ */
+static void
+fixup_raddr_conflict(struct vc4_compile *c,
+ struct qpu_reg *src0, struct qpu_reg *src1)
+{
+ if ((src0->mux != QPU_MUX_A && src0->mux != QPU_MUX_B) ||
+ src0->mux != src1->mux ||
+ src0->addr == src1->addr) {
+ return;
+ }
+
+ if (swap_file(src0) || swap_file(src1))
+ return;
+
+ queue(c, qpu_a_MOV(qpu_r3(), *src1));
+ *src1 = qpu_r3();
+}
+
+static void
+serialize_one_inst(struct vc4_compile *c, uint64_t inst)
+{
+ if (c->qpu_inst_count >= c->qpu_inst_size) {
+ c->qpu_inst_size = MAX2(16, c->qpu_inst_size * 2);
+ c->qpu_insts = realloc(c->qpu_insts,
+ c->qpu_inst_size * sizeof(uint64_t));
+ }
+ c->qpu_insts[c->qpu_inst_count++] = inst;
+}
+
+static void
+serialize_insts(struct vc4_compile *c)
+{
+ int last_sfu_write = -10;
+ bool scoreboard_wait_emitted = false;
+
+ while (!is_empty_list(&c->qpu_inst_list)) {
+ struct queued_qpu_inst *q =
+ (struct queued_qpu_inst *)first_elem(&c->qpu_inst_list);
+ uint32_t last_waddr_a = QPU_W_NOP, last_waddr_b = QPU_W_NOP;
+ uint32_t raddr_a = QPU_GET_FIELD(q->inst, QPU_RADDR_A);
+ uint32_t raddr_b = QPU_GET_FIELD(q->inst, QPU_RADDR_B);
+
+ if (c->qpu_inst_count > 0) {
+ uint64_t last_inst = c->qpu_insts[c->qpu_inst_count -
+ 1];
+ uint32_t last_waddr_add = QPU_GET_FIELD(last_inst,
+ QPU_WADDR_ADD);
+ uint32_t last_waddr_mul = QPU_GET_FIELD(last_inst,
+ QPU_WADDR_MUL);
+
+ if (last_inst & QPU_WS) {
+ last_waddr_a = last_waddr_mul;
+ last_waddr_b = last_waddr_add;
+ } else {
+ last_waddr_a = last_waddr_add;
+ last_waddr_b = last_waddr_mul;
+ }
+ }
+
+ uint32_t src_muxes[] = {
+ QPU_GET_FIELD(q->inst, QPU_ADD_A),
+ QPU_GET_FIELD(q->inst, QPU_ADD_B),
+ QPU_GET_FIELD(q->inst, QPU_MUL_A),
+ QPU_GET_FIELD(q->inst, QPU_MUL_B),
+ };
+
+ /* "An instruction must not read from a location in physical
+ * regfile A or B that was written to by the previous
+ * instruction."
+ */
+ bool needs_raddr_vs_waddr_nop = false;
+ bool reads_r4 = false;
+ for (int i = 0; i < ARRAY_SIZE(src_muxes); i++) {
+ if ((raddr_a < 32 &&
+ src_muxes[i] == QPU_MUX_A &&
+ last_waddr_a == raddr_a) ||
+ (raddr_b < 32 &&
+ src_muxes[i] == QPU_MUX_B &&
+ last_waddr_b == raddr_b)) {
+ needs_raddr_vs_waddr_nop = true;
+ }
+ if (src_muxes[i] == QPU_MUX_R4)
+ reads_r4 = true;
+ }
+
+ if (needs_raddr_vs_waddr_nop) {
+ serialize_one_inst(c, qpu_NOP());
+ }
+
+ /* "After an SFU lookup instruction, accumulator r4 must not
+ * be read in the following two instructions. Any other
+ * instruction that results in r4 being written (that is, TMU
+ * read, TLB read, SFU lookup) cannot occur in the two
+ * instructions following an SFU lookup."
+ */
+ if (reads_r4) {
+ while (c->qpu_inst_count - last_sfu_write < 3) {
+ serialize_one_inst(c, qpu_NOP());
+ }
+ }
+
+ uint32_t waddr_a = QPU_GET_FIELD(q->inst, QPU_WADDR_ADD);
+ uint32_t waddr_m = QPU_GET_FIELD(q->inst, QPU_WADDR_MUL);
+ if ((waddr_a >= QPU_W_SFU_RECIP && waddr_a <= QPU_W_SFU_LOG) ||
+ (waddr_m >= QPU_W_SFU_RECIP && waddr_m <= QPU_W_SFU_LOG)) {
+ last_sfu_write = c->qpu_inst_count;
+ }
+
+ /* "A scoreboard wait must not occur in the first two
+ * instructions of a fragment shader. This is either the
+ * explicit Wait for Scoreboard signal or an implicit wait
+ * with the first tile-buffer read or write instruction."
+ */
+ if (!scoreboard_wait_emitted &&
+ (waddr_a == QPU_W_TLB_Z || waddr_m == QPU_W_TLB_Z ||
+ waddr_a == QPU_W_TLB_COLOR_MS ||
+ waddr_m == QPU_W_TLB_COLOR_MS ||
+ waddr_a == QPU_W_TLB_COLOR_ALL ||
+ waddr_m == QPU_W_TLB_COLOR_ALL ||
+ QPU_GET_FIELD(q->inst, QPU_SIG) == QPU_SIG_COLOR_LOAD)) {
+ while (c->qpu_inst_count < 3 ||
+ QPU_GET_FIELD(c->qpu_insts[c->qpu_inst_count - 1],
+ QPU_SIG) != QPU_SIG_NONE) {
+ serialize_one_inst(c, qpu_NOP());
+ }
+ c->qpu_insts[c->qpu_inst_count - 1] =
+ qpu_set_sig(c->qpu_insts[c->qpu_inst_count - 1],
+ QPU_SIG_WAIT_FOR_SCOREBOARD);
+ scoreboard_wait_emitted = true;