+static void
+emit_alu_add(struct iris_batch *batch, unsigned dst_reg,
+ unsigned reg_a, unsigned reg_b)
+{
+ uint32_t *math = iris_get_command_space(batch, 5 * sizeof(uint32_t));
+
+ math[0] = MI_MATH | (5 - 2);
+ math[1] = _MI_ALU2(LOAD, MI_ALU_SRCA, reg_a);
+ math[2] = _MI_ALU2(LOAD, MI_ALU_SRCB, reg_b);
+ math[3] = _MI_ALU0(ADD);
+ math[4] = _MI_ALU2(STORE, dst_reg, MI_ALU_ACCU);
+}
+
+static void
+emit_alu_shl(struct iris_batch *batch, unsigned dst_reg,
+ unsigned src_reg, unsigned shift)
+{
+ assert(shift > 0);
+
+ int dwords = 1 + 4 * shift;
+
+ uint32_t *math = iris_get_command_space(batch, sizeof(uint32_t) * dwords);
+
+ math[0] = MI_MATH | ((1 + 4 * shift) - 2);
+
+ for (unsigned i = 0; i < shift; i++) {
+ unsigned add_src = (i == 0) ? src_reg : dst_reg;
+ math[1 + (i * 4) + 0] = _MI_ALU2(LOAD, MI_ALU_SRCA, add_src);
+ math[1 + (i * 4) + 1] = _MI_ALU2(LOAD, MI_ALU_SRCB, add_src);
+ math[1 + (i * 4) + 2] = _MI_ALU0(ADD);
+ math[1 + (i * 4) + 3] = _MI_ALU2(STORE, dst_reg, MI_ALU_ACCU);
+ }
+}
+
+/* Emit dwords to multiply GPR0 by N */
+static void
+build_alu_multiply_gpr0(uint32_t *dw, unsigned *dw_count, uint32_t N)
+{
+ VK_OUTARRAY_MAKE(out, dw, dw_count);
+
+#define APPEND_ALU(op, x, y) \
+ vk_outarray_append(&out, alu_dw) *alu_dw = _MI_ALU(MI_ALU_##op, x, y)
+
+ assert(N > 0);
+ unsigned top_bit = 31 - __builtin_clz(N);
+ for (int i = top_bit - 1; i >= 0; i--) {
+ /* We get our initial data in GPR0 and we write the final data out to
+ * GPR0 but we use GPR1 as our scratch register.
+ */
+ unsigned src_reg = i == top_bit - 1 ? MI_ALU_R0 : MI_ALU_R1;
+ unsigned dst_reg = i == 0 ? MI_ALU_R0 : MI_ALU_R1;
+
+ /* Shift the current value left by 1 */
+ APPEND_ALU(LOAD, MI_ALU_SRCA, src_reg);
+ APPEND_ALU(LOAD, MI_ALU_SRCB, src_reg);
+ APPEND_ALU(ADD, 0, 0);
+
+ if (N & (1 << i)) {
+ /* Store ACCU to R1 and add R0 to R1 */
+ APPEND_ALU(STORE, MI_ALU_R1, MI_ALU_ACCU);
+ APPEND_ALU(LOAD, MI_ALU_SRCA, MI_ALU_R0);
+ APPEND_ALU(LOAD, MI_ALU_SRCB, MI_ALU_R1);
+ APPEND_ALU(ADD, 0, 0);
+ }
+
+ APPEND_ALU(STORE, dst_reg, MI_ALU_ACCU);
+ }
+
+#undef APPEND_ALU
+}
+
+static void
+emit_mul_gpr0(struct iris_batch *batch, uint32_t N)
+{
+ uint32_t num_dwords;
+ build_alu_multiply_gpr0(NULL, &num_dwords, N);
+
+ uint32_t *math = iris_get_command_space(batch, 4 * num_dwords);
+ math[0] = MI_MATH | (num_dwords - 2);
+ build_alu_multiply_gpr0(&math[1], &num_dwords, N);
+}
+
+void
+iris_math_div32_gpr0(struct iris_context *ice,
+ struct iris_batch *batch,
+ uint32_t D)
+{
+ /* Zero out the top of GPR0 */
+ emit_lri32(batch, CS_GPR(0) + 4, 0);
+
+ if (D == 0) {
+ /* This invalid, but we should do something so we set GPR0 to 0. */
+ emit_lri32(batch, CS_GPR(0), 0);
+ } else if (util_is_power_of_two_or_zero(D)) {
+ unsigned log2_D = util_logbase2(D);
+ assert(log2_D < 32);
+ /* We right-shift by log2(D) by left-shifting by 32 - log2(D) and taking
+ * the top 32 bits of the result.
+ */
+ emit_alu_shl(batch, MI_ALU_R0, MI_ALU_R0, 32 - log2_D);
+ emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri32(batch, CS_GPR(0) + 4, 0);
+ } else {
+ struct util_fast_udiv_info m = util_compute_fast_udiv_info(D, 32, 32);
+ assert(m.multiplier <= UINT32_MAX);
+
+ if (m.pre_shift) {
+ /* We right-shift by L by left-shifting by 32 - l and taking the top
+ * 32 bits of the result.
+ */
+ if (m.pre_shift < 32)
+ emit_alu_shl(batch, MI_ALU_R0, MI_ALU_R0, 32 - m.pre_shift);
+ emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri32(batch, CS_GPR(0) + 4, 0);
+ }
+
+ /* Do the 32x32 multiply into gpr0 */
+ emit_mul_gpr0(batch, m.multiplier);
+
+ if (m.increment) {
+ /* If we need to increment, save off a copy of GPR0 */
+ emit_lri32(batch, CS_GPR(1) + 0, m.multiplier);
+ emit_lri32(batch, CS_GPR(1) + 4, 0);
+ emit_alu_add(batch, MI_ALU_R0, MI_ALU_R0, MI_ALU_R1);
+ }
+
+ /* Shift by 32 */
+ emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri32(batch, CS_GPR(0) + 4, 0);
+
+ if (m.post_shift) {
+ /* We right-shift by L by left-shifting by 32 - l and taking the top
+ * 32 bits of the result.
+ */
+ if (m.post_shift < 32)
+ emit_alu_shl(batch, MI_ALU_R0, MI_ALU_R0, 32 - m.post_shift);
+ emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri32(batch, CS_GPR(0) + 4, 0);
+ }
+ }
+}
+
+void
+iris_math_add32_gpr0(struct iris_context *ice,
+ struct iris_batch *batch,
+ uint32_t x)
+{
+ emit_lri32(batch, CS_GPR(1), x);
+ emit_alu_add(batch, MI_ALU_R0, MI_ALU_R0, MI_ALU_R1);
+}
+
+/*
+ * GPR0 = (GPR0 == 0) ? 0 : 1;
+ */
+static void
+gpr0_to_bool(struct iris_context *ice)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+
+ ice->vtbl.load_register_imm64(batch, CS_GPR(1), 1ull);
+
+ static const uint32_t math[] = {
+ MI_MATH | (9 - 2),
+ MI_ALU2(LOAD, SRCA, R0),
+ MI_ALU1(LOAD0, SRCB),
+ MI_ALU0(ADD),
+ MI_ALU2(STOREINV, R0, ZF),
+ MI_ALU2(LOAD, SRCA, R0),
+ MI_ALU2(LOAD, SRCB, R1),
+ MI_ALU0(AND),
+ MI_ALU2(STORE, R0, ACCU),
+ };
+ iris_batch_emit(batch, math, sizeof(math));
+}
+
+static void
+load_overflow_data_to_cs_gprs(struct iris_context *ice,
+ struct iris_query *q,
+ int idx)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+ struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res);
+ uint32_t offset = q->query_state_ref.offset;
+
+ ice->vtbl.load_register_mem64(batch, CS_GPR(1), bo, offset +
+ offsetof(struct iris_query_so_overflow,
+ stream[idx].prim_storage_needed[0]));
+ ice->vtbl.load_register_mem64(batch, CS_GPR(2), bo, offset +
+ offsetof(struct iris_query_so_overflow,
+ stream[idx].prim_storage_needed[1]));
+
+ ice->vtbl.load_register_mem64(batch, CS_GPR(3), bo, offset +
+ offsetof(struct iris_query_so_overflow,
+ stream[idx].num_prims[0]));
+ ice->vtbl.load_register_mem64(batch, CS_GPR(4), bo, offset +
+ offsetof(struct iris_query_so_overflow,
+ stream[idx].num_prims[1]));
+}
+
+/*
+ * R3 = R4 - R3;
+ * R1 = R2 - R1;
+ * R1 = R3 - R1;
+ * R0 = R0 | R1;
+ */
+static void
+calc_overflow_for_stream(struct iris_context *ice)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+ static const uint32_t maths[] = {
+ MI_MATH | (17 - 2),
+ MI_ALU2(LOAD, SRCA, R4),
+ MI_ALU2(LOAD, SRCB, R3),
+ MI_ALU0(SUB),
+ MI_ALU2(STORE, R3, ACCU),
+ MI_ALU2(LOAD, SRCA, R2),
+ MI_ALU2(LOAD, SRCB, R1),
+ MI_ALU0(SUB),
+ MI_ALU2(STORE, R1, ACCU),
+ MI_ALU2(LOAD, SRCA, R3),
+ MI_ALU2(LOAD, SRCB, R1),
+ MI_ALU0(SUB),
+ MI_ALU2(STORE, R1, ACCU),
+ MI_ALU2(LOAD, SRCA, R1),
+ MI_ALU2(LOAD, SRCB, R0),
+ MI_ALU0(OR),
+ MI_ALU2(STORE, R0, ACCU),
+ };
+
+ iris_batch_emit(batch, maths, sizeof(maths));
+}
+
+static void
+overflow_result_to_gpr0(struct iris_context *ice, struct iris_query *q)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+
+ ice->vtbl.load_register_imm64(batch, CS_GPR(0), 0ull);
+
+ if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) {
+ load_overflow_data_to_cs_gprs(ice, q, q->index);
+ calc_overflow_for_stream(ice);
+ } else {
+ for (int i = 0; i < MAX_VERTEX_STREAMS; i++) {
+ load_overflow_data_to_cs_gprs(ice, q, i);
+ calc_overflow_for_stream(ice);
+ }
+ }
+
+ gpr0_to_bool(ice);
+}
+
+/*
+ * GPR0 = GPR0 & ((1ull << n) -1);
+ */
+static void
+keep_gpr0_lower_n_bits(struct iris_context *ice, uint32_t n)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+
+ ice->vtbl.load_register_imm64(batch, CS_GPR(1), (1ull << n) - 1);
+ static const uint32_t math[] = {
+ MI_MATH | (5 - 2),
+ MI_ALU2(LOAD, SRCA, R0),
+ MI_ALU2(LOAD, SRCB, R1),
+ MI_ALU0(AND),
+ MI_ALU2(STORE, R0, ACCU),
+ };
+ iris_batch_emit(batch, math, sizeof(math));
+}
+
+/*
+ * GPR0 = GPR0 << 30;
+ */
+static void
+shl_gpr0_by_30_bits(struct iris_context *ice)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+ /* First we mask 34 bits of GPR0 to prevent overflow */
+ keep_gpr0_lower_n_bits(ice, 34);
+
+ static const uint32_t shl_math[] = {
+ MI_ALU2(LOAD, SRCA, R0),
+ MI_ALU2(LOAD, SRCB, R0),
+ MI_ALU0(ADD),
+ MI_ALU2(STORE, R0, ACCU),
+ };
+
+ const uint32_t outer_count = 5;
+ const uint32_t inner_count = 6;
+ const uint32_t cmd_len = 1 + inner_count * ARRAY_SIZE(shl_math);
+ const uint32_t batch_len = cmd_len * outer_count;
+ uint32_t *map = iris_get_command_space(batch, batch_len * 4);
+ uint32_t offset = 0;
+ for (int o = 0; o < outer_count; o++) {
+ map[offset++] = MI_MATH | (cmd_len - 2);
+ for (int i = 0; i < inner_count; i++) {
+ memcpy(&map[offset], shl_math, sizeof(shl_math));
+ offset += 4;
+ }
+ }
+}
+
+/*
+ * GPR0 = GPR0 >> 2;
+ *
+ * Note that the upper 30 bits of GPR0 are lost!
+ */
+static void
+shr_gpr0_by_2_bits(struct iris_context *ice)
+{
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+ shl_gpr0_by_30_bits(ice);
+ ice->vtbl.load_register_reg32(batch, CS_GPR(0) + 4, CS_GPR(0));
+ ice->vtbl.load_register_imm32(batch, CS_GPR(0) + 4, 0);
+}
+