freedreno/ir3: some SSBO cleanups/fixes

Signed-off-by: Rob Clark <robdclark@gmail.com>

diff --git a/src/gallium/drivers/freedreno/ir3/ir3.h b/src/gallium/drivers/freedreno/ir3/ir3.h
index b85e5e38bda252acff3bdd2675a32a30ba191b8c..e5b1a2dce09a44b3ed032f02f45ee0d1bc9713a3 100644 (file)
--- a/src/gallium/drivers/freedreno/ir3/ir3.h
+++ b/src/gallium/drivers/freedreno/ir3/ir3.h
@@ -893,6 +893,8 @@ static inline unsigned ir3_cat3_absneg(opc_t opc)
        }
 }
 
+#define MASK(n) ((1 << (n)) - 1)
+
 /* iterator for an instructions's sources (reg), also returns src #: */
 #define foreach_src_n(__srcreg, __n, __instr) \
        if ((__instr)->regs_count) \

diff --git a/src/gallium/drivers/freedreno/ir3/ir3_compiler_nir.c b/src/gallium/drivers/freedreno/ir3/ir3_compiler_nir.c
index f464d487ca7a442f666108ba91909e63c56270da..0d642772f9e5b1db08ef4f13b631103c2599d476 100644 (file)
--- a/src/gallium/drivers/freedreno/ir3/ir3_compiler_nir.c
+++ b/src/gallium/drivers/freedreno/ir3/ir3_compiler_nir.c
@@ -1231,7 +1231,7 @@ emit_intrinsic_load_ubo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
 }
 
 static void
-mark_ssbo_read(struct ir3_context *ctx, struct ir3_instruction *instr)
+mark_read(struct ir3_context *ctx, struct ir3_instruction *instr)
 {
        instr->regs[0]->instr = ctx->last_write;
        instr->regs[0]->flags |= IR3_REG_SSA;
@@ -1239,13 +1239,14 @@ mark_ssbo_read(struct ir3_context *ctx, struct ir3_instruction *instr)
 }
 
 static void
-mark_ssbo_write(struct ir3_context *ctx, struct ir3_instruction *instr)
+mark_write(struct ir3_context *ctx, struct ir3_instruction *instr)
 {
        instr->regs[0]->instr = ctx->last_access;
        instr->regs[0]->flags |= IR3_REG_SSA;
        ctx->last_write = ctx->last_access = instr;
 }
 
+/* src[] = { buffer_index, offset }. No const_index */
 static void
 emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
                struct ir3_instruction **dst)
@@ -1269,10 +1270,11 @@ emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
 
        ldgb = ir3_LDGB(b, create_immed(b, const_offset->u32[0]), 0,
                        src0, 0, src1, 0);
-       ldgb->regs[0]->wrmask = (1 << intr->num_components) - 1;
+       ldgb->regs[0]->wrmask = MASK(intr->num_components);
        ldgb->cat6.iim_val = intr->num_components;
+       ldgb->cat6.d = 4;
        ldgb->cat6.type = TYPE_U32;
-       mark_ssbo_read(ctx, ldgb);
+       mark_read(ctx, ldgb);
 
        split_dest(b, dst, ldgb, 0, intr->num_components);
 }
@@ -1284,7 +1286,12 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
        struct ir3_block *b = ctx->block;
        struct ir3_instruction *stgb, *src0, *src1, *src2, *offset;
        nir_const_value *const_offset;
-       unsigned ncomp = ffs(~intr->const_index[0]) - 1;
+       /* TODO handle wrmask properly, see _store_shared().. but I think
+        * it is more a PITA than that, since blob ends up loading the
+        * masked components and writing them back out.
+        */
+       unsigned wrmask = intr->const_index[0];
+       unsigned ncomp = ffs(~wrmask) - 1;
 
        /* can this be non-const buffer_index?  how do we handle that? */
        const_offset = nir_src_as_const_value(intr->src[1]);
@@ -1305,8 +1312,9 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
        stgb = ir3_STGB(b, create_immed(b, const_offset->u32[0]), 0,
                        src0, 0, src1, 0, src2, 0);
        stgb->cat6.iim_val = ncomp;
+       stgb->cat6.d = 4;
        stgb->cat6.type = TYPE_U32;
-       mark_ssbo_write(ctx, stgb);
+       mark_write(ctx, stgb);
 
        array_insert(b, b->keeps, stgb);
 }
@@ -1326,8 +1334,25 @@ emit_intrinsic_ssbo_size(struct ir3_context *ctx, nir_intrinsic_instr *intr,
        dst[0] = create_uniform(ctx, idx);
 }
 
+/*
+ * SSBO atomic intrinsics
+ *
+ * All of the SSBO atomic memory operations read a value from memory,
+ * compute a new value using one of the operations below, write the new
+ * value to memory, and return the original value read.
+ *
+ * All operations take 3 sources except CompSwap that takes 4. These
+ * sources represent:
+ *
+ * 0: The SSBO buffer index.
+ * 1: The offset into the SSBO buffer of the variable that the atomic
+ *    operation will operate on.
+ * 2: The data parameter to the atomic function (i.e. the value to add
+ *    in ssbo_atomic_add, etc).
+ * 3: For CompSwap only: the second data parameter.
+ */
 static struct ir3_instruction *
-emit_intrinsic_atomic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
+emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
 {
        struct ir3_block *b = ctx->block;
        struct ir3_instruction *atomic, *ssbo, *src0, *src1, *src2, *offset;
@@ -1341,9 +1366,9 @@ emit_intrinsic_atomic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
 
        offset = get_src(ctx, &intr->src[1])[0];
 
-       /* src0 is data (or uvec2(data, compare)
+       /* src0 is data (or uvec2(data, compare))
         * src1 is offset
-        * src2 is uvec2(offset*4, 0)
+        * src2 is uvec2(offset*4, 0) (appears to be 64b byte offset)
         *
         * Note that nir already multiplies the offset by four
         */
@@ -1397,8 +1422,9 @@ emit_intrinsic_atomic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
        }
 
        atomic->cat6.iim_val = 1;
+       atomic->cat6.d = 4;
        atomic->cat6.type = type;
-       mark_ssbo_write(ctx, atomic);
+       mark_write(ctx, atomic);
 
        /* even if nothing consume the result, we can't DCE the instruction: */
        array_insert(b, b->keeps, atomic);
@@ -1558,11 +1584,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
        case nir_intrinsic_ssbo_atomic_xor:
        case nir_intrinsic_ssbo_atomic_exchange:
        case nir_intrinsic_ssbo_atomic_comp_swap:
-               if (info->has_dest) {
-                       dst[0] = emit_intrinsic_atomic(ctx, intr);
-               } else {
-                       emit_intrinsic_atomic(ctx, intr);
-               }
+               dst[0] = emit_intrinsic_atomic_ssbo(ctx, intr);
                break;
        case nir_intrinsic_barrier:
        case nir_intrinsic_memory_barrier: