bifrost: Add support for nir_op_imul

Unfortunately this doesn't map nicely to the existing instruction
classes, so we'll make a new one for now.

Signed-off-by: Chris Forbes <chrisforbes@google.com>
Reviewed-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6091>

diff --git a/src/panfrost/bifrost/bi_pack.c b/src/panfrost/bifrost/bi_pack.c
index 99a155fd56acc4798932ae1ca603a1f08338c708..b7ae11327934ec4a42000f044ffd3144f3a32f1d 100644 (file)
--- a/src/panfrost/bifrost/bi_pack.c
+++ b/src/panfrost/bifrost/bi_pack.c
@@ -1141,6 +1141,14 @@ bi_pack_fma_imath(bi_instruction *ins, bi_registers *regs)
         return bi_pack_fma_2src(ins, regs, op);
 }
 
+static unsigned
+bi_pack_fma_imul(bi_instruction *ins, bi_registers *regs)
+{
+        assert(ins->op.imul == BI_IMUL_IMUL);
+        unsigned op = BIFROST_FMA_IMUL_32;
+        return bi_pack_fma_2src(ins, regs, op);
+}
+
 static unsigned
 bi_pack_fma(bi_clause *clause, bi_bundle bundle, bi_registers *regs)
 {
@@ -1174,6 +1182,8 @@ bi_pack_fma(bi_clause *clause, bi_bundle bundle, bi_registers *regs)
                 return bi_pack_fma_round(bundle.fma, regs);
         case BI_REDUCE_FMA:
                 return bi_pack_fma_reduce(bundle.fma, regs);
+        case BI_IMUL:
+                return bi_pack_fma_imul(bundle.fma, regs);
         default:
                 unreachable("Cannot encode class as FMA");
         }

diff --git a/src/panfrost/bifrost/bi_print.c b/src/panfrost/bifrost/bi_print.c
index e8fb326271b04c57f53cb519c235d8576d788d28..6dcfdbd3e78a5c2c2dc3658ee55d5df09ff09fbd 100644 (file)
--- a/src/panfrost/bifrost/bi_print.c
+++ b/src/panfrost/bifrost/bi_print.c
@@ -155,6 +155,7 @@ bi_class_name(enum bi_class cl)
         case BI_TABLE: return "table";
         case BI_TEX: return "tex";
         case BI_ROUND: return "round";
+        case BI_IMUL: return "imul";
         default: return "unknown_class";
         }
 }

diff --git a/src/panfrost/bifrost/bi_tables.c b/src/panfrost/bifrost/bi_tables.c
index 81365937f042718f0e761cf83d1e3384c3757c56..942de5eae5af910fa1faec99d72e7ff0a1fcb84e 100644 (file)
--- a/src/panfrost/bifrost/bi_tables.c
+++ b/src/panfrost/bifrost/bi_tables.c
@@ -56,4 +56,5 @@ unsigned bi_class_props[BI_NUM_CLASSES] = {
         [BI_SELECT]             = BI_SCHED_ALL | BI_SWIZZLABLE,
         [BI_TEX]               = BI_SCHED_HI_LATENCY | BI_SCHED_ADD | BI_VECTOR | BI_DATA_REG_DEST,
         [BI_ROUND]             = BI_ROUNDMODE | BI_SCHED_ALL,
+        [BI_IMUL]       = BI_SCHED_FMA,
 };

diff --git a/src/panfrost/bifrost/bifrost.h b/src/panfrost/bifrost/bifrost.h
index 0864b3a497e3f197f7683b9c3e98aecb5c9897c2..a41076bcea3122edf3b67fdd8bcd58ddfb629141 100644 (file)
--- a/src/panfrost/bifrost/bifrost.h
+++ b/src/panfrost/bifrost/bifrost.h
@@ -117,6 +117,7 @@ struct bifrost_fma_inst {
 
 #define BIFROST_FMA_IADD_32 (0x4ff98 >> 3)
 #define BIFROST_FMA_ISUB_32 (0x4ffd8 >> 3)
+#define BIFROST_FMA_IMUL_32 ((BIFROST_FMA_EXT | 0x7818) >> 3)
 
 struct bifrost_fma_2src {
         unsigned src0 : 3;

diff --git a/src/panfrost/bifrost/bifrost_compile.c b/src/panfrost/bifrost/bifrost_compile.c
index 566c0e5988144f0ad90bb3c935d238f40d4936fe..2419a4fbcf5fa39b2b274a19f57c5289cb1dd806 100644 (file)
--- a/src/panfrost/bifrost/bifrost_compile.c
+++ b/src/panfrost/bifrost/bifrost_compile.c
@@ -512,6 +512,9 @@ bi_class_for_nir_alu(nir_op op)
         case nir_op_isub:
                 return BI_IMATH;
 
+        case nir_op_imul:
+                return BI_IMUL;
+
         case nir_op_iand:
         case nir_op_ior:
         case nir_op_ixor:
@@ -821,6 +824,9 @@ emit_alu(bi_context *ctx, nir_alu_instr *instr)
                 alu.src_types[2] = alu.src_types[1];
                 alu.src[1] = BIR_INDEX_ZERO;
                 break;
+        case nir_op_imul:
+                alu.op.imul = BI_IMUL_IMUL;
+                break;
         case nir_op_fmax:
         case nir_op_imax:
         case nir_op_umax:

diff --git a/src/panfrost/bifrost/compiler.h b/src/panfrost/bifrost/compiler.h
index 2038a3299d15359b75060225d245fadf37137217..4c77ac5edf9fda197c7fe61b3b86e2fc8a968f64 100644 (file)
--- a/src/panfrost/bifrost/compiler.h
+++ b/src/panfrost/bifrost/compiler.h
@@ -76,6 +76,7 @@ enum bi_class {
         BI_TABLE,
         BI_TEX,
         BI_ROUND,
+        BI_IMUL,
         BI_NUM_CLASSES
 };
 
@@ -171,6 +172,10 @@ enum bi_imath_op {
         BI_IMATH_SUB,
 };
 
+enum bi_imul_op {
+        BI_IMUL_IMUL,
+};
+
 enum bi_table_op {
         /* fp32 log2() with low precision, suitable for GL or half_log2() in
          * CL. In the first argument, takes x. Letting u be such that x =
@@ -281,6 +286,7 @@ typedef struct {
                 enum bi_frexp_op frexp;
                 enum bi_tex_op texture;
                 enum bi_imath_op imath;
+                enum bi_imul_op imul;
 
                 /* For FMA/ADD, should we add a biased exponent? */
                 bool mscale;