nir/glsl: Add another way of doing lower_imul64 for gen8+

[mesa.git] / src / compiler / nir / nir_lower_int64.c

diff --git a/src/compiler/nir/nir_lower_int64.c b/src/compiler/nir/nir_lower_int64.c
index 1c4b4b337973bbc8b211c8188844c7ebd619cacd..6aae1816bd24a1a0a73f614370ff061df7754da9 100644 (file)
--- a/src/compiler/nir/nir_lower_int64.c
+++ b/src/compiler/nir/nir_lower_int64.c
@@ -383,6 +383,16 @@ lower_imin64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y)
    return nir_bcsel(b, lower_int64_compare(b, nir_op_ilt, x, y), x, y);
 }
 
+static nir_ssa_def *
+lower_mul_2x32_64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y,
+                  bool sign_extend)
+{
+   nir_ssa_def *res_hi = sign_extend ? nir_imul_high(b, x, y)
+                                     : nir_umul_high(b, x, y);
+
+   return nir_pack_64_2x32_split(b, nir_imul(b, x, y), res_hi);
+}
+
 static nir_ssa_def *
 lower_imul64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y)
 {
@@ -391,12 +401,13 @@ lower_imul64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y)
    nir_ssa_def *y_lo = nir_unpack_64_2x32_split_x(b, y);
    nir_ssa_def *y_hi = nir_unpack_64_2x32_split_y(b, y);
 
-   nir_ssa_def *res_lo = nir_imul(b, x_lo, y_lo);
-   nir_ssa_def *res_hi = nir_iadd(b, nir_umul_high(b, x_lo, y_lo),
+   nir_ssa_def *mul_lo = nir_umul_2x32_64(b, x_lo, y_lo);
+   nir_ssa_def *res_hi = nir_iadd(b, nir_unpack_64_2x32_split_y(b, mul_lo),
                          nir_iadd(b, nir_imul(b, x_lo, y_hi),
                                      nir_imul(b, x_hi, y_lo)));
 
-   return nir_pack_64_2x32_split(b, res_lo, res_hi);
+   return nir_pack_64_2x32_split(b, nir_unpack_64_2x32_split_x(b, mul_lo),
+                                 res_hi);
 }
 
 static nir_ssa_def *
@@ -441,9 +452,8 @@ lower_mul_high64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y,
           * so we're guaranteed that we can add in two more 32-bit values
           * without overflowing tmp.
           */
-         nir_ssa_def *tmp =
-            nir_pack_64_2x32_split(b, nir_imul(b, x32[i], y32[j]),
-                                      nir_umul_high(b, x32[i], y32[j]));
+         nir_ssa_def *tmp = nir_umul_2x32_64(b, x32[i], y32[i]);
+
          if (res[i + j])
             tmp = nir_iadd(b, tmp, nir_u2u64(b, res[i + j]));
          if (carry)
@@ -626,6 +636,9 @@ opcode_to_options_mask(nir_op opcode)
    switch (opcode) {
    case nir_op_imul:
       return nir_lower_imul64;
+   case nir_op_imul_2x32_64:
+   case nir_op_umul_2x32_64:
+      return nir_lower_imul_2x32_64;
    case nir_op_imul_high:
    case nir_op_umul_high:
       return nir_lower_imul_high64;
@@ -688,6 +701,10 @@ lower_int64_alu_instr(nir_builder *b, nir_alu_instr *alu)
    switch (alu->op) {
    case nir_op_imul:
       return lower_imul64(b, src[0], src[1]);
+   case nir_op_imul_2x32_64:
+      return lower_mul_2x32_64(b, src[0], src[1], true);
+   case nir_op_umul_2x32_64:
+      return lower_mul_2x32_64(b, src[0], src[1], false);
    case nir_op_imul_high:
       return lower_mul_high64(b, src[0], src[1], true);
    case nir_op_umul_high: