X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fnir%2Fnir_lower_double_ops.c;h=a6a9b9584261419f797a16e2aad392ce19cb5992;hb=a1ed83fddd7518cc92b609d60129426bd9b4158d;hp=5a7eab807151951693e2745689e9b3124b91caa4;hpb=fe2cbcf3ee97566815c422391aeb345000ab733a;p=mesa.git

diff --git a/src/compiler/nir/nir_lower_double_ops.c b/src/compiler/nir/nir_lower_double_ops.c
index 5a7eab80715..a6a9b958426 100644
--- a/src/compiler/nir/nir_lower_double_ops.c
+++ b/src/compiler/nir/nir_lower_double_ops.c
@@ -26,6 +26,8 @@
 #include "nir_builder.h"
 #include "c99_math.h"
 
+#include <float.h>
+
 /*
  * Lowers some unsupported double operations, using only:
  *
@@ -47,7 +49,9 @@ set_exponent(nir_builder *b, nir_ssa_def *src, nir_ssa_def *exp)
    /* The exponent is bits 52-62, or 20-30 of the high word, so set the exponent
     * to 1023
     */
-   nir_ssa_def *new_hi = nir_bfi(b, nir_imm_int(b, 0x7ff00000), exp, hi);
+   nir_ssa_def *new_hi = nir_bitfield_insert(b, hi, exp,
+                                             nir_imm_int(b, 20),
+                                             nir_imm_int(b, 11));
    /* recombine */
    return nir_pack_64_2x32_split(b, lo, new_hi);
 }
@@ -142,8 +146,8 @@ lower_rcp(nir_builder *b, nir_ssa_def *src)
     * See https://en.wikipedia.org/wiki/Division_algorithm for more details.
     */
 
-   ra = nir_ffma(b, ra, nir_ffma(b, ra, src, nir_imm_double(b, -1)), ra);
-   ra = nir_ffma(b, ra, nir_ffma(b, ra, src, nir_imm_double(b, -1)), ra);
+   ra = nir_ffma(b, nir_fneg(b, ra), nir_ffma(b, ra, src, nir_imm_double(b, -1)), ra);
+   ra = nir_ffma(b, nir_fneg(b, ra), nir_ffma(b, ra, src, nir_imm_double(b, -1)), ra);
 
    return fix_inv_result(b, ra, src, new_exp);
 }
@@ -289,9 +293,20 @@ lower_sqrt_rsq(nir_builder *b, nir_ssa_def *src, bool sqrt)
        * 0 -> 0 and
        * +inf -> +inf
        */
-      res = nir_bcsel(b, nir_ior(b, nir_feq(b, src, nir_imm_double(b, 0.0)),
+      const bool preserve_denorms =
+         b->shader->info.float_controls_execution_mode &
+         FLOAT_CONTROLS_DENORM_PRESERVE_FP64;
+      nir_ssa_def *src_flushed = src;
+      if (!preserve_denorms) {
+         src_flushed = nir_bcsel(b,
+                                 nir_flt(b, nir_fabs(b, src),
+                                         nir_imm_double(b, DBL_MIN)),
+                                 nir_imm_double(b, 0.0),
+                                 src);
+      }
+      res = nir_bcsel(b, nir_ior(b, nir_feq(b, src_flushed, nir_imm_double(b, 0.0)),
                                  nir_feq(b, src, nir_imm_double(b, INFINITY))),
-                                 src, res);
+                                 src_flushed, res);
    } else {
       res = fix_inv_result(b, res, src, new_exp);
    }
@@ -392,50 +407,18 @@ lower_fract(nir_builder *b, nir_ssa_def *src)
 static nir_ssa_def *
 lower_round_even(nir_builder *b, nir_ssa_def *src)
 {
-   /* If fract(src) == 0.5, then we will have to decide the rounding direction.
-    * We will do this by computing the mod(abs(src), 2) and testing if it
-    * is < 1 or not.
-    *
-    * We compute mod(abs(src), 2) as:
-    * abs(src) - 2.0 * floor(abs(src) / 2.0)
-    */
-   nir_ssa_def *two = nir_imm_double(b, 2.0);
-   nir_ssa_def *abs_src = nir_fabs(b, src);
-   nir_ssa_def *mod =
-      nir_fsub(b,
-               abs_src,
-               nir_fmul(b,
-                        two,
-                        nir_ffloor(b,
-                                   nir_fmul(b,
-                                            abs_src,
-                                            nir_imm_double(b, 0.5)))));
-
-   /*
-    * If fract(src) != 0.5, then we round as floor(src + 0.5)
-    *
-    * If fract(src) == 0.5, then we have to check the modulo:
-    *
-    *   if it is < 1 we need a trunc operation so we get:
-    *      0.5 -> 0,   -0.5 -> -0
-    *      2.5 -> 2,   -2.5 -> -2
-    *
-    *   otherwise we need to check if src >= 0, in which case we need to round
-    *   upwards, or not, in which case we need to round downwards so we get:
-    *      1.5 -> 2,   -1.5 -> -2
-    *      3.5 -> 4,   -3.5 -> -4
-    */
-   nir_ssa_def *fract = nir_ffract(b, src);
-   return nir_bcsel(b,
-                    nir_fne(b, fract, nir_imm_double(b, 0.5)),
-                    nir_ffloor(b, nir_fadd(b, src, nir_imm_double(b, 0.5))),
-                    nir_bcsel(b,
-                              nir_flt(b, mod, nir_imm_double(b, 1.0)),
-                              nir_ftrunc(b, src),
-                              nir_bcsel(b,
-                                        nir_fge(b, src, nir_imm_double(b, 0.0)),
-                                        nir_fadd(b, src, nir_imm_double(b, 0.5)),
-                                        nir_fsub(b, src, nir_imm_double(b, 0.5)))));
+   /* Add and subtract 2**52 to round off any fractional bits. */
+   nir_ssa_def *two52 = nir_imm_double(b, (double)(1ull << 52));
+   nir_ssa_def *sign = nir_iand(b, nir_unpack_64_2x32_split_y(b, src),
+                                nir_imm_int(b, 1ull << 31));
+
+   b->exact = true;
+   nir_ssa_def *res = nir_fsub(b, nir_fadd(b, nir_fabs(b, src), two52), two52);
+   b->exact = false;
+
+   return nir_bcsel(b, nir_flt(b, nir_fabs(b, src), two52),
+                    nir_pack_64_2x32_split(b, nir_unpack_64_2x32_split_x(b, res),
+                                           nir_ior(b, nir_unpack_64_2x32_split_y(b, res), sign)), src);
 }
 
 static nir_ssa_def *
@@ -445,155 +428,339 @@ lower_mod(nir_builder *b, nir_ssa_def *src0, nir_ssa_def *src1)
     *
     * If the division is lowered, it could add some rounding errors that make
     * floor() to return the quotient minus one when x = N * y. If this is the
-    * case, we return zero because mod(x, y) output value is [0, y).
+    * case, we should return zero because mod(x, y) output value is [0, y).
+    * But fortunately Vulkan spec allows this kind of errors; from Vulkan
+    * spec, appendix A (Precision and Operation of SPIR-V instructions:
+    *
+    *   "The OpFRem and OpFMod instructions use cheap approximations of
+    *   remainder, and the error can be large due to the discontinuity in
+    *   trunc() and floor(). This can produce mathematically unexpected
+    *   results in some cases, such as FMod(x,x) computing x rather than 0,
+    *   and can also cause the result to have a different sign than the
+    *   infinitely precise result."
+    *
+    * In practice this means the output value is actually in the interval
+    * [0, y].
+    *
+    * While Vulkan states this behaviour explicitly, OpenGL does not, and thus
+    * we need to assume that value should be in range [0, y); but on the other
+    * hand, mod(a,b) is defined as "a - b * floor(a/b)" and OpenGL allows for
+    * some error in division, so a/a could actually end up being 1.0 - 1ULP;
+    * so in this case floor(a/a) would end up as 0, and hence mod(a,a) == a.
+    *
+    * In summary, in the practice mod(a,a) can be "a" both for OpenGL and
+    * Vulkan.
     */
    nir_ssa_def *floor = nir_ffloor(b, nir_fdiv(b, src0, src1));
-   nir_ssa_def *mod = nir_fsub(b, src0, nir_fmul(b, src1, floor));
 
-   return nir_bcsel(b,
-                    nir_fne(b, mod, src1),
-                    mod,
-                    nir_imm_double(b, 0.0));
+   return nir_fsub(b, src0, nir_fmul(b, src1, floor));
 }
 
-static bool
-lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
-                    nir_lower_doubles_options options)
+static nir_ssa_def *
+lower_doubles_instr_to_soft(nir_builder *b, nir_alu_instr *instr,
+                            const nir_shader *softfp64,
+                            nir_lower_doubles_options options)
 {
+   if (!(options & nir_lower_fp64_full_software))
+      return NULL;
+
    assert(instr->dest.dest.is_ssa);
-   if (instr->dest.dest.ssa.bit_size != 64)
-      return false;
+
+   const char *name;
+   const struct glsl_type *return_type = glsl_uint64_t_type();
 
    switch (instr->op) {
-   case nir_op_frcp:
-      if (!(options & nir_lower_drcp))
+   case nir_op_f2i64:
+      if (instr->src[0].src.ssa->bit_size != 64)
          return false;
+      name = "__fp64_to_int64";
+      return_type = glsl_int64_t_type();
       break;
-
-   case nir_op_fsqrt:
-      if (!(options & nir_lower_dsqrt))
+   case nir_op_f2u64:
+      if (instr->src[0].src.ssa->bit_size != 64)
          return false;
+      name = "__fp64_to_uint64";
       break;
-
-   case nir_op_frsq:
-      if (!(options & nir_lower_drsq))
-         return false;
+   case nir_op_f2f64:
+      name = "__fp32_to_fp64";
+      break;
+   case nir_op_f2f32:
+      name = "__fp64_to_fp32";
+      return_type = glsl_float_type();
+      break;
+   case nir_op_f2i32:
+      name = "__fp64_to_int";
+      return_type = glsl_int_type();
+      break;
+   case nir_op_f2u32:
+      name = "__fp64_to_uint";
+      return_type = glsl_uint_type();
+      break;
+   case nir_op_f2b1:
+   case nir_op_f2b32:
+      name = "__fp64_to_bool";
+      return_type = glsl_bool_type();
+      break;
+   case nir_op_b2f64:
+      name = "__bool_to_fp64";
+      break;
+   case nir_op_i2f64:
+      if (instr->src[0].src.ssa->bit_size == 64)
+         name = "__int64_to_fp64";
+      else
+         name = "__int_to_fp64";
+      break;
+   case nir_op_u2f64:
+      if (instr->src[0].src.ssa->bit_size == 64)
+         name = "__uint64_to_fp64";
+      else
+         name = "__uint_to_fp64";
+      break;
+   case nir_op_fabs:
+      name = "__fabs64";
+      break;
+   case nir_op_fneg:
+      name = "__fneg64";
+      break;
+   case nir_op_fround_even:
+      name = "__fround64";
       break;
-
    case nir_op_ftrunc:
-      if (!(options & nir_lower_dtrunc))
-         return false;
+      name = "__ftrunc64";
       break;
-
    case nir_op_ffloor:
-      if (!(options & nir_lower_dfloor))
-         return false;
+      name = "__ffloor64";
       break;
-
-   case nir_op_fceil:
-      if (!(options & nir_lower_dceil))
-         return false;
-      break;
-
    case nir_op_ffract:
-      if (!(options & nir_lower_dfract))
-         return false;
+      name = "__ffract64";
       break;
-
-   case nir_op_fround_even:
-      if (!(options & nir_lower_dround_even))
-         return false;
+   case nir_op_fsign:
+      name = "__fsign64";
       break;
-
-   case nir_op_fmod:
-      if (!(options & nir_lower_dmod))
-         return false;
+   case nir_op_feq:
+      name = "__feq64";
+      return_type = glsl_bool_type();
+      break;
+   case nir_op_fne:
+      name = "__fne64";
+      return_type = glsl_bool_type();
+      break;
+   case nir_op_flt:
+      name = "__flt64";
+      return_type = glsl_bool_type();
+      break;
+   case nir_op_fge:
+      name = "__fge64";
+      return_type = glsl_bool_type();
+      break;
+   case nir_op_fmin:
+      name = "__fmin64";
+      break;
+   case nir_op_fmax:
+      name = "__fmax64";
+      break;
+   case nir_op_fadd:
+      name = "__fadd64";
+      break;
+   case nir_op_fmul:
+      name = "__fmul64";
+      break;
+   case nir_op_ffma:
+      name = "__ffma64";
+      break;
+   case nir_op_fsat:
+      name = "__fsat64";
       break;
-
    default:
       return false;
    }
 
-   b->cursor = nir_before_instr(&instr->instr);
+   nir_function *func = NULL;
+   nir_foreach_function(function, softfp64) {
+      if (strcmp(function->name, name) == 0) {
+         func = function;
+         break;
+      }
+   }
+   if (!func || !func->impl) {
+      fprintf(stderr, "Cannot find function \"%s\"\n", name);
+      assert(func);
+   }
 
-   nir_ssa_def *src = nir_fmov_alu(b, instr->src[0],
-                                   instr->dest.dest.ssa.num_components);
+   nir_ssa_def *params[4] = { NULL, };
 
-   nir_ssa_def *result;
+   nir_variable *ret_tmp =
+      nir_local_variable_create(b->impl, return_type, "return_tmp");
+   nir_deref_instr *ret_deref = nir_build_deref_var(b, ret_tmp);
+   params[0] = &ret_deref->dest.ssa;
 
-   switch (instr->op) {
+   assert(nir_op_infos[instr->op].num_inputs + 1 == func->num_params);
+   for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
+      assert(i + 1 < ARRAY_SIZE(params));
+      params[i + 1] = nir_mov_alu(b, instr->src[i], 1);
+   }
+
+   nir_inline_function_impl(b, func->impl, params);
+
+   return nir_load_deref(b, ret_deref);
+}
+
+nir_lower_doubles_options
+nir_lower_doubles_op_to_options_mask(nir_op opcode)
+{
+   switch (opcode) {
+   case nir_op_frcp:          return nir_lower_drcp;
+   case nir_op_fsqrt:         return nir_lower_dsqrt;
+   case nir_op_frsq:          return nir_lower_drsq;
+   case nir_op_ftrunc:        return nir_lower_dtrunc;
+   case nir_op_ffloor:        return nir_lower_dfloor;
+   case nir_op_fceil:         return nir_lower_dceil;
+   case nir_op_ffract:        return nir_lower_dfract;
+   case nir_op_fround_even:   return nir_lower_dround_even;
+   case nir_op_fmod:          return nir_lower_dmod;
+   case nir_op_fsub:          return nir_lower_dsub;
+   case nir_op_fdiv:          return nir_lower_ddiv;
+   default:                   return 0;
+   }
+}
+
+struct lower_doubles_data {
+   const nir_shader *softfp64;
+   nir_lower_doubles_options options;
+};
+
+static bool
+should_lower_double_instr(const nir_instr *instr, const void *_data)
+{
+   const struct lower_doubles_data *data = _data;
+   const nir_lower_doubles_options options = data->options;
+
+   if (instr->type != nir_instr_type_alu)
+      return false;
+
+   const nir_alu_instr *alu = nir_instr_as_alu(instr);
+
+   assert(alu->dest.dest.is_ssa);
+   bool is_64 = alu->dest.dest.ssa.bit_size == 64;
+
+   unsigned num_srcs = nir_op_infos[alu->op].num_inputs;
+   for (unsigned i = 0; i < num_srcs; i++) {
+      is_64 |= (nir_src_bit_size(alu->src[i].src) == 64);
+   }
+
+   if (!is_64)
+      return false;
+
+   if (options & nir_lower_fp64_full_software)
+      return true;
+
+   return options & nir_lower_doubles_op_to_options_mask(alu->op);
+}
+
+static nir_ssa_def *
+lower_doubles_instr(nir_builder *b, nir_instr *instr, void *_data)
+{
+   const struct lower_doubles_data *data = _data;
+   const nir_lower_doubles_options options = data->options;
+   nir_alu_instr *alu = nir_instr_as_alu(instr);
+
+   nir_ssa_def *soft_def =
+      lower_doubles_instr_to_soft(b, alu, data->softfp64, options);
+   if (soft_def)
+      return soft_def;
+
+   if (!(options & nir_lower_doubles_op_to_options_mask(alu->op)))
+      return NULL;
+
+   nir_ssa_def *src = nir_mov_alu(b, alu->src[0],
+                                  alu->dest.dest.ssa.num_components);
+
+   switch (alu->op) {
    case nir_op_frcp:
-      result = lower_rcp(b, src);
-      break;
+      return lower_rcp(b, src);
    case nir_op_fsqrt:
-      result = lower_sqrt_rsq(b, src, true);
-      break;
+      return lower_sqrt_rsq(b, src, true);
    case nir_op_frsq:
-      result = lower_sqrt_rsq(b, src, false);
-      break;
+      return lower_sqrt_rsq(b, src, false);
    case nir_op_ftrunc:
-      result = lower_trunc(b, src);
-      break;
+      return lower_trunc(b, src);
    case nir_op_ffloor:
-      result = lower_floor(b, src);
-      break;
+      return lower_floor(b, src);
    case nir_op_fceil:
-      result = lower_ceil(b, src);
-      break;
+      return lower_ceil(b, src);
    case nir_op_ffract:
-      result = lower_fract(b, src);
-      break;
+      return lower_fract(b, src);
    case nir_op_fround_even:
-      result = lower_round_even(b, src);
-      break;
+      return lower_round_even(b, src);
 
+   case nir_op_fdiv:
+   case nir_op_fsub:
    case nir_op_fmod: {
-      nir_ssa_def *src1 = nir_fmov_alu(b, instr->src[1],
-                                       instr->dest.dest.ssa.num_components);
-      result = lower_mod(b, src, src1);
+      nir_ssa_def *src1 = nir_mov_alu(b, alu->src[1],
+                                      alu->dest.dest.ssa.num_components);
+      switch (alu->op) {
+      case nir_op_fdiv:
+         return nir_fmul(b, src, nir_frcp(b, src1));
+      case nir_op_fsub:
+         return nir_fadd(b, src, nir_fneg(b, src1));
+      case nir_op_fmod:
+         return lower_mod(b, src, src1);
+      default:
+         unreachable("unhandled opcode");
+      }
    }
-      break;
    default:
       unreachable("unhandled opcode");
    }
-
-   nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(result));
-   nir_instr_remove(&instr->instr);
-   return true;
 }
 
 static bool
 nir_lower_doubles_impl(nir_function_impl *impl,
+                       const nir_shader *softfp64,
                        nir_lower_doubles_options options)
 {
-   bool progress = false;
-
-   nir_builder b;
-   nir_builder_init(&b, impl);
-
-   nir_foreach_block(block, impl) {
-      nir_foreach_instr_safe(instr, block) {
-         if (instr->type == nir_instr_type_alu)
-            progress |= lower_doubles_instr(&b, nir_instr_as_alu(instr),
-                                            options);
-      }
-   }
-
-   if (progress)
+   struct lower_doubles_data data = {
+      .softfp64 = softfp64,
+      .options = options,
+   };
+
+   bool progress =
+      nir_function_impl_lower_instructions(impl,
+                                           should_lower_double_instr,
+                                           lower_doubles_instr,
+                                           &data);
+
+   if (progress && (options & nir_lower_fp64_full_software)) {
+      /* SSA and register indices are completely messed up now */
+      nir_index_ssa_defs(impl);
+      nir_index_local_regs(impl);
+
+      nir_metadata_preserve(impl, nir_metadata_none);
+
+      /* And we have deref casts we need to clean up thanks to function
+       * inlining.
+       */
+      nir_opt_deref_impl(impl);
+   } else if (progress) {
       nir_metadata_preserve(impl, nir_metadata_block_index |
                                   nir_metadata_dominance);
+   } else {
+      nir_metadata_preserve(impl, nir_metadata_all);
+   }
 
    return progress;
 }
 
 bool
-nir_lower_doubles(nir_shader *shader, nir_lower_doubles_options options)
+nir_lower_doubles(nir_shader *shader,
+                  const nir_shader *softfp64,
+                  nir_lower_doubles_options options)
 {
    bool progress = false;
 
    nir_foreach_function(function, shader) {
       if (function->impl) {
-         progress |= nir_lower_doubles_impl(function->impl, options);
+         progress |= nir_lower_doubles_impl(function->impl, softfp64, options);
       }
    }