nir/lower_doubles: Inline functions directly in lower_doubles

Instead of trusting the caller to already have created a softfp64
function shader and added all its functions to our shader, we simply
take the softfp64 shader as an argument and do the function inlining
ouselves.  This means that there's no more nasty functions lying around
that the caller needs to worry about cleaning up.

Reviewed-by: Matt Turner <mattst88@gmail.com>
Reviewed-by: Jordan Justen <jordan.l.justen@intel.com>
Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>

diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h
index 5d6b15bc42f3fd59b28c056a1f5fd638fcbade42..7be62ba97ae8702cf6c9649f44da3362a02a55aa 100644 (file)
--- a/src/compiler/nir/nir.h
+++ b/src/compiler/nir/nir.h
@@ -3279,7 +3279,8 @@ nir_lower_int64_options nir_lower_int64_op_to_options_mask(nir_op opcode);
 bool nir_lower_int64(nir_shader *shader, nir_lower_int64_options options);
 
 nir_lower_doubles_options nir_lower_doubles_op_to_options_mask(nir_op opcode);
-bool nir_lower_doubles(nir_shader *shader, nir_lower_doubles_options options);
+bool nir_lower_doubles(nir_shader *shader, const nir_shader *softfp64,
+                       nir_lower_doubles_options options);
 bool nir_lower_pack(nir_shader *shader);
 
 bool nir_normalize_cubemap_coords(nir_shader *shader);

diff --git a/src/compiler/nir/nir_lower_double_ops.c b/src/compiler/nir/nir_lower_double_ops.c
index 69f4b3a78db3db68bbe5c9ac1dc30610e71b4b75..863046e65c7e9b5ef9800437b494b35e696e4eb1 100644 (file)
--- a/src/compiler/nir/nir_lower_double_ops.c
+++ b/src/compiler/nir/nir_lower_double_ops.c
@@ -426,6 +426,7 @@ lower_mod(nir_builder *b, nir_ssa_def *src0, nir_ssa_def *src1)
 
 static bool
 lower_doubles_instr_to_soft(nir_builder *b, nir_alu_instr *instr,
+                            const nir_shader *softfp64,
                             nir_lower_doubles_options options)
 {
    if (!(options & nir_lower_fp64_full_software))
@@ -553,35 +554,34 @@ lower_doubles_instr_to_soft(nir_builder *b, nir_alu_instr *instr,
       return false;
    }
 
-   nir_shader *shader = b->shader;
    nir_function *func = NULL;
-
-   nir_foreach_function(function, shader) {
+   nir_foreach_function(function, softfp64) {
       if (strcmp(function->name, name) == 0) {
          func = function;
          break;
       }
    }
-   if (!func) {
+   if (!func || !func->impl) {
       fprintf(stderr, "Cannot find function \"%s\"\n", name);
       assert(func);
    }
 
    b->cursor = nir_before_instr(&instr->instr);
 
-   nir_call_instr *call = nir_call_instr_create(shader, func);
+   nir_ssa_def *params[4] = { NULL, };
 
    nir_variable *ret_tmp =
       nir_local_variable_create(b->impl, return_type, "return_tmp");
    nir_deref_instr *ret_deref = nir_build_deref_var(b, ret_tmp);
-   call->params[0] = nir_src_for_ssa(&ret_deref->dest.ssa);
+   params[0] = &ret_deref->dest.ssa;
 
+   assert(nir_op_infos[instr->op].num_inputs + 1 == func->num_params);
    for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
-      nir_src arg = nir_src_for_ssa(nir_imov_alu(b, instr->src[i], 1));
-      nir_src_copy(&call->params[i + 1], &arg, call);
+      assert(i + 1 < ARRAY_SIZE(params));
+      params[i + 1] = nir_imov_alu(b, instr->src[i], 1);
    }
 
-   nir_builder_instr_insert(b, &call->instr);
+   nir_inline_function_impl(b, func->impl, params);
 
    nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
                             nir_src_for_ssa(nir_load_deref(b, ret_deref)));
@@ -608,6 +608,7 @@ nir_lower_doubles_op_to_options_mask(nir_op opcode)
 
 static bool
 lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
+                    const nir_shader *softfp64,
                     nir_lower_doubles_options options)
 {
    assert(instr->dest.dest.is_ssa);
@@ -621,7 +622,7 @@ lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
    if (!is_64)
       return false;
 
-   if (lower_doubles_instr_to_soft(b, instr, options))
+   if (lower_doubles_instr_to_soft(b, instr, softfp64, options))
       return true;
 
    if (!(options & nir_lower_doubles_op_to_options_mask(instr->op)))
@@ -677,6 +678,7 @@ lower_doubles_instr(nir_builder *b, nir_alu_instr *instr,
 
 static bool
 nir_lower_doubles_impl(nir_function_impl *impl,
+                       const nir_shader *softfp64,
                        nir_lower_doubles_options options)
 {
    bool progress = false;
@@ -684,17 +686,30 @@ nir_lower_doubles_impl(nir_function_impl *impl,
    nir_builder b;
    nir_builder_init(&b, impl);
 
-   nir_foreach_block(block, impl) {
+   nir_foreach_block_safe(block, impl) {
       nir_foreach_instr_safe(instr, block) {
          if (instr->type == nir_instr_type_alu)
             progress |= lower_doubles_instr(&b, nir_instr_as_alu(instr),
-                                            options);
+                                            softfp64, options);
       }
    }
 
    if (progress) {
-      nir_metadata_preserve(impl, nir_metadata_block_index |
-                                  nir_metadata_dominance);
+      if (options & nir_lower_fp64_full_software) {
+         /* SSA and register indices are completely messed up now */
+         nir_index_ssa_defs(impl);
+         nir_index_local_regs(impl);
+
+         nir_metadata_preserve(impl, nir_metadata_none);
+
+         /* And we have deref casts we need to clean up thanks to function
+          * inlining.
+          */
+         nir_opt_deref_impl(impl);
+      } else {
+         nir_metadata_preserve(impl, nir_metadata_block_index |
+                                     nir_metadata_dominance);
+      }
     } else {
 #ifndef NDEBUG
       impl->valid_metadata &= ~nir_metadata_not_properly_reset;
@@ -705,13 +720,15 @@ nir_lower_doubles_impl(nir_function_impl *impl,
 }
 
 bool
-nir_lower_doubles(nir_shader *shader, nir_lower_doubles_options options)
+nir_lower_doubles(nir_shader *shader,
+                  const nir_shader *softfp64,
+                  nir_lower_doubles_options options)
 {
    bool progress = false;
 
    nir_foreach_function(function, shader) {
       if (function->impl) {
-         progress |= nir_lower_doubles_impl(function->impl, options);
+         progress |= nir_lower_doubles_impl(function->impl, softfp64, options);
       }
    }
 

diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c
index 28864c4f0163bd416ac084c3795e8f9bb9aff90e..ad3eab6d8ce4d81d639dc469afb87a2ab07615e5 100644 (file)
--- a/src/gallium/drivers/iris/iris_program.c
+++ b/src/gallium/drivers/iris/iris_program.c
@@ -1384,7 +1384,7 @@ iris_create_uncompiled_shader(struct pipe_context *ctx,
    if (!ish)
       return NULL;
 
-   nir = brw_preprocess_nir(screen->compiler, nir);
+   nir = brw_preprocess_nir(screen->compiler, nir, NULL);
 
    NIR_PASS_V(nir, brw_nir_lower_image_load_store, devinfo);
    NIR_PASS_V(nir, iris_lower_storage_image_derefs);

diff --git a/src/intel/blorp/blorp.c b/src/intel/blorp/blorp.c
index ecd4289db173005e193ed1c3b6f0f5bb072f104b..cb5d0f73e772e7fe76d0bd5f17193c41c4905e96 100644 (file)
--- a/src/intel/blorp/blorp.c
+++ b/src/intel/blorp/blorp.c
@@ -192,7 +192,7 @@ blorp_compile_fs(struct blorp_context *blorp, void *mem_ctx,
     */
    wm_prog_data->base.binding_table.texture_start = BLORP_TEXTURE_BT_INDEX;
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
    nir_remove_dead_variables(nir, nir_var_shader_in);
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 
@@ -221,7 +221,7 @@ blorp_compile_vs(struct blorp_context *blorp, void *mem_ctx,
    nir->options =
       compiler->glsl_compiler_options[MESA_SHADER_VERTEX].NirOptions;
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 
    vs_prog_data->inputs_read = nir->info.inputs_read;

diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c
index 58b89a1bd3ce3bae1426138580da74db47a7c724..34aaa29a5cb432e5d31786c19a6a1339a285758b 100644 (file)
--- a/src/intel/compiler/brw_nir.c
+++ b/src/intel/compiler/brw_nir.c
@@ -653,7 +653,8 @@ lower_bit_size_callback(const nir_alu_instr *alu, UNUSED void *data)
  * is_scalar = true to scalarize everything prior to code gen.
  */
 nir_shader *
-brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
+brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir,
+                   const nir_shader *softfp64)
 {
    const struct gen_device_info *devinfo = compiler->devinfo;
    UNUSED bool progress; /* Written by OPT */
@@ -677,7 +678,7 @@ brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
       progress = false;
 
       OPT(nir_lower_int64, nir->options->lower_int64_options);
-      OPT(nir_lower_doubles, nir->options->lower_doubles_options);
+      OPT(nir_lower_doubles, softfp64, nir->options->lower_doubles_options);
 
       /* Necessary to lower add -> sub and div -> mul/rcp */
       OPT(nir_opt_algebraic);
@@ -685,21 +686,6 @@ brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
       lowered_64bit_ops |= progress;
    } while (progress);
 
-   if (lowered_64bit_ops) {
-      OPT(nir_lower_constant_initializers, nir_var_function_temp);
-      OPT(nir_lower_returns);
-      OPT(nir_inline_functions);
-      OPT(nir_opt_deref);
-   }
-
-   const nir_function *entry_point = nir_shader_get_entrypoint(nir)->function;
-   foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
-      if (func != entry_point) {
-         exec_node_remove(&func->node);
-      }
-   }
-   assert(exec_list_length(&nir->functions) == 1);
-
    if (nir->info.stage == MESA_SHADER_GEOMETRY)
       OPT(nir_lower_gs_intrinsics);
 
@@ -1098,7 +1084,7 @@ brw_nir_create_passthrough_tcs(void *mem_ctx, const struct brw_compiler *compile
 
    nir_validate_shader(nir, "in brw_nir_create_passthrough_tcs");
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
 
    return nir;
 }

diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h
index bc81950d47e30df35fe71beb5213c5f95b48761b..7c73079c35d0efd6e53d9ec9a59026692a36c97b 100644 (file)
--- a/src/intel/compiler/brw_nir.h
+++ b/src/intel/compiler/brw_nir.h
@@ -93,7 +93,8 @@ enum {
 void brw_nir_analyze_boolean_resolves(nir_shader *nir);
 
 nir_shader *brw_preprocess_nir(const struct brw_compiler *compiler,
-                               nir_shader *nir);
+                               nir_shader *nir,
+                               const nir_shader *softfp64);
 
 void
 brw_nir_link_shaders(const struct brw_compiler *compiler,

diff --git a/src/intel/vulkan/anv_pipeline.c b/src/intel/vulkan/anv_pipeline.c
index a460a1ff428d493f0c1a373b9fbf8cc8fe46815c..dbff784fb105a3a8d6da5cec6684167c63bb53ba 100644 (file)
--- a/src/intel/vulkan/anv_pipeline.c
+++ b/src/intel/vulkan/anv_pipeline.c
@@ -231,7 +231,7 @@ anv_shader_compile_to_nir(struct anv_device *device,
    /* Vulkan uses the separate-shader linking model */
    nir->info.separate_shader = true;
 
-   nir = brw_preprocess_nir(compiler, nir);
+   nir = brw_preprocess_nir(compiler, nir, NULL);
 
    return nir;
 }

diff --git a/src/mesa/drivers/dri/i965/brw_program.c b/src/mesa/drivers/dri/i965/brw_program.c
index f40d2c3354980f4889bf2015ee4ffd0b96547a48..ffb49c35cd4ae1909ada437ff03d8bc39efc0620 100644 (file)
--- a/src/mesa/drivers/dri/i965/brw_program.c
+++ b/src/mesa/drivers/dri/i965/brw_program.c
@@ -112,14 +112,14 @@ brw_create_nir(struct brw_context *brw,
 
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 
+   nir_shader *softfp64 = NULL;
    if ((options->lower_doubles_options & nir_lower_fp64_full_software) &&
        nir->info.uses_64bit) {
-      nir_shader *fp64 = glsl_float64_funcs_to_nir(ctx, options);
-      ralloc_steal(ralloc_parent(nir), fp64);
-      exec_list_append(&nir->functions, &fp64->functions);
+      softfp64 = glsl_float64_funcs_to_nir(ctx, options);
+      ralloc_steal(ralloc_parent(nir), softfp64);
    }
 
-   nir = brw_preprocess_nir(brw->screen->compiler, nir);
+   nir = brw_preprocess_nir(brw->screen->compiler, nir, softfp64);
 
    NIR_PASS_V(nir, brw_nir_lower_image_load_store, devinfo);
 

diff --git a/src/mesa/state_tracker/st_glsl_to_nir.cpp b/src/mesa/state_tracker/st_glsl_to_nir.cpp
index d62a89ec5b1a4befe4a6512c29dfe2cb709e33d4..3d01b91f4258c0ad50f6c4fad9d224bed174fafd 100644 (file)
--- a/src/mesa/state_tracker/st_glsl_to_nir.cpp
+++ b/src/mesa/state_tracker/st_glsl_to_nir.cpp
@@ -379,11 +379,11 @@ st_glsl_to_nir(struct st_context *st, struct gl_program *prog,
    }
 
    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
+   nir_shader *softfp64 = NULL;
    if (nir->info.uses_64bit &&
        (options->lower_doubles_options & nir_lower_fp64_full_software) != 0) {
-      nir_shader *fp64 = glsl_float64_funcs_to_nir(st->ctx, options);
-      ralloc_steal(ralloc_parent(nir), fp64);
-      exec_list_append(&nir->functions, &fp64->functions);
+      softfp64 = glsl_float64_funcs_to_nir(st->ctx, options);
+      ralloc_steal(ralloc_parent(nir), softfp64);
    }
 
    nir_variable_mode mask =
@@ -424,26 +424,11 @@ st_glsl_to_nir(struct st_context *st, struct gl_program *prog,
          }
          if (options->lower_doubles_options) {
             NIR_PASS(progress, nir, nir_lower_doubles,
-                     options->lower_doubles_options);
+                     softfp64, options->lower_doubles_options);
          }
          NIR_PASS(progress, nir, nir_opt_algebraic);
          lowered_64bit_ops |= progress;
       } while (progress);
-
-      if (lowered_64bit_ops) {
-         NIR_PASS_V(nir, nir_lower_constant_initializers, nir_var_function_temp);
-         NIR_PASS_V(nir, nir_lower_returns);
-         NIR_PASS_V(nir, nir_inline_functions);
-         NIR_PASS_V(nir, nir_opt_deref);
-      }
-
-      const nir_function *entry_point = nir_shader_get_entrypoint(nir)->function;
-      foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
-         if (func != entry_point) {
-            exec_node_remove(&func->node);
-         }
-      }
-      assert(exec_list_length(&nir->functions) == 1);
    }
 
    st_nir_opts(nir, is_scalar);