From 2e1df6a17ff82c4a456caa8be4bfae1fac009b6a Mon Sep 17 00:00:00 2001 From: Jesse Natalie Date: Fri, 21 Aug 2020 10:18:14 -0700 Subject: [PATCH] nir: Move compute system value lowering to a separate pass The actual variable -> intrinsic lowering stays where it is, but ops which convert one intrinsic to be implemented in terms of another have moved. Reviewed-by: Karol Herbst Reviewed-by: Jason Ekstrand Part-of: --- src/amd/vulkan/radv_shader.c | 2 + src/broadcom/compiler/vir.c | 1 + src/compiler/nir/nir.h | 2 + src/compiler/nir/nir_lower_system_values.c | 263 ++++++++++-------- src/freedreno/vulkan/tu_shader.c | 2 + src/gallium/auxiliary/nir/tgsi_to_nir.c | 1 + .../drivers/freedreno/ir3/ir3_cmdline.c | 3 + .../frontends/clover/nir/invocation.cpp | 2 + src/gallium/frontends/vallium/val_pipeline.c | 1 + src/intel/compiler/brw_nir.c | 1 + src/mesa/state_tracker/st_glsl_to_nir.cpp | 2 + src/mesa/state_tracker/st_nir_builtins.c | 1 + src/mesa/state_tracker/st_program.c | 1 + 13 files changed, 172 insertions(+), 110 deletions(-) diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c index 03cfe993beb..01a22b7a757 100644 --- a/src/amd/vulkan/radv_shader.c +++ b/src/amd/vulkan/radv_shader.c @@ -540,6 +540,8 @@ radv_shader_compile_to_nir(struct radv_device *device, NIR_PASS_V(nir, nir_propagate_invariant); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); + NIR_PASS_V(nir, nir_lower_clip_cull_distance_arrays); if (device->instance->debug_flags & RADV_DEBUG_DISCARD_TO_DEMOTE) diff --git a/src/broadcom/compiler/vir.c b/src/broadcom/compiler/vir.c index d06f789161f..83eb4d14b38 100644 --- a/src/broadcom/compiler/vir.c +++ b/src/broadcom/compiler/vir.c @@ -586,6 +586,7 @@ v3d_lower_nir(struct v3d_compile *c) NIR_PASS_V(c->s, nir_lower_tex, &tex_options); NIR_PASS_V(c->s, nir_lower_system_values); + NIR_PASS_V(c->s, nir_lower_compute_system_values); NIR_PASS_V(c->s, nir_lower_vars_to_scratch, nir_var_function_temp, diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index bb43ab18a9d..7432afd8d94 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -4276,6 +4276,8 @@ bool nir_lower_subgroups(nir_shader *shader, bool nir_lower_system_values(nir_shader *shader); +bool nir_lower_compute_system_values(nir_shader *shader); + enum PACKED nir_lower_tex_packing { nir_lower_tex_packing_none = 0, /* The sampler returns up to 2 32-bit words of half floats or 16-bit signed diff --git a/src/compiler/nir/nir_lower_system_values.c b/src/compiler/nir/nir_lower_system_values.c index bc6a3931767..bc80d184f72 100644 --- a/src/compiler/nir/nir_lower_system_values.c +++ b/src/compiler/nir/nir_lower_system_values.c @@ -92,6 +92,150 @@ lower_system_value_instr(nir_builder *b, nir_instr *instr, void *_state) return NULL; } + case nir_intrinsic_load_helper_invocation: + if (b->shader->options->lower_helper_invocation) { + nir_ssa_def *tmp; + tmp = nir_ishl(b, nir_imm_int(b, 1), + nir_load_sample_id_no_per_sample(b)); + tmp = nir_iand(b, nir_load_sample_mask_in(b), tmp); + return nir_inot(b, nir_i2b(b, tmp)); + } else { + return NULL; + } + + case nir_intrinsic_load_local_invocation_id: + case nir_intrinsic_load_local_invocation_index: + case nir_intrinsic_load_local_group_size: + return sanitize_32bit_sysval(b, intrin); + + case nir_intrinsic_load_deref: { + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + if (deref->mode != nir_var_system_value) + return NULL; + + if (deref->deref_type != nir_deref_type_var) { + /* The only one system value that is an array and that is + * gl_SampleMask which is always an array of one element. + */ + assert(deref->deref_type == nir_deref_type_array); + deref = nir_deref_instr_parent(deref); + assert(deref->deref_type == nir_deref_type_var); + assert(deref->var->data.location == SYSTEM_VALUE_SAMPLE_MASK_IN); + } + nir_variable *var = deref->var; + + switch (var->data.location) { + case SYSTEM_VALUE_INSTANCE_INDEX: + return nir_iadd(b, nir_load_instance_id(b), + nir_load_base_instance(b)); + + case SYSTEM_VALUE_SUBGROUP_EQ_MASK: + case SYSTEM_VALUE_SUBGROUP_GE_MASK: + case SYSTEM_VALUE_SUBGROUP_GT_MASK: + case SYSTEM_VALUE_SUBGROUP_LE_MASK: + case SYSTEM_VALUE_SUBGROUP_LT_MASK: { + nir_intrinsic_op op = + nir_intrinsic_from_system_value(var->data.location); + nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op); + nir_ssa_dest_init_for_type(&load->instr, &load->dest, + var->type, NULL); + load->num_components = load->dest.ssa.num_components; + nir_builder_instr_insert(b, &load->instr); + return &load->dest.ssa; + } + + case SYSTEM_VALUE_DEVICE_INDEX: + if (b->shader->options->lower_device_index_to_zero) + return nir_imm_int(b, 0); + break; + + case SYSTEM_VALUE_GLOBAL_GROUP_SIZE: + return build_global_group_size(b, bit_size); + + case SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel, + INTERP_MODE_NOPERSPECTIVE); + + case SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid, + INTERP_MODE_NOPERSPECTIVE); + + case SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample, + INTERP_MODE_NOPERSPECTIVE); + + case SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel, + INTERP_MODE_SMOOTH); + + case SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid, + INTERP_MODE_SMOOTH); + + case SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample, + INTERP_MODE_SMOOTH); + + case SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL: + return nir_load_barycentric(b, nir_intrinsic_load_barycentric_model, + INTERP_MODE_NONE); + + default: + break; + } + + nir_intrinsic_op sysval_op = + nir_intrinsic_from_system_value(var->data.location); + return nir_load_system_value(b, sysval_op, 0, + intrin->dest.ssa.num_components, + intrin->dest.ssa.bit_size); + } + + default: + return NULL; + } +} + +bool +nir_lower_system_values(nir_shader *shader) +{ + bool progress = nir_shader_lower_instructions(shader, + lower_system_value_filter, + lower_system_value_instr, + NULL); + + /* We're going to delete the variables so we need to clean up all those + * derefs we left lying around. + */ + if (progress) + nir_remove_dead_derefs(shader); + + nir_foreach_variable_with_modes_safe(var, shader, nir_var_system_value) + exec_node_remove(&var->node); + + return progress; +} + +static bool +lower_compute_system_value_filter(const nir_instr *instr, const void *_options) +{ + return instr->type == nir_instr_type_intrinsic; +} + +static nir_ssa_def * +lower_compute_system_value_instr(nir_builder *b, + nir_instr *instr, void *_options) +{ + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + /* All the intrinsics we care about are loads */ + if (!nir_intrinsic_infos[intrin->intrinsic].has_dest) + return NULL; + + assert(intrin->dest.is_ssa); + const unsigned bit_size = intrin->dest.ssa.bit_size; + + switch (intrin->intrinsic) { case nir_intrinsic_load_local_invocation_id: /* If lower_cs_local_id_from_index is true, then we derive the local * index from the local id. @@ -215,121 +359,20 @@ lower_system_value_instr(nir_builder *b, nir_instr *instr, void *_state) return index; } - case nir_intrinsic_load_helper_invocation: - if (b->shader->options->lower_helper_invocation) { - nir_ssa_def *tmp; - tmp = nir_ishl(b, nir_imm_int(b, 1), - nir_load_sample_id_no_per_sample(b)); - tmp = nir_iand(b, nir_load_sample_mask_in(b), tmp); - return nir_inot(b, nir_i2b(b, tmp)); - } else { - return NULL; - } - - case nir_intrinsic_load_deref: { - nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); - if (deref->mode != nir_var_system_value) - return NULL; - - if (deref->deref_type != nir_deref_type_var) { - /* The only one system value that is an array and that is - * gl_SampleMask which is always an array of one element. - */ - assert(deref->deref_type == nir_deref_type_array); - deref = nir_deref_instr_parent(deref); - assert(deref->deref_type == nir_deref_type_var); - assert(deref->var->data.location == SYSTEM_VALUE_SAMPLE_MASK_IN); - } - nir_variable *var = deref->var; - - switch (var->data.location) { - case SYSTEM_VALUE_INSTANCE_INDEX: - return nir_iadd(b, nir_load_instance_id(b), - nir_load_base_instance(b)); - - case SYSTEM_VALUE_SUBGROUP_EQ_MASK: - case SYSTEM_VALUE_SUBGROUP_GE_MASK: - case SYSTEM_VALUE_SUBGROUP_GT_MASK: - case SYSTEM_VALUE_SUBGROUP_LE_MASK: - case SYSTEM_VALUE_SUBGROUP_LT_MASK: { - nir_intrinsic_op op = - nir_intrinsic_from_system_value(var->data.location); - nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op); - nir_ssa_dest_init_for_type(&load->instr, &load->dest, - var->type, NULL); - load->num_components = load->dest.ssa.num_components; - nir_builder_instr_insert(b, &load->instr); - return &load->dest.ssa; - } - - case SYSTEM_VALUE_DEVICE_INDEX: - if (b->shader->options->lower_device_index_to_zero) - return nir_imm_int(b, 0); - break; - - case SYSTEM_VALUE_GLOBAL_GROUP_SIZE: - return build_global_group_size(b, bit_size); - - case SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel, - INTERP_MODE_NOPERSPECTIVE); - - case SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid, - INTERP_MODE_NOPERSPECTIVE); - - case SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample, - INTERP_MODE_NOPERSPECTIVE); - - case SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel, - INTERP_MODE_SMOOTH); - - case SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid, - INTERP_MODE_SMOOTH); - - case SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample, - INTERP_MODE_SMOOTH); - - case SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL: - return nir_load_barycentric(b, nir_intrinsic_load_barycentric_model, - INTERP_MODE_NONE); - - default: - break; - } - - nir_intrinsic_op sysval_op = - nir_intrinsic_from_system_value(var->data.location); - return nir_load_system_value(b, sysval_op, 0, - intrin->dest.ssa.num_components, - intrin->dest.ssa.bit_size); - } - default: return NULL; } } bool -nir_lower_system_values(nir_shader *shader) +nir_lower_compute_system_values(nir_shader *shader) { - bool progress = nir_shader_lower_instructions(shader, - lower_system_value_filter, - lower_system_value_instr, - NULL); - - /* We're going to delete the variables so we need to clean up all those - * derefs we left lying around. - */ - if (progress) - nir_remove_dead_derefs(shader); - - nir_foreach_variable_with_modes_safe(var, shader, nir_var_system_value) - exec_node_remove(&var->node); - - return progress; + if (shader->info.stage != MESA_SHADER_COMPUTE && + shader->info.stage != MESA_SHADER_KERNEL) + return false; + + return nir_shader_lower_instructions(shader, + lower_compute_system_value_filter, + lower_compute_system_value_instr, + NULL); } diff --git a/src/freedreno/vulkan/tu_shader.c b/src/freedreno/vulkan/tu_shader.c index ec8e2661e8a..aca5ea02f71 100644 --- a/src/freedreno/vulkan/tu_shader.c +++ b/src/freedreno/vulkan/tu_shader.c @@ -765,6 +765,8 @@ tu_shader_create(struct tu_device *dev, nir_assign_io_var_locations(nir, nir_var_shader_out, &nir->num_outputs, stage); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); + NIR_PASS_V(nir, nir_lower_frexp); if (stage == MESA_SHADER_FRAGMENT) { diff --git a/src/gallium/auxiliary/nir/tgsi_to_nir.c b/src/gallium/auxiliary/nir/tgsi_to_nir.c index 3035f5460a5..7b1b055af37 100644 --- a/src/gallium/auxiliary/nir/tgsi_to_nir.c +++ b/src/gallium/auxiliary/nir/tgsi_to_nir.c @@ -2559,6 +2559,7 @@ ttn_finalize_nir(struct ttn_compile *c, struct pipe_screen *screen) NIR_PASS_V(nir, nir_split_var_copies); NIR_PASS_V(nir, nir_lower_var_copies); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); if (c->cap_packed_uniforms) NIR_PASS_V(nir, nir_lower_uniforms_to_ubo, 16); diff --git a/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c b/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c index 5dad4da11d2..54227e4886d 100644 --- a/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c +++ b/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c @@ -185,6 +185,8 @@ load_glsl(unsigned num_files, char* const* files, gl_shader_stage stage) ir3_glsl_type_size); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); + NIR_PASS_V(nir, nir_lower_frexp); NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out | nir_var_uniform, @@ -401,6 +403,7 @@ main(int argc, char **argv) /* TODO do this somewhere else */ nir_lower_int64(nir); nir_lower_system_values(nir); + nir_lower_compute_system_values(nir); } else if (num_files > 0) { nir = load_glsl(num_files, filenames, stage); } else { diff --git a/src/gallium/frontends/clover/nir/invocation.cpp b/src/gallium/frontends/clover/nir/invocation.cpp index 032fb03cfd8..f5747205222 100644 --- a/src/gallium/frontends/clover/nir/invocation.cpp +++ b/src/gallium/frontends/clover/nir/invocation.cpp @@ -169,6 +169,8 @@ module clover::nir::spirv_to_nir(const module &mod, const device &dev, spirv_options.global_addr_format); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); + if (compiler_options->lower_int64_options) NIR_PASS_V(nir, nir_lower_int64); diff --git a/src/gallium/frontends/vallium/val_pipeline.c b/src/gallium/frontends/vallium/val_pipeline.c index 136ac93fbc3..779e2ecdc5f 100644 --- a/src/gallium/frontends/vallium/val_pipeline.c +++ b/src/gallium/frontends/vallium/val_pipeline.c @@ -562,6 +562,7 @@ val_shader_compile_to_ir(struct val_pipeline *pipeline, if (stage == MESA_SHADER_FRAGMENT) val_lower_input_attachments(nir, false); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); NIR_PASS_V(nir, nir_lower_clip_cull_distance_arrays); nir_remove_dead_variables(nir, nir_var_uniform, NULL); diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c index 567bb42c805..024cea7df5b 100644 --- a/src/intel/compiler/brw_nir.c +++ b/src/intel/compiler/brw_nir.c @@ -709,6 +709,7 @@ brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir, } OPT(nir_lower_system_values); + OPT(nir_lower_compute_system_values); const nir_lower_subgroups_options subgroups_options = { .ballot_bit_size = 32, diff --git a/src/mesa/state_tracker/st_glsl_to_nir.cpp b/src/mesa/state_tracker/st_glsl_to_nir.cpp index 99bbbef79db..b5b85ae46c0 100644 --- a/src/mesa/state_tracker/st_glsl_to_nir.cpp +++ b/src/mesa/state_tracker/st_glsl_to_nir.cpp @@ -771,6 +771,8 @@ st_link_nir(struct gl_context *ctx, st->pipe->screen); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); + NIR_PASS_V(nir, nir_lower_clip_cull_distance_arrays); st_shader_gather_info(nir, shader->Program); diff --git a/src/mesa/state_tracker/st_nir_builtins.c b/src/mesa/state_tracker/st_nir_builtins.c index 77ec8725ff7..cd078f3a561 100644 --- a/src/mesa/state_tracker/st_nir_builtins.c +++ b/src/mesa/state_tracker/st_nir_builtins.c @@ -43,6 +43,7 @@ st_nir_finish_builtin_shader(struct st_context *st, NIR_PASS_V(nir, nir_split_var_copies); NIR_PASS_V(nir, nir_lower_var_copies); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); if (nir->options->lower_to_scalar) { nir_variable_mode mask = diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c index 7e7373dc85e..7ba6344051c 100644 --- a/src/mesa/state_tracker/st_program.c +++ b/src/mesa/state_tracker/st_program.c @@ -388,6 +388,7 @@ st_translate_prog_to_nir(struct st_context *st, struct gl_program *prog, NIR_PASS_V(nir, st_nir_lower_wpos_ytransform, prog, screen); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values); /* Optimise NIR */ NIR_PASS_V(nir, nir_opt_constant_folding); -- 2.30.2