From 951cf94521a710fa2fa70329ff77934ada45bb70 Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Tue, 20 Aug 2019 22:32:50 -0500 Subject: [PATCH] nir: Add explicit signs to image min/max intrinsics This better matches all the other atomic intrinsics such as those for SSBOs and shared variables where the sign is part of the intrinsic opcode. Both generators (GLSL and SPIR-V) know the sign from the type of the image variable or handle. In SPIR-V, signed min/max are separate opcodes from unsigned. Reviewed-by: Kenneth Graunke Reviewed-by: Eric Anholt --- src/amd/common/ac_nir_to_llvm.c | 47 +++++++++------ src/amd/vulkan/radv_shader_info.c | 12 ++-- src/broadcom/compiler/nir_to_vir.c | 6 +- src/broadcom/compiler/v3d40_tex.c | 6 +- .../glsl/gl_nir_lower_bindless_images.c | 6 +- .../glsl/gl_nir_lower_samplers_as_deref.c | 6 +- src/compiler/glsl/gl_nir_opt_access.c | 12 ++-- src/compiler/glsl/glsl_to_nir.cpp | 20 +++++-- src/compiler/nir/nir.c | 6 +- src/compiler/nir/nir_divergence_analysis.c | 18 ++++-- src/compiler/nir/nir_intrinsics.py | 6 +- .../nir/nir_lower_non_uniform_access.c | 12 ++-- src/compiler/spirv/spirv_to_nir.c | 8 +-- src/freedreno/ir3/ir3_a4xx.c | 6 +- src/freedreno/ir3/ir3_a6xx.c | 6 +- src/freedreno/ir3/ir3_compiler_nir.c | 6 +- src/freedreno/ir3/ir3_nir.c | 6 +- src/gallium/drivers/iris/iris_program.c | 18 ++++-- .../nouveau/codegen/nv50_ir_from_nir.cpp | 60 ++++++++++++------- src/gallium/drivers/radeonsi/si_shader_nir.c | 12 ++-- src/intel/compiler/brw_fs_nir.cpp | 44 +++++++++----- .../compiler/brw_nir_analyze_ubo_ranges.c | 6 +- .../compiler/brw_nir_lower_image_load_store.c | 6 +- .../vulkan/anv_nir_apply_pipeline_layout.c | 12 ++-- .../drivers/dri/i965/brw_nir_uniforms.cpp | 6 +- 25 files changed, 236 insertions(+), 117 deletions(-) diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c index d97387ef13d..33a95bc71ae 100644 --- a/src/amd/common/ac_nir_to_llvm.c +++ b/src/amd/common/ac_nir_to_llvm.c @@ -2621,20 +2621,19 @@ static LLVMValueRef visit_image_atomic(struct ac_nir_context *ctx, ASSERTED int length; enum glsl_sampler_dim dim; - bool is_unsigned = false; bool is_array; if (bindless) { - if (instr->intrinsic == nir_intrinsic_bindless_image_atomic_min || - instr->intrinsic == nir_intrinsic_bindless_image_atomic_max) { + if (instr->intrinsic == nir_intrinsic_bindless_image_atomic_imin || + instr->intrinsic == nir_intrinsic_bindless_image_atomic_umin || + instr->intrinsic == nir_intrinsic_bindless_image_atomic_imax || + instr->intrinsic == nir_intrinsic_bindless_image_atomic_umax) { const GLenum format = nir_intrinsic_format(instr); assert(format == GL_R32UI || format == GL_R32I); - is_unsigned = format == GL_R32UI; } dim = nir_intrinsic_image_dim(instr); is_array = nir_intrinsic_image_array(instr); } else { const struct glsl_type *type = get_image_deref(instr)->type; - is_unsigned = glsl_get_sampler_result_type(type) == GLSL_TYPE_UINT; dim = glsl_get_sampler_dim(type); is_array = glsl_sampler_type_is_array(type); } @@ -2645,15 +2644,25 @@ static LLVMValueRef visit_image_atomic(struct ac_nir_context *ctx, atomic_name = "add"; atomic_subop = ac_atomic_add; break; - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_image_deref_atomic_min: - atomic_name = is_unsigned ? "umin" : "smin"; - atomic_subop = is_unsigned ? ac_atomic_umin : ac_atomic_smin; + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_image_deref_atomic_imin: + atomic_name = "smin"; + atomic_subop = ac_atomic_smin; break; - case nir_intrinsic_bindless_image_atomic_max: - case nir_intrinsic_image_deref_atomic_max: - atomic_name = is_unsigned ? "umax" : "smax"; - atomic_subop = is_unsigned ? ac_atomic_umax : ac_atomic_smax; + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_image_deref_atomic_umin: + atomic_name = "umin"; + atomic_subop = ac_atomic_umin; + break; + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_image_deref_atomic_imax: + atomic_name = "smax"; + atomic_subop = ac_atomic_smax; + break; + case nir_intrinsic_bindless_image_atomic_umax: + case nir_intrinsic_image_deref_atomic_umax: + atomic_name = "umax"; + atomic_subop = ac_atomic_umax; break; case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_image_deref_atomic_and: @@ -3407,8 +3416,10 @@ static void visit_intrinsic(struct ac_nir_context *ctx, visit_image_store(ctx, instr, false); break; case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_max: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: @@ -3419,8 +3430,10 @@ static void visit_intrinsic(struct ac_nir_context *ctx, result = visit_image_atomic(ctx, instr, true); break; case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/amd/vulkan/radv_shader_info.c b/src/amd/vulkan/radv_shader_info.c index 946f7aa5f23..4742036a1dc 100644 --- a/src/amd/vulkan/radv_shader_info.c +++ b/src/amd/vulkan/radv_shader_info.c @@ -284,8 +284,10 @@ gather_intrinsic_info(const nir_shader *nir, const nir_intrinsic_instr *instr, case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: @@ -297,8 +299,10 @@ gather_intrinsic_info(const nir_shader *nir, const nir_intrinsic_instr *instr, if (instr->intrinsic == nir_intrinsic_image_deref_store || instr->intrinsic == nir_intrinsic_image_deref_atomic_add || - instr->intrinsic == nir_intrinsic_image_deref_atomic_min || - instr->intrinsic == nir_intrinsic_image_deref_atomic_max || + instr->intrinsic == nir_intrinsic_image_deref_atomic_imin || + instr->intrinsic == nir_intrinsic_image_deref_atomic_umin || + instr->intrinsic == nir_intrinsic_image_deref_atomic_imax || + instr->intrinsic == nir_intrinsic_image_deref_atomic_umax || instr->intrinsic == nir_intrinsic_image_deref_atomic_and || instr->intrinsic == nir_intrinsic_image_deref_atomic_or || instr->intrinsic == nir_intrinsic_image_deref_atomic_xor || diff --git a/src/broadcom/compiler/nir_to_vir.c b/src/broadcom/compiler/nir_to_vir.c index a8fb0335bfa..91e95f9ee5a 100644 --- a/src/broadcom/compiler/nir_to_vir.c +++ b/src/broadcom/compiler/nir_to_vir.c @@ -1980,8 +1980,10 @@ ntq_emit_intrinsic(struct v3d_compile *c, nir_intrinsic_instr *instr) case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/broadcom/compiler/v3d40_tex.c b/src/broadcom/compiler/v3d40_tex.c index 30f1293c85c..9ee1692a77c 100644 --- a/src/broadcom/compiler/v3d40_tex.c +++ b/src/broadcom/compiler/v3d40_tex.c @@ -252,9 +252,11 @@ v3d40_image_load_store_tmu_op(nir_intrinsic_instr *instr) return V3D_TMU_OP_REGULAR; case nir_intrinsic_image_deref_atomic_add: return v3d_get_op_for_atomic_add(instr, 3); - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: return V3D_TMU_OP_WRITE_UMIN_FULL_L1_CLEAR; - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: return V3D_TMU_OP_WRITE_UMAX; case nir_intrinsic_image_deref_atomic_and: return V3D_TMU_OP_WRITE_AND_READ_INC; diff --git a/src/compiler/glsl/gl_nir_lower_bindless_images.c b/src/compiler/glsl/gl_nir_lower_bindless_images.c index 4c76977195b..345d8f60670 100644 --- a/src/compiler/glsl/gl_nir_lower_bindless_images.c +++ b/src/compiler/glsl/gl_nir_lower_bindless_images.c @@ -48,8 +48,10 @@ lower_impl(nir_builder *b, nir_instr *instr) { switch (intrinsic->intrinsic) { case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c index 5907abfd9a9..3c8ef4f0521 100644 --- a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c +++ b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c @@ -271,8 +271,10 @@ lower_intrinsic(nir_intrinsic_instr *instr, if (instr->intrinsic == nir_intrinsic_image_deref_load || instr->intrinsic == nir_intrinsic_image_deref_store || instr->intrinsic == nir_intrinsic_image_deref_atomic_add || - instr->intrinsic == nir_intrinsic_image_deref_atomic_min || - instr->intrinsic == nir_intrinsic_image_deref_atomic_max || + instr->intrinsic == nir_intrinsic_image_deref_atomic_imin || + instr->intrinsic == nir_intrinsic_image_deref_atomic_umin || + instr->intrinsic == nir_intrinsic_image_deref_atomic_imax || + instr->intrinsic == nir_intrinsic_image_deref_atomic_umax || instr->intrinsic == nir_intrinsic_image_deref_atomic_and || instr->intrinsic == nir_intrinsic_image_deref_atomic_or || instr->intrinsic == nir_intrinsic_image_deref_atomic_xor || diff --git a/src/compiler/glsl/gl_nir_opt_access.c b/src/compiler/glsl/gl_nir_opt_access.c index 760214fbbff..cfd7f392897 100644 --- a/src/compiler/glsl/gl_nir_opt_access.c +++ b/src/compiler/glsl/gl_nir_opt_access.c @@ -56,8 +56,10 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) switch (instr->intrinsic) { case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: @@ -82,8 +84,10 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_max: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: diff --git a/src/compiler/glsl/glsl_to_nir.cpp b/src/compiler/glsl/glsl_to_nir.cpp index 3166bf2c4e9..1e096a66c20 100644 --- a/src/compiler/glsl/glsl_to_nir.cpp +++ b/src/compiler/glsl/glsl_to_nir.cpp @@ -970,10 +970,20 @@ nir_visitor::visit(ir_call *ir) : nir_intrinsic_image_deref_atomic_fadd; break; case ir_intrinsic_image_atomic_min: - op = nir_intrinsic_image_deref_atomic_min; + if (ir->return_deref->type == glsl_type::int_type) + op = nir_intrinsic_image_deref_atomic_imin; + else if (ir->return_deref->type == glsl_type::uint_type) + op = nir_intrinsic_image_deref_atomic_umin; + else + unreachable("Invalid type"); break; case ir_intrinsic_image_atomic_max: - op = nir_intrinsic_image_deref_atomic_max; + if (ir->return_deref->type == glsl_type::int_type) + op = nir_intrinsic_image_deref_atomic_imax; + else if (ir->return_deref->type == glsl_type::uint_type) + op = nir_intrinsic_image_deref_atomic_umax; + else + unreachable("Invalid type"); break; case ir_intrinsic_image_atomic_and: op = nir_intrinsic_image_deref_atomic_and; @@ -1255,8 +1265,10 @@ nir_visitor::visit(ir_call *ir) case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c index 6547a295852..1cec8171a7f 100644 --- a/src/compiler/nir/nir.c +++ b/src/compiler/nir/nir.c @@ -2185,8 +2185,10 @@ nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src, CASE(load) CASE(store) CASE(atomic_add) - CASE(atomic_min) - CASE(atomic_max) + CASE(atomic_imin) + CASE(atomic_umin) + CASE(atomic_imax) + CASE(atomic_umax) CASE(atomic_and) CASE(atomic_or) CASE(atomic_xor) diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index fd61c602aff..301d7a4820b 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -310,8 +310,10 @@ visit_intrinsic(bool *divergent, nir_intrinsic_instr *instr, case nir_intrinsic_ssbo_atomic_fmin: case nir_intrinsic_ssbo_atomic_fcomp_swap: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: @@ -319,8 +321,10 @@ visit_intrinsic(bool *divergent, nir_intrinsic_instr *instr, case nir_intrinsic_image_deref_atomic_comp_swap: case nir_intrinsic_image_deref_atomic_fadd: case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_atomic_max: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: @@ -328,8 +332,10 @@ visit_intrinsic(bool *divergent, nir_intrinsic_instr *instr, case nir_intrinsic_image_atomic_comp_swap: case nir_intrinsic_image_atomic_fadd: case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_max: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index e1b336c2f60..824cfb5e23a 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -368,8 +368,10 @@ def image(name, src_comp=[], **kwargs): image("load", src_comp=[4, 1], dest_comp=0, flags=[CAN_ELIMINATE]) image("store", src_comp=[4, 1, 0]) image("atomic_add", src_comp=[4, 1, 1], dest_comp=1) -image("atomic_min", src_comp=[4, 1, 1], dest_comp=1) -image("atomic_max", src_comp=[4, 1, 1], dest_comp=1) +image("atomic_imin", src_comp=[4, 1, 1], dest_comp=1) +image("atomic_umin", src_comp=[4, 1, 1], dest_comp=1) +image("atomic_imax", src_comp=[4, 1, 1], dest_comp=1) +image("atomic_umax", src_comp=[4, 1, 1], dest_comp=1) image("atomic_and", src_comp=[4, 1, 1], dest_comp=1) image("atomic_or", src_comp=[4, 1, 1], dest_comp=1) image("atomic_xor", src_comp=[4, 1, 1], dest_comp=1) diff --git a/src/compiler/nir/nir_lower_non_uniform_access.c b/src/compiler/nir/nir_lower_non_uniform_access.c index 0ab32100ef2..5b6b858ca38 100644 --- a/src/compiler/nir/nir_lower_non_uniform_access.c +++ b/src/compiler/nir/nir_lower_non_uniform_access.c @@ -179,8 +179,10 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_atomic_max: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: @@ -192,8 +194,10 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_max: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: diff --git a/src/compiler/spirv/spirv_to_nir.c b/src/compiler/spirv/spirv_to_nir.c index 08649be080c..8cf26713053 100644 --- a/src/compiler/spirv/spirv_to_nir.c +++ b/src/compiler/spirv/spirv_to_nir.c @@ -2478,10 +2478,10 @@ vtn_handle_image(struct vtn_builder *b, SpvOp opcode, OP(AtomicIDecrement, atomic_add) OP(AtomicIAdd, atomic_add) OP(AtomicISub, atomic_add) - OP(AtomicSMin, atomic_min) - OP(AtomicUMin, atomic_min) - OP(AtomicSMax, atomic_max) - OP(AtomicUMax, atomic_max) + OP(AtomicSMin, atomic_imin) + OP(AtomicUMin, atomic_umin) + OP(AtomicSMax, atomic_imax) + OP(AtomicUMax, atomic_umax) OP(AtomicAnd, atomic_and) OP(AtomicOr, atomic_or) OP(AtomicXor, atomic_xor) diff --git a/src/freedreno/ir3/ir3_a4xx.c b/src/freedreno/ir3/ir3_a4xx.c index 30e452540bf..e1d188652b8 100644 --- a/src/freedreno/ir3/ir3_a4xx.c +++ b/src/freedreno/ir3/ir3_a4xx.c @@ -317,10 +317,12 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) case nir_intrinsic_image_deref_atomic_add: atomic = ir3_ATOMIC_ADD_G(b, image, 0, src0, 0, src1, 0, src2, 0); break; - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: atomic = ir3_ATOMIC_MIN_G(b, image, 0, src0, 0, src1, 0, src2, 0); break; - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: atomic = ir3_ATOMIC_MAX_G(b, image, 0, src0, 0, src1, 0, src2, 0); break; case nir_intrinsic_image_deref_atomic_and: diff --git a/src/freedreno/ir3/ir3_a6xx.c b/src/freedreno/ir3/ir3_a6xx.c index 845e81dc056..3f5127e72ee 100644 --- a/src/freedreno/ir3/ir3_a6xx.c +++ b/src/freedreno/ir3/ir3_a6xx.c @@ -276,10 +276,12 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) case nir_intrinsic_image_deref_atomic_add: atomic = ir3_ATOMIC_ADD_G(b, ibo, 0, src0, 0, src1, 0); break; - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: atomic = ir3_ATOMIC_MIN_G(b, ibo, 0, src0, 0, src1, 0); break; - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: atomic = ir3_ATOMIC_MAX_G(b, ibo, 0, src0, 0, src1, 0); break; case nir_intrinsic_image_deref_atomic_and: diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c index 3f4a0f43c99..3c58496097a 100644 --- a/src/freedreno/ir3/ir3_compiler_nir.c +++ b/src/freedreno/ir3/ir3_compiler_nir.c @@ -1415,8 +1415,10 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) emit_intrinsic_image_size(ctx, intr, dst); break; case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/freedreno/ir3/ir3_nir.c b/src/freedreno/ir3/ir3_nir.c index f258c46f94b..05426bc7a0e 100644 --- a/src/freedreno/ir3/ir3_nir.c +++ b/src/freedreno/ir3/ir3_nir.c @@ -311,8 +311,10 @@ ir3_nir_scan_driver_consts(nir_shader *shader, layout->ssbo_size.count += 1; /* one const per */ break; case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c index 7a64951d0bd..579055ea345 100644 --- a/src/gallium/drivers/iris/iris_program.c +++ b/src/gallium/drivers/iris/iris_program.c @@ -152,8 +152,10 @@ iris_lower_storage_image_derefs(nir_shader *nir) case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: @@ -723,8 +725,10 @@ iris_setup_binding_table(const struct gen_device_info *devinfo, case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_atomic_max: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: @@ -816,8 +820,10 @@ iris_setup_binding_table(const struct gen_device_info *devinfo, case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_atomic_max: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: diff --git a/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp b/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp index 950923a0daf..378638bf3a4 100644 --- a/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp +++ b/src/gallium/drivers/nouveau/codegen/nv50_ir_from_nir.cpp @@ -515,12 +515,18 @@ Converter::getOperation(nir_intrinsic_op op) case nir_intrinsic_bindless_image_atomic_exchange: case nir_intrinsic_image_atomic_exchange: case nir_intrinsic_image_deref_atomic_exchange: - case nir_intrinsic_bindless_image_atomic_max: - case nir_intrinsic_image_atomic_max: - case nir_intrinsic_image_deref_atomic_max: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: + case nir_intrinsic_image_atomic_umax: + case nir_intrinsic_image_deref_atomic_umax: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_deref_atomic_or: @@ -608,17 +614,23 @@ Converter::getSubOp(nir_intrinsic_op op) case nir_intrinsic_shared_atomic_or: case nir_intrinsic_ssbo_atomic_or: return NV50_IR_SUBOP_ATOM_OR; - case nir_intrinsic_bindless_image_atomic_max: - case nir_intrinsic_image_atomic_max: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: + case nir_intrinsic_image_atomic_umax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_shared_atomic_imax: case nir_intrinsic_shared_atomic_umax: case nir_intrinsic_ssbo_atomic_imax: case nir_intrinsic_ssbo_atomic_umax: return NV50_IR_SUBOP_ATOM_MAX; - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_shared_atomic_imin: case nir_intrinsic_shared_atomic_umin: case nir_intrinsic_ssbo_atomic_imin: @@ -2374,8 +2386,10 @@ Converter::visit(nir_intrinsic_instr *insn) case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_comp_swap: case nir_intrinsic_bindless_image_atomic_exchange: - case nir_intrinsic_bindless_image_atomic_max: - case nir_intrinsic_bindless_image_atomic_min: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: case nir_intrinsic_bindless_image_load: @@ -2405,8 +2419,10 @@ Converter::visit(nir_intrinsic_instr *insn) case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_comp_swap: case nir_intrinsic_bindless_image_atomic_exchange: - case nir_intrinsic_bindless_image_atomic_max: - case nir_intrinsic_bindless_image_atomic_min: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: ty = getDType(insn); @@ -2472,8 +2488,10 @@ Converter::visit(nir_intrinsic_instr *insn) case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_comp_swap: case nir_intrinsic_image_deref_atomic_exchange: - case nir_intrinsic_image_deref_atomic_max: - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: case nir_intrinsic_image_deref_load: @@ -2507,8 +2525,10 @@ Converter::visit(nir_intrinsic_instr *insn) case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_comp_swap: case nir_intrinsic_image_deref_atomic_exchange: - case nir_intrinsic_image_deref_atomic_max: - case nir_intrinsic_image_deref_atomic_min: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: ty = getDType(insn); diff --git a/src/gallium/drivers/radeonsi/si_shader_nir.c b/src/gallium/drivers/radeonsi/si_shader_nir.c index 518d11c3b48..c6ca057ea9f 100644 --- a/src/gallium/drivers/radeonsi/si_shader_nir.c +++ b/src/gallium/drivers/radeonsi/si_shader_nir.c @@ -281,8 +281,10 @@ static void scan_instruction(const struct nir_shader *nir, info->num_memory_instructions++; /* we only care about stores */ break; case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_max: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: @@ -299,8 +301,10 @@ static void scan_instruction(const struct nir_shader *nir, info->num_memory_instructions++; /* we only care about stores */ break; case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index 827096241f3..8980163401a 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -3972,8 +3972,10 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_atomic_max: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: @@ -3982,8 +3984,10 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_max: + case nir_intrinsic_bindless_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_umax: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: @@ -4003,8 +4007,10 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_image_atomic_max: + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: @@ -4053,15 +4059,25 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr if (op != BRW_AOP_ADD) num_srcs = 3; break; - case nir_intrinsic_image_atomic_min: - case nir_intrinsic_bindless_image_atomic_min: - assert(format == GL_R32UI || format == GL_R32I); - op = (format == GL_R32I) ? BRW_AOP_IMIN : BRW_AOP_UMIN; + case nir_intrinsic_image_atomic_imin: + case nir_intrinsic_bindless_image_atomic_imin: + assert(format == GL_R32I); + op = BRW_AOP_IMIN; break; - case nir_intrinsic_image_atomic_max: - case nir_intrinsic_bindless_image_atomic_max: - assert(format == GL_R32UI || format == GL_R32I); - op = (format == GL_R32I) ? BRW_AOP_IMAX : BRW_AOP_UMAX; + case nir_intrinsic_image_atomic_umin: + case nir_intrinsic_bindless_image_atomic_umin: + assert(format == GL_R32UI); + op = BRW_AOP_UMIN; + break; + case nir_intrinsic_image_atomic_imax: + case nir_intrinsic_bindless_image_atomic_imax: + assert(format == GL_R32I); + op = BRW_AOP_IMAX; + break; + case nir_intrinsic_image_atomic_umax: + case nir_intrinsic_bindless_image_atomic_umax: + assert(format == GL_R32UI); + op = BRW_AOP_UMAX; break; case nir_intrinsic_image_atomic_and: case nir_intrinsic_bindless_image_atomic_and: diff --git a/src/intel/compiler/brw_nir_analyze_ubo_ranges.c b/src/intel/compiler/brw_nir_analyze_ubo_ranges.c index 4c5e03380e1..35b0c945343 100644 --- a/src/intel/compiler/brw_nir_analyze_ubo_ranges.c +++ b/src/intel/compiler/brw_nir_analyze_ubo_ranges.c @@ -129,8 +129,10 @@ analyze_ubos_block(struct ubo_analysis_state *state, nir_block *block) case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/intel/compiler/brw_nir_lower_image_load_store.c b/src/intel/compiler/brw_nir_lower_image_load_store.c index c23f64fb338..b54cf32f443 100644 --- a/src/intel/compiler/brw_nir_lower_image_load_store.c +++ b/src/intel/compiler/brw_nir_lower_image_load_store.c @@ -763,8 +763,10 @@ brw_nir_lower_image_load_store(nir_shader *shader, break; case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/intel/vulkan/anv_nir_apply_pipeline_layout.c b/src/intel/vulkan/anv_nir_apply_pipeline_layout.c index f9a281b7019..1d13aa60492 100644 --- a/src/intel/vulkan/anv_nir_apply_pipeline_layout.c +++ b/src/intel/vulkan/anv_nir_apply_pipeline_layout.c @@ -114,8 +114,10 @@ get_used_bindings_block(nir_block *block, case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: @@ -1043,8 +1045,10 @@ apply_pipeline_layout_block(nir_block *block, case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: diff --git a/src/mesa/drivers/dri/i965/brw_nir_uniforms.cpp b/src/mesa/drivers/dri/i965/brw_nir_uniforms.cpp index 2ea316e7420..e31c91175c9 100644 --- a/src/mesa/drivers/dri/i965/brw_nir_uniforms.cpp +++ b/src/mesa/drivers/dri/i965/brw_nir_uniforms.cpp @@ -333,8 +333,10 @@ brw_nir_lower_gl_images(nir_shader *shader, case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_min: - case nir_intrinsic_image_deref_atomic_max: + case nir_intrinsic_image_deref_atomic_imin: + case nir_intrinsic_image_deref_atomic_umin: + case nir_intrinsic_image_deref_atomic_imax: + case nir_intrinsic_image_deref_atomic_umax: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_or: case nir_intrinsic_image_deref_atomic_xor: -- 2.30.2