From 345b5847b42bc1889d8665ebd129913550da4352 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 5 May 2020 09:13:20 +0200 Subject: [PATCH] nir: Replace the scoped_memory barrier by a scoped_barrier SPIRV OpControlBarrier can have both a memory and a control barrier which some hardware can handle with a single instruction. Let's turn the scoped_memory_barrier into a scoped barrier which can embed both barrier types. Note that control-only or memory-only barriers can be supported through this new intrinsic by passing NIR_SCOPE_NONE to the unused barrier type. Signed-off-by: Boris Brezillon Suggested-by: Caio Marcelo de Oliveira Filho Reviewed-by: Caio Marcelo de Oliveira Filho Part-of: --- src/compiler/nir/nir.h | 13 ++++++++--- src/compiler/nir/nir_builder.h | 23 ++++++++++++++----- src/compiler/nir/nir_intrinsics.py | 13 +++++++---- src/compiler/nir/nir_opt_access.c | 2 +- src/compiler/nir/nir_opt_barriers.c | 3 ++- src/compiler/nir/nir_opt_combine_stores.c | 2 +- src/compiler/nir/nir_opt_copy_prop_vars.c | 4 ++-- src/compiler/nir/nir_opt_dead_write_vars.c | 2 +- .../nir/nir_opt_load_store_vectorize.c | 5 +++- src/compiler/nir/nir_print.c | 5 +++- src/compiler/spirv/spirv_to_nir.c | 12 +++------- src/intel/compiler/brw_compiler.c | 2 +- src/intel/compiler/brw_fs_nir.cpp | 5 ++-- src/intel/compiler/brw_vec4_nir.cpp | 6 +++-- 14 files changed, 61 insertions(+), 36 deletions(-) diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 79973655f26..b7f7e46c9b0 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -1525,6 +1525,7 @@ typedef enum { } nir_memory_semantics; typedef enum { + NIR_SCOPE_NONE, NIR_SCOPE_INVOCATION, NIR_SCOPE_SUBGROUP, NIR_SCOPE_WORKGROUP, @@ -1696,6 +1697,11 @@ typedef enum { */ NIR_INTRINSIC_MEMORY_SCOPE, + /** + * Value of nir_scope. + */ + NIR_INTRINSIC_EXECUTION_SCOPE, + NIR_INTRINSIC_NUM_INDEX_FLAGS, } nir_intrinsic_index_flag; @@ -1835,6 +1841,7 @@ INTRINSIC_IDX_ACCESSORS(driver_location, DRIVER_LOCATION, unsigned) INTRINSIC_IDX_ACCESSORS(memory_semantics, MEMORY_SEMANTICS, nir_memory_semantics) INTRINSIC_IDX_ACCESSORS(memory_modes, MEMORY_MODES, nir_variable_mode) INTRINSIC_IDX_ACCESSORS(memory_scope, MEMORY_SCOPE, nir_scope) +INTRINSIC_IDX_ACCESSORS(execution_scope, EXECUTION_SCOPE, nir_scope) static inline void nir_intrinsic_set_align(nir_intrinsic_instr *intrin, @@ -3127,10 +3134,10 @@ typedef struct nir_shader_compiler_options { * to imul with masked inputs and iadd */ bool has_umad24; - /* Whether to generate only scoped_memory_barrier intrinsics instead of the - * set of memory barrier intrinsics based on GLSL. + /* Whether to generate only scoped_barrier intrinsics instead of the set of + * memory and control barrier intrinsics based on GLSL. */ - bool use_scoped_memory_barrier; + bool use_scoped_barrier; /** * Is this the Intel vec4 backend? diff --git a/src/compiler/nir/nir_builder.h b/src/compiler/nir/nir_builder.h index 30206962255..6c4b2da6f5c 100644 --- a/src/compiler/nir/nir_builder.h +++ b/src/compiler/nir/nir_builder.h @@ -1359,18 +1359,29 @@ nir_compare_func(nir_builder *b, enum compare_func func, unreachable("bad compare func"); } +static inline void +nir_scoped_barrier(nir_builder *b, + nir_scope exec_scope, + nir_scope mem_scope, + nir_memory_semantics mem_semantics, + nir_variable_mode mem_modes) +{ + nir_intrinsic_instr *intrin = + nir_intrinsic_instr_create(b->shader, nir_intrinsic_scoped_barrier); + nir_intrinsic_set_execution_scope(intrin, exec_scope); + nir_intrinsic_set_memory_scope(intrin, mem_scope); + nir_intrinsic_set_memory_semantics(intrin, mem_semantics); + nir_intrinsic_set_memory_modes(intrin, mem_modes); + nir_builder_instr_insert(b, &intrin->instr); +} + static inline void nir_scoped_memory_barrier(nir_builder *b, nir_scope scope, nir_memory_semantics semantics, nir_variable_mode modes) { - nir_intrinsic_instr *intrin = - nir_intrinsic_instr_create(b->shader, nir_intrinsic_scoped_memory_barrier); - nir_intrinsic_set_memory_scope(intrin, scope); - nir_intrinsic_set_memory_semantics(intrin, semantics); - nir_intrinsic_set_memory_modes(intrin, modes); - nir_builder_instr_insert(b, &intrin->instr); + nir_scoped_barrier(b, NIR_SCOPE_NONE, scope, semantics, modes); } #endif /* NIR_BUILDER_H */ diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 26f421cfb95..8fb9eef1584 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -132,6 +132,8 @@ MEMORY_SEMANTICS = "NIR_INTRINSIC_MEMORY_SEMANTICS" MEMORY_MODES = "NIR_INTRINSIC_MEMORY_MODES" # Scope of a memory operation MEMORY_SCOPE = "NIR_INTRINSIC_MEMORY_SCOPE" +# Scope of a control barrier +EXECUTION_SCOPE = "NIR_INTRINSIC_EXECUTION_SCOPE" # # Possible flags: @@ -219,11 +221,12 @@ barrier("control_barrier") # intrinsic. barrier("memory_barrier") -# Memory barrier with explicit scope. Follows the semantics of SPIR-V -# OpMemoryBarrier, used to implement Vulkan Memory Model. Storage that the -# barrierr applies is represented using NIR variable modes. -intrinsic("scoped_memory_barrier", - indices=[MEMORY_SEMANTICS, MEMORY_MODES, MEMORY_SCOPE]) +# Control/Memory barrier with explicit scope. Follows the semantics of SPIR-V +# OpMemoryBarrier and OpControlBarrier, used to implement Vulkan Memory Model. +# Storage that the barrier applies is represented using NIR variable modes. +# For an OpMemoryBarrier, set EXECUTION_SCOPE to NIR_SCOPE_NONE. +intrinsic("scoped_barrier", + indices=[EXECUTION_SCOPE, MEMORY_SEMANTICS, MEMORY_MODES, MEMORY_SCOPE]) # Shader clock intrinsic with semantics analogous to the clock2x32ARB() # GLSL intrinsic. diff --git a/src/compiler/nir/nir_opt_access.c b/src/compiler/nir/nir_opt_access.c index e2a6b8f2747..fb432a9430f 100644 --- a/src/compiler/nir/nir_opt_access.c +++ b/src/compiler/nir/nir_opt_access.c @@ -134,7 +134,7 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) state->image_barriers = true; break; - case nir_intrinsic_scoped_memory_barrier: + case nir_intrinsic_scoped_barrier: /* TODO: Could be more granular if we had nir_var_mem_image. */ if (nir_intrinsic_memory_modes(instr) & (nir_var_mem_ubo | nir_var_mem_ssbo | diff --git a/src/compiler/nir/nir_opt_barriers.c b/src/compiler/nir/nir_opt_barriers.c index c1de0ed9163..609cbc0afdc 100644 --- a/src/compiler/nir/nir_opt_barriers.c +++ b/src/compiler/nir/nir_opt_barriers.c @@ -39,7 +39,8 @@ nir_opt_combine_memory_barriers_impl( } nir_intrinsic_instr *current = nir_instr_as_intrinsic(instr); - if (current->intrinsic != nir_intrinsic_scoped_memory_barrier) { + if (current->intrinsic != nir_intrinsic_scoped_barrier || + nir_intrinsic_memory_scope(current) != NIR_SCOPE_NONE) { prev = NULL; continue; } diff --git a/src/compiler/nir/nir_opt_combine_stores.c b/src/compiler/nir/nir_opt_combine_stores.c index 7495df2d52c..8295b0fd7a7 100644 --- a/src/compiler/nir/nir_opt_combine_stores.c +++ b/src/compiler/nir/nir_opt_combine_stores.c @@ -338,7 +338,7 @@ combine_stores_block(struct combine_stores_state *state, nir_block *block) combine_stores_with_modes(state, nir_var_shader_out); break; - case nir_intrinsic_scoped_memory_barrier: + case nir_intrinsic_scoped_barrier: if (nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_RELEASE) { combine_stores_with_modes(state, nir_intrinsic_memory_modes(intrin)); diff --git a/src/compiler/nir/nir_opt_copy_prop_vars.c b/src/compiler/nir/nir_opt_copy_prop_vars.c index d471c89fbb6..a1ea55cbfbf 100644 --- a/src/compiler/nir/nir_opt_copy_prop_vars.c +++ b/src/compiler/nir/nir_opt_copy_prop_vars.c @@ -174,7 +174,7 @@ gather_vars_written(struct copy_prop_var_state *state, nir_var_mem_global; break; - case nir_intrinsic_scoped_memory_barrier: + case nir_intrinsic_scoped_barrier: if (nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_ACQUIRE) written->modes |= nir_intrinsic_memory_modes(intrin); break; @@ -831,7 +831,7 @@ copy_prop_vars_block(struct copy_prop_var_state *state, apply_barrier_for_modes(copies, nir_var_shader_out); break; - case nir_intrinsic_scoped_memory_barrier: + case nir_intrinsic_scoped_barrier: if (debug) dump_instr(instr); if (nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_ACQUIRE) diff --git a/src/compiler/nir/nir_opt_dead_write_vars.c b/src/compiler/nir/nir_opt_dead_write_vars.c index c77b56a476c..adba32ac310 100644 --- a/src/compiler/nir/nir_opt_dead_write_vars.c +++ b/src/compiler/nir/nir_opt_dead_write_vars.c @@ -155,7 +155,7 @@ remove_dead_write_vars_local(void *mem_ctx, nir_block *block) clear_unused_for_modes(&unused_writes, nir_var_shader_out); break; - case nir_intrinsic_scoped_memory_barrier: { + case nir_intrinsic_scoped_barrier: { if (nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_RELEASE) { clear_unused_for_modes(&unused_writes, nir_intrinsic_memory_modes(intrin)); diff --git a/src/compiler/nir/nir_opt_load_store_vectorize.c b/src/compiler/nir/nir_opt_load_store_vectorize.c index b35bcdaf988..7777a63c5ae 100644 --- a/src/compiler/nir/nir_opt_load_store_vectorize.c +++ b/src/compiler/nir/nir_opt_load_store_vectorize.c @@ -1225,7 +1225,10 @@ handle_barrier(struct vectorize_ctx *ctx, bool *progress, nir_function_impl *imp case nir_intrinsic_memory_barrier_shared: modes = nir_var_mem_shared; break; - case nir_intrinsic_scoped_memory_barrier: + case nir_intrinsic_scoped_barrier: + if (nir_intrinsic_memory_scope(intrin) == NIR_SCOPE_NONE) + break; + modes = nir_intrinsic_memory_modes(intrin); acquire = nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_ACQUIRE; release = nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_RELEASE; diff --git a/src/compiler/nir/nir_print.c b/src/compiler/nir/nir_print.c index e9b6b1097cd..a7d1a9d7fef 100644 --- a/src/compiler/nir/nir_print.c +++ b/src/compiler/nir/nir_print.c @@ -786,6 +786,7 @@ print_intrinsic_instr(nir_intrinsic_instr *instr, print_state *state) [NIR_INTRINSIC_MEMORY_SEMANTICS] = "mem_semantics", [NIR_INTRINSIC_MEMORY_MODES] = "mem_modes", [NIR_INTRINSIC_MEMORY_SCOPE] = "mem_scope", + [NIR_INTRINSIC_EXECUTION_SCOPE] = "exec_scope", }; for (unsigned idx = 1; idx < NIR_INTRINSIC_NUM_INDEX_FLAGS; idx++) { if (!info->index_map[idx]) @@ -896,9 +897,11 @@ print_intrinsic_instr(nir_intrinsic_instr *instr, print_state *state) break; } + case NIR_INTRINSIC_EXECUTION_SCOPE: case NIR_INTRINSIC_MEMORY_SCOPE: { - fprintf(fp, " mem_scope="); + fprintf(fp, " %s=", index_name[idx]); switch (nir_intrinsic_memory_scope(instr)) { + case NIR_SCOPE_NONE: fprintf(fp, "NONE"); break; case NIR_SCOPE_DEVICE: fprintf(fp, "DEVICE"); break; case NIR_SCOPE_QUEUE_FAMILY: fprintf(fp, "QUEUE_FAMILY"); break; case NIR_SCOPE_WORKGROUP: fprintf(fp, "WORKGROUP"); break; diff --git a/src/compiler/spirv/spirv_to_nir.c b/src/compiler/spirv/spirv_to_nir.c index fd3674b04e8..7f8b1f07e02 100644 --- a/src/compiler/spirv/spirv_to_nir.c +++ b/src/compiler/spirv/spirv_to_nir.c @@ -2137,14 +2137,8 @@ vtn_emit_scoped_memory_barrier(struct vtn_builder *b, SpvScope scope, if (nir_semantics == 0 || modes == 0) return; - nir_scope nir_scope = vtn_scope_to_nir_scope(b, scope); - nir_intrinsic_instr *intrin = - nir_intrinsic_instr_create(b->shader, nir_intrinsic_scoped_memory_barrier); - nir_intrinsic_set_memory_semantics(intrin, nir_semantics); - - nir_intrinsic_set_memory_modes(intrin, modes); - nir_intrinsic_set_memory_scope(intrin, nir_scope); - nir_builder_instr_insert(&b->nb, &intrin->instr); + nir_scope nir_mem_scope = vtn_scope_to_nir_scope(b, scope); + nir_scoped_barrier(&b->nb, NIR_SCOPE_NONE, nir_mem_scope, nir_semantics, modes); } struct vtn_ssa_value * @@ -3525,7 +3519,7 @@ void vtn_emit_memory_barrier(struct vtn_builder *b, SpvScope scope, SpvMemorySemanticsMask semantics) { - if (b->shader->options->use_scoped_memory_barrier) { + if (b->shader->options->use_scoped_barrier) { vtn_emit_scoped_memory_barrier(b, scope, semantics); return; } diff --git a/src/intel/compiler/brw_compiler.c b/src/intel/compiler/brw_compiler.c index 0c28b81e912..c26b6eb3bbc 100644 --- a/src/intel/compiler/brw_compiler.c +++ b/src/intel/compiler/brw_compiler.c @@ -48,7 +48,7 @@ .use_interpolated_input_intrinsics = true, \ .vertex_id_zero_based = true, \ .lower_base_vertex = true, \ - .use_scoped_memory_barrier = true, \ + .use_scoped_barrier = true, \ .support_8bit_alu = true, \ .support_16bit_alu = true diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index 37c1d2f4cc9..37f03507bb0 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -4225,7 +4225,8 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr break; } - case nir_intrinsic_scoped_memory_barrier: + case nir_intrinsic_scoped_barrier: + assert(nir_intrinsic_execution_scope(instr) == NIR_SCOPE_NONE); case nir_intrinsic_group_memory_barrier: case nir_intrinsic_memory_barrier_shared: case nir_intrinsic_memory_barrier_buffer: @@ -4239,7 +4240,7 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr SHADER_OPCODE_INTERLOCK : SHADER_OPCODE_MEMORY_FENCE; switch (instr->intrinsic) { - case nir_intrinsic_scoped_memory_barrier: { + case nir_intrinsic_scoped_barrier: { nir_variable_mode modes = nir_intrinsic_memory_modes(instr); l3_fence = modes & (nir_var_shader_out | nir_var_mem_ssbo | diff --git a/src/intel/compiler/brw_vec4_nir.cpp b/src/intel/compiler/brw_vec4_nir.cpp index 76446adcf54..13a5d2f4fa9 100644 --- a/src/intel/compiler/brw_vec4_nir.cpp +++ b/src/intel/compiler/brw_vec4_nir.cpp @@ -700,8 +700,10 @@ vec4_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr) break; } - case nir_intrinsic_memory_barrier: - case nir_intrinsic_scoped_memory_barrier: { + case nir_intrinsic_scoped_barrier: + assert(nir_intrinsic_execution_scope(instr) == NIR_SCOPE_NONE); + /* Fall through. */ + case nir_intrinsic_memory_barrier: { const vec4_builder bld = vec4_builder(this).at_end().annotate(current_annotation, base_ir); const dst_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD); -- 2.30.2