From 655a7033493692961428cf0861477ae89f458e9b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Daniel=20Sch=C3=BCrmann?= Date: Tue, 29 Oct 2019 11:57:11 +0100 Subject: [PATCH] aco: remove potential critical edge on loops. Reviewed-by: Rhys Perry --- src/amd/compiler/aco_insert_exec_mask.cpp | 8 ++--- .../compiler/aco_instruction_selection.cpp | 33 ++++++++++++------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/amd/compiler/aco_insert_exec_mask.cpp b/src/amd/compiler/aco_insert_exec_mask.cpp index 3f4b48e661f..530faab2167 100644 --- a/src/amd/compiler/aco_insert_exec_mask.cpp +++ b/src/amd/compiler/aco_insert_exec_mask.cpp @@ -895,15 +895,11 @@ void add_branch_code(exec_ctx& ctx, Block* block) } if (block->kind & block_kind_continue_or_break) { + assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind & block_kind_loop_header); + assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind & block_kind_loop_exit); assert(block->instructions.back()->opcode == aco_opcode::p_branch); block->instructions.pop_back(); - /* because of how linear_succs is created, this needs to be swapped */ - std::swap(block->linear_succs[0], block->linear_succs[1]); - - assert(ctx.program->blocks[block->linear_succs[1]].kind & block_kind_loop_header); - assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind & block_kind_loop_exit); - if (ctx.info[idx].exec.back().second & mask_type_loop) { bld.branch(aco_opcode::p_cbranch_nz, bld.exec(ctx.info[idx].exec.back().first), block->linear_succs[1], block->linear_succs[0]); } else { diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index c279f04a031..e935dc11a9b 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -7057,26 +7057,35 @@ static void visit_loop(isel_context *ctx, nir_loop *loop) * workaround, break the loop when the loop mask is empty instead of * always continuing. */ ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform); - - /* create "loop_almost_exit" to avoid critical edges */ unsigned block_idx = ctx->block->index; - Block *loop_almost_exit = ctx->program->create_and_insert_block(); - loop_almost_exit->loop_nest_depth = ctx->cf_info.loop_nest_depth; - loop_almost_exit->kind = block_kind_uniform; - bld.reset(loop_almost_exit); + + /* create helper blocks to avoid critical edges */ + Block *break_block = ctx->program->create_and_insert_block(); + break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth; + break_block->kind = block_kind_uniform; + bld.reset(break_block); bld.branch(aco_opcode::p_branch); + add_linear_edge(block_idx, break_block); + add_linear_edge(break_block->index, &loop_exit); - add_linear_edge(block_idx, loop_almost_exit); - add_linear_edge(loop_almost_exit->index, &loop_exit); + Block *continue_block = ctx->program->create_and_insert_block(); + continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth; + continue_block->kind = block_kind_uniform; + bld.reset(continue_block); + bld.branch(aco_opcode::p_branch); + add_linear_edge(block_idx, continue_block); + add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]); + add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]); ctx->block = &ctx->program->blocks[block_idx]; } else { ctx->block->kind |= (block_kind_continue | block_kind_uniform); + if (!ctx->cf_info.parent_loop.has_divergent_branch) + add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]); + else + add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]); } - if (!ctx->cf_info.parent_loop.has_divergent_branch) - add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]); - else - add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]); + bld.reset(ctx->block); bld.branch(aco_opcode::p_branch); } -- 2.30.2