From 922cd38172b8a2bc286bd082fde9cad4e278765b Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Wed, 28 Mar 2018 19:03:00 +0200 Subject: [PATCH] radv: implement out-of-order rasterization when it's safe on VI+ Disabled by default for now, it can be enabled with RADV_PERFTEST=outoforder. No CTS regressions on Polaris, and all Vulkan games I tested look good as well. Expect small performance improvements for applications where out-of-order rasterization can be enabled by the driver. Loosely based on RadeonSI. Signed-off-by: Samuel Pitoiset Reviewed-by: Bas Nieuwenhuizen --- src/amd/vulkan/radv_cmd_buffer.c | 28 ++++ src/amd/vulkan/radv_debug.h | 1 + src/amd/vulkan/radv_device.c | 7 + src/amd/vulkan/radv_extensions.py | 2 +- src/amd/vulkan/radv_pipeline.c | 240 +++++++++++++++++++++++++++++- src/amd/vulkan/radv_private.h | 4 + 6 files changed, 279 insertions(+), 3 deletions(-) diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index b3d6fc484ef..b18718458fe 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -1171,10 +1171,24 @@ radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer) { + struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; + uint32_t pa_sc_mode_cntl_1 = + pipeline ? pipeline->graphics.ms.pa_sc_mode_cntl_1 : 0; uint32_t db_count_control; if(!cmd_buffer->state.active_occlusion_queries) { if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { + if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) && + pipeline->graphics.disable_out_of_order_rast_for_occlusion) { + /* Re-enable out-of-order rasterization if the + * bound pipeline supports it and if it's has + * been disabled before starting occlusion + * queries. + */ + radeon_set_context_reg(cmd_buffer->cs, + R_028A4C_PA_SC_MODE_CNTL_1, + pa_sc_mode_cntl_1); + } db_count_control = 0; } else { db_count_control = S_028004_ZPASS_INCREMENT_DISABLE(1); @@ -1189,6 +1203,20 @@ void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer) S_028004_ZPASS_ENABLE(1) | S_028004_SLICE_EVEN_ENABLE(1) | S_028004_SLICE_ODD_ENABLE(1); + + if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) && + pipeline->graphics.disable_out_of_order_rast_for_occlusion) { + /* If the bound pipeline has enabled + * out-of-order rasterization, we should + * disable it before starting occlusion + * queries. + */ + pa_sc_mode_cntl_1 &= C_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE; + + radeon_set_context_reg(cmd_buffer->cs, + R_028A4C_PA_SC_MODE_CNTL_1, + pa_sc_mode_cntl_1); + } } else { db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) | S_028004_SAMPLE_RATE(sample_rate); diff --git a/src/amd/vulkan/radv_debug.h b/src/amd/vulkan/radv_debug.h index 08877676b5d..f35991fa4e5 100644 --- a/src/amd/vulkan/radv_debug.h +++ b/src/amd/vulkan/radv_debug.h @@ -50,6 +50,7 @@ enum { RADV_PERFTEST_SISCHED = 0x2, RADV_PERFTEST_LOCAL_BOS = 0x4, RADV_PERFTEST_BINNING = 0x8, + RADV_PERFTEST_OUT_OF_ORDER = 0x10, }; bool diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index 65727571a3a..41f8242754c 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -307,6 +307,12 @@ radv_physical_device_init(struct radv_physical_device *device, device->has_scissor_bug = device->rad_info.family == CHIP_VEGA10 || device->rad_info.family == CHIP_RAVEN; + /* Out-of-order primitive rasterization. */ + device->has_out_of_order_rast = device->rad_info.chip_class >= VI && + device->rad_info.max_se >= 2; + device->out_of_order_rast_allowed = device->has_out_of_order_rast && + (device->instance->perftest_flags & RADV_PERFTEST_OUT_OF_ORDER); + radv_physical_device_init_mem_types(device); radv_fill_device_extension_table(device, &device->supported_extensions); @@ -391,6 +397,7 @@ static const struct debug_control radv_perftest_options[] = { {"sisched", RADV_PERFTEST_SISCHED}, {"localbos", RADV_PERFTEST_LOCAL_BOS}, {"binning", RADV_PERFTEST_BINNING}, + {"outoforderrast", RADV_PERFTEST_OUT_OF_ORDER}, {NULL, 0} }; diff --git a/src/amd/vulkan/radv_extensions.py b/src/amd/vulkan/radv_extensions.py index 8cb0b7aa7d5..bc63a34896a 100644 --- a/src/amd/vulkan/radv_extensions.py +++ b/src/amd/vulkan/radv_extensions.py @@ -95,7 +95,7 @@ EXTENSIONS = [ Extension('VK_EXT_shader_viewport_index_layer', 1, True), Extension('VK_AMD_draw_indirect_count', 1, True), Extension('VK_AMD_gcn_shader', 1, True), - Extension('VK_AMD_rasterization_order', 1, 'device->rad_info.chip_class >= VI && device->rad_info.max_se >= 2'), + Extension('VK_AMD_rasterization_order', 1, 'device->has_out_of_order_rast'), Extension('VK_AMD_shader_info', 1, True), Extension('VK_AMD_shader_trinary_minmax', 1, True), ] diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index c6828329fdf..08abb9dbc47 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -63,10 +63,25 @@ struct radv_blend_state { uint32_t cb_shader_mask; uint32_t db_alpha_to_mask; + uint32_t commutative_4bit; + bool single_cb_enable; bool mrt0_is_dual_src; }; +struct radv_dsa_order_invariance { + /* Whether the final result in Z/S buffers is guaranteed to be + * invariant under changes to the order in which fragments arrive. + */ + bool zs; + + /* Whether the set of fragments that pass the combined Z/S test is + * guaranteed to be invariant under changes to the order in which + * fragments arrive. + */ + bool pass_set; +}; + struct radv_tessellation_state { uint32_t ls_hs_config; unsigned num_patches; @@ -530,6 +545,40 @@ radv_pipeline_compute_get_int_clamp(const VkGraphicsPipelineCreateInfo *pCreateI } } +static void +radv_blend_check_commutativity(struct radv_blend_state *blend, + VkBlendOp op, VkBlendFactor src, + VkBlendFactor dst, unsigned chanmask) +{ + /* Src factor is allowed when it does not depend on Dst. */ + static const uint32_t src_allowed = + (1u << VK_BLEND_FACTOR_ONE) | + (1u << VK_BLEND_FACTOR_SRC_COLOR) | + (1u << VK_BLEND_FACTOR_SRC_ALPHA) | + (1u << VK_BLEND_FACTOR_SRC_ALPHA_SATURATE) | + (1u << VK_BLEND_FACTOR_CONSTANT_COLOR) | + (1u << VK_BLEND_FACTOR_CONSTANT_ALPHA) | + (1u << VK_BLEND_FACTOR_SRC1_COLOR) | + (1u << VK_BLEND_FACTOR_SRC1_ALPHA) | + (1u << VK_BLEND_FACTOR_ZERO) | + (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR) | + (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA) | + (1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR) | + (1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA) | + (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR) | + (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA); + + if (dst == VK_BLEND_FACTOR_ONE && + (src_allowed && (1u << src))) { + /* Addition is commutative, but floating point addition isn't + * associative: subtle changes can be introduced via different + * rounding. Be conservative, only enable for min and max. + */ + if (op == VK_BLEND_OP_MAX || op == VK_BLEND_OP_MIN) + blend->commutative_4bit |= chanmask; + } +} + static struct radv_blend_state radv_pipeline_init_blend_state(struct radv_pipeline *pipeline, const VkGraphicsPipelineCreateInfo *pCreateInfo, @@ -600,6 +649,11 @@ radv_pipeline_init_blend_state(struct radv_pipeline *pipeline, dstA = VK_BLEND_FACTOR_ONE; } + radv_blend_check_commutativity(&blend, eqRGB, srcRGB, dstRGB, + 0x7 << (4 * i)); + radv_blend_check_commutativity(&blend, eqA, srcA, dstA, + 0x8 << (4 * i)); + /* Blending optimizations for RB+. * These transformations don't change the behavior. * @@ -750,13 +804,182 @@ static uint8_t radv_pipeline_get_ps_iter_samples(const VkPipelineMultisampleStat return ps_iter_samples; } +static bool +radv_is_depth_write_enabled(const VkPipelineDepthStencilStateCreateInfo *pCreateInfo) +{ + return pCreateInfo->depthTestEnable && + pCreateInfo->depthWriteEnable && + pCreateInfo->depthCompareOp != VK_COMPARE_OP_NEVER; +} + +static bool +radv_writes_stencil(const VkStencilOpState *state) +{ + return state->writeMask && + (state->failOp != VK_STENCIL_OP_KEEP || + state->passOp != VK_STENCIL_OP_KEEP || + state->depthFailOp != VK_STENCIL_OP_KEEP); +} + +static bool +radv_is_stencil_write_enabled(const VkPipelineDepthStencilStateCreateInfo *pCreateInfo) +{ + return pCreateInfo->stencilTestEnable && + (radv_writes_stencil(&pCreateInfo->front) || + radv_writes_stencil(&pCreateInfo->back)); +} + +static bool +radv_is_ds_write_enabled(const VkPipelineDepthStencilStateCreateInfo *pCreateInfo) +{ + return radv_is_depth_write_enabled(pCreateInfo) || + radv_is_stencil_write_enabled(pCreateInfo); +} + +static bool +radv_order_invariant_stencil_op(VkStencilOp op) +{ + /* REPLACE is normally order invariant, except when the stencil + * reference value is written by the fragment shader. Tracking this + * interaction does not seem worth the effort, so be conservative. + */ + return op != VK_STENCIL_OP_INCREMENT_AND_CLAMP && + op != VK_STENCIL_OP_DECREMENT_AND_CLAMP && + op != VK_STENCIL_OP_REPLACE; +} + +static bool +radv_order_invariant_stencil_state(const VkStencilOpState *state) +{ + /* Compute whether, assuming Z writes are disabled, this stencil state + * is order invariant in the sense that the set of passing fragments as + * well as the final stencil buffer result does not depend on the order + * of fragments. + */ + return !state->writeMask || + /* The following assumes that Z writes are disabled. */ + (state->compareOp == VK_COMPARE_OP_ALWAYS && + radv_order_invariant_stencil_op(state->passOp) && + radv_order_invariant_stencil_op(state->depthFailOp)) || + (state->compareOp == VK_COMPARE_OP_NEVER && + radv_order_invariant_stencil_op(state->failOp)); +} + +static bool +radv_pipeline_out_of_order_rast(struct radv_pipeline *pipeline, + struct radv_blend_state *blend, + const VkGraphicsPipelineCreateInfo *pCreateInfo) +{ + RADV_FROM_HANDLE(radv_render_pass, pass, pCreateInfo->renderPass); + struct radv_subpass *subpass = pass->subpasses + pCreateInfo->subpass; + unsigned colormask = blend->cb_target_enabled_4bit; + + if (!pipeline->device->physical_device->out_of_order_rast_allowed) + return false; + + /* Be conservative if a logic operation is enabled with color buffers. */ + if (colormask && pCreateInfo->pColorBlendState->logicOpEnable) + return false; + + /* Default depth/stencil invariance when no attachment is bound. */ + struct radv_dsa_order_invariance dsa_order_invariant = { + .zs = true, .pass_set = true + }; + + if (pCreateInfo->pDepthStencilState && + subpass->depth_stencil_attachment.attachment != VK_ATTACHMENT_UNUSED) { + const VkPipelineDepthStencilStateCreateInfo *vkds = + pCreateInfo->pDepthStencilState; + struct radv_render_pass_attachment *attachment = + pass->attachments + subpass->depth_stencil_attachment.attachment; + bool has_stencil = vk_format_is_stencil(attachment->format); + struct radv_dsa_order_invariance order_invariance[2]; + struct radv_shader_variant *ps = + pipeline->shaders[MESA_SHADER_FRAGMENT]; + + /* Compute depth/stencil order invariance in order to know if + * it's safe to enable out-of-order. + */ + bool zfunc_is_ordered = + vkds->depthCompareOp == VK_COMPARE_OP_NEVER || + vkds->depthCompareOp == VK_COMPARE_OP_LESS || + vkds->depthCompareOp == VK_COMPARE_OP_LESS_OR_EQUAL || + vkds->depthCompareOp == VK_COMPARE_OP_GREATER || + vkds->depthCompareOp == VK_COMPARE_OP_GREATER_OR_EQUAL; + + bool nozwrite_and_order_invariant_stencil = + !radv_is_ds_write_enabled(vkds) || + (!radv_is_depth_write_enabled(vkds) && + radv_order_invariant_stencil_state(&vkds->front) && + radv_order_invariant_stencil_state(&vkds->back)); + + order_invariance[1].zs = + nozwrite_and_order_invariant_stencil || + (!radv_is_stencil_write_enabled(vkds) && + zfunc_is_ordered); + order_invariance[0].zs = + !radv_is_depth_write_enabled(vkds) || zfunc_is_ordered; + + order_invariance[1].pass_set = + nozwrite_and_order_invariant_stencil || + (!radv_is_stencil_write_enabled(vkds) && + (vkds->depthCompareOp == VK_COMPARE_OP_ALWAYS || + vkds->depthCompareOp == VK_COMPARE_OP_NEVER)); + order_invariance[0].pass_set = + !radv_is_depth_write_enabled(vkds) || + (vkds->depthCompareOp == VK_COMPARE_OP_ALWAYS || + vkds->depthCompareOp == VK_COMPARE_OP_NEVER); + + dsa_order_invariant = order_invariance[has_stencil]; + if (!dsa_order_invariant.zs) + return false; + + /* The set of PS invocations is always order invariant, + * except when early Z/S tests are requested. + */ + if (ps && + ps->info.info.ps.writes_memory && + ps->info.fs.early_fragment_test && + !dsa_order_invariant.pass_set) + return false; + + /* Determine if out-of-order rasterization should be disabled + * when occlusion queries are used. + */ + pipeline->graphics.disable_out_of_order_rast_for_occlusion = + !dsa_order_invariant.pass_set; + } + + /* No color buffers are enabled for writing. */ + if (!colormask) + return true; + + unsigned blendmask = colormask & blend->blend_enable_4bit; + + if (blendmask) { + /* Only commutative blending. */ + if (blendmask & ~blend->commutative_4bit) + return false; + + if (!dsa_order_invariant.pass_set) + return false; + } + + if (colormask & ~blendmask) + return false; + + return true; +} + static void radv_pipeline_init_multisample_state(struct radv_pipeline *pipeline, + struct radv_blend_state *blend, const VkGraphicsPipelineCreateInfo *pCreateInfo) { const VkPipelineMultisampleStateCreateInfo *vkms = pCreateInfo->pMultisampleState; struct radv_multisample_state *ms = &pipeline->graphics.ms; unsigned num_tile_pipes = pipeline->device->physical_device->rad_info.num_tile_pipes; + bool out_of_order_rast = false; int ps_iter_samples = 1; uint32_t mask = 0xffff; @@ -808,8 +1031,21 @@ radv_pipeline_init_multisample_state(struct radv_pipeline *pipeline, const struct VkPipelineRasterizationStateRasterizationOrderAMD *raster_order = vk_find_struct_const(pCreateInfo->pRasterizationState->pNext, PIPELINE_RASTERIZATION_STATE_RASTERIZATION_ORDER_AMD); if (raster_order && raster_order->rasterizationOrder == VK_RASTERIZATION_ORDER_RELAXED_AMD) { + /* Out-of-order rasterization is explicitly enabled by the + * application. + */ + out_of_order_rast = true; + } else { + /* Determine if the driver can enable out-of-order + * rasterization internally. + */ + out_of_order_rast = + radv_pipeline_out_of_order_rast(pipeline, blend, pCreateInfo); + } + + if (out_of_order_rast) { ms->pa_sc_mode_cntl_1 |= S_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(1) | - S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7); + S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7); } if (vkms && vkms->pSampleMask) { @@ -3094,7 +3330,7 @@ radv_pipeline_init(struct radv_pipeline *pipeline, pStages); pipeline->graphics.spi_baryc_cntl = S_0286E0_FRONT_FACE_ALL_BITS(1); - radv_pipeline_init_multisample_state(pipeline, pCreateInfo); + radv_pipeline_init_multisample_state(pipeline, &blend, pCreateInfo); uint32_t gs_out; uint32_t prim = si_translate_prim(pCreateInfo->pInputAssemblyState->topology); diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index a5cbad6cfe0..4485efaa097 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -291,6 +291,9 @@ struct radv_physical_device { bool cpdma_prefetch_writes_memory; bool has_scissor_bug; + bool has_out_of_order_rast; + bool out_of_order_rast_allowed; + /* This is the drivers on-disk cache used as a fallback as opposed to * the pipeline cache defined by apps. */ @@ -1232,6 +1235,7 @@ struct radv_pipeline { struct radv_prim_vertex_count prim_vertex_count; bool can_use_guardband; uint32_t needed_dynamic_state; + bool disable_out_of_order_rast_for_occlusion; } graphics; }; -- 2.30.2