From 69014d8c94f99fbf0de26f3a76426a92762821a0 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Mon, 15 Jun 2020 16:34:52 -0400 Subject: [PATCH] radeonsi: implement CP register shadowing Acked-by: Pierre-Eric Pelloux-Prayer Part-of: --- src/gallium/drivers/radeonsi/Makefile.sources | 1 + src/gallium/drivers/radeonsi/meson.build | 1 + src/gallium/drivers/radeonsi/si_build_pm4.h | 7 + .../drivers/radeonsi/si_cp_reg_shadowing.c | 189 ++++++++++++++++++ src/gallium/drivers/radeonsi/si_descriptors.c | 7 + src/gallium/drivers/radeonsi/si_gfx_cs.c | 19 +- src/gallium/drivers/radeonsi/si_pipe.c | 8 +- src/gallium/drivers/radeonsi/si_pipe.h | 5 + src/gallium/drivers/radeonsi/si_state.c | 16 +- src/gallium/drivers/radeonsi/si_state.h | 2 +- src/gallium/drivers/radeonsi/si_state_draw.c | 7 +- 11 files changed, 245 insertions(+), 17 deletions(-) create mode 100644 src/gallium/drivers/radeonsi/si_cp_reg_shadowing.c diff --git a/src/gallium/drivers/radeonsi/Makefile.sources b/src/gallium/drivers/radeonsi/Makefile.sources index 13a89d68695..0ea232edff5 100644 --- a/src/gallium/drivers/radeonsi/Makefile.sources +++ b/src/gallium/drivers/radeonsi/Makefile.sources @@ -16,6 +16,7 @@ C_SOURCES := \ si_compute.h \ si_compute_blit.c \ si_cp_dma.c \ + si_cp_reg_shadowing.c \ si_debug.c \ si_descriptors.c \ si_dma_cs.c \ diff --git a/src/gallium/drivers/radeonsi/meson.build b/src/gallium/drivers/radeonsi/meson.build index b30b857e39c..b62647df3fa 100644 --- a/src/gallium/drivers/radeonsi/meson.build +++ b/src/gallium/drivers/radeonsi/meson.build @@ -32,6 +32,7 @@ files_libradeonsi = files( 'si_compute.h', 'si_compute_blit.c', 'si_cp_dma.c', + 'si_cp_reg_shadowing.c', 'si_debug.c', 'si_descriptors.c', 'si_dma_cs.c', diff --git a/src/gallium/drivers/radeonsi/si_build_pm4.h b/src/gallium/drivers/radeonsi/si_build_pm4.h index 8a9b6ea5e34..f7f0e26d7d0 100644 --- a/src/gallium/drivers/radeonsi/si_build_pm4.h +++ b/src/gallium/drivers/radeonsi/si_build_pm4.h @@ -60,6 +60,13 @@ static inline void radeon_set_context_reg(struct radeon_cmdbuf *cs, unsigned reg radeon_emit(cs, value); } +static inline void radeon_set_context_reg_seq_array(struct radeon_cmdbuf *cs, unsigned reg, + unsigned num, const uint32_t *values) +{ + radeon_set_context_reg_seq(cs, reg, num); + radeon_emit_array(cs, values, num); +} + static inline void radeon_set_context_reg_idx(struct radeon_cmdbuf *cs, unsigned reg, unsigned idx, unsigned value) { diff --git a/src/gallium/drivers/radeonsi/si_cp_reg_shadowing.c b/src/gallium/drivers/radeonsi/si_cp_reg_shadowing.c new file mode 100644 index 00000000000..ba03250b31e --- /dev/null +++ b/src/gallium/drivers/radeonsi/si_cp_reg_shadowing.c @@ -0,0 +1,189 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "si_build_pm4.h" +#include "ac_shadowed_regs.h" +#include "util/u_memory.h" + +static void si_build_load_reg(struct si_screen *sscreen, struct si_pm4_state *pm4, + enum ac_reg_range_type type, + struct si_resource *shadow_regs) +{ + uint64_t gpu_address = shadow_regs->gpu_address; + unsigned packet, num_ranges, offset; + const struct ac_reg_range *ranges; + + ac_get_reg_ranges(sscreen->info.chip_class, sscreen->info.family, + type, &num_ranges, &ranges); + + switch (type) { + case SI_REG_RANGE_UCONFIG: + gpu_address += SI_SHADOWED_UCONFIG_REG_OFFSET; + offset = CIK_UCONFIG_REG_OFFSET; + packet = PKT3_LOAD_UCONFIG_REG; + break; + case SI_REG_RANGE_CONTEXT: + gpu_address += SI_SHADOWED_CONTEXT_REG_OFFSET; + offset = SI_CONTEXT_REG_OFFSET; + packet = PKT3_LOAD_CONTEXT_REG; + break; + default: + gpu_address += SI_SHADOWED_SH_REG_OFFSET; + offset = SI_SH_REG_OFFSET; + packet = PKT3_LOAD_SH_REG; + break; + } + + si_pm4_cmd_add(pm4, PKT3(packet, 1 + num_ranges * 2, 0)); + si_pm4_cmd_add(pm4, gpu_address); + si_pm4_cmd_add(pm4, gpu_address >> 32); + for (unsigned i = 0; i < num_ranges; i++) { + si_pm4_cmd_add(pm4, (ranges[i].offset - offset) / 4); + si_pm4_cmd_add(pm4, ranges[i].size / 4); + } +} + +static struct si_pm4_state * +si_create_shadowing_ib_preamble(struct si_context *sctx) +{ + struct si_pm4_state *pm4 = CALLOC_STRUCT(si_pm4_state); + + if (sctx->chip_class == GFX10) { + /* SQ_NON_EVENT must be emitted before GE_PC_ALLOC is written. */ + si_pm4_cmd_add(pm4, PKT3(PKT3_EVENT_WRITE, 0, 0)); + si_pm4_cmd_add(pm4, EVENT_TYPE(V_028A90_SQ_NON_EVENT) | EVENT_INDEX(0)); + } + + if (sctx->screen->dpbb_allowed) { + si_pm4_cmd_add(pm4, PKT3(PKT3_EVENT_WRITE, 0, 0)); + si_pm4_cmd_add(pm4, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0)); + } + + /* Wait for idle, because we'll update VGT ring pointers. */ + si_pm4_cmd_add(pm4, PKT3(PKT3_EVENT_WRITE, 0, 0)); + si_pm4_cmd_add(pm4, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4)); + + /* VGT_FLUSH is required even if VGT is idle. It resets VGT pointers. */ + si_pm4_cmd_add(pm4, PKT3(PKT3_EVENT_WRITE, 0, 0)); + si_pm4_cmd_add(pm4, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0)); + + if (sctx->chip_class >= GFX10) { + unsigned gcr_cntl = S_586_GL2_INV(1) | S_586_GL2_WB(1) | + S_586_GLM_INV(1) | S_586_GLM_WB(1) | + S_586_GL1_INV(1) | S_586_GLV_INV(1) | + S_586_GLK_INV(1) | S_586_GLI_INV(V_586_GLI_ALL); + + si_pm4_cmd_add(pm4, PKT3(PKT3_ACQUIRE_MEM, 6, 0)); + si_pm4_cmd_add(pm4, 0); /* CP_COHER_CNTL */ + si_pm4_cmd_add(pm4, 0xffffffff); /* CP_COHER_SIZE */ + si_pm4_cmd_add(pm4, 0xffffff); /* CP_COHER_SIZE_HI */ + si_pm4_cmd_add(pm4, 0); /* CP_COHER_BASE */ + si_pm4_cmd_add(pm4, 0); /* CP_COHER_BASE_HI */ + si_pm4_cmd_add(pm4, 0x0000000A); /* POLL_INTERVAL */ + si_pm4_cmd_add(pm4, gcr_cntl); /* GCR_CNTL */ + } else if (sctx->chip_class == GFX9) { + unsigned cp_coher_cntl = S_0301F0_SH_ICACHE_ACTION_ENA(1) | + S_0301F0_SH_KCACHE_ACTION_ENA(1) | + S_0301F0_TC_ACTION_ENA(1) | + S_0301F0_TCL1_ACTION_ENA(1) | + S_0301F0_TC_WB_ACTION_ENA(1); + + si_pm4_cmd_add(pm4, PKT3(PKT3_ACQUIRE_MEM, 5, 0)); + si_pm4_cmd_add(pm4, cp_coher_cntl); /* CP_COHER_CNTL */ + si_pm4_cmd_add(pm4, 0xffffffff); /* CP_COHER_SIZE */ + si_pm4_cmd_add(pm4, 0xffffff); /* CP_COHER_SIZE_HI */ + si_pm4_cmd_add(pm4, 0); /* CP_COHER_BASE */ + si_pm4_cmd_add(pm4, 0); /* CP_COHER_BASE_HI */ + si_pm4_cmd_add(pm4, 0x0000000A); /* POLL_INTERVAL */ + } else { + unreachable("invalid chip"); + } + + si_pm4_cmd_add(pm4, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + si_pm4_cmd_add(pm4, 0); + + si_pm4_cmd_add(pm4, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); + si_pm4_cmd_add(pm4, + CC0_UPDATE_LOAD_ENABLES(1) | + CC0_LOAD_PER_CONTEXT_STATE(1) | + CC0_LOAD_CS_SH_REGS(1) | + CC0_LOAD_GFX_SH_REGS(1) | + CC0_LOAD_GLOBAL_UCONFIG(1)); + si_pm4_cmd_add(pm4, + CC1_UPDATE_SHADOW_ENABLES(1) | + CC1_SHADOW_PER_CONTEXT_STATE(1) | + CC1_SHADOW_CS_SH_REGS(1) | + CC1_SHADOW_GFX_SH_REGS(1) | + CC1_SHADOW_GLOBAL_UCONFIG(1)); + + for (unsigned i = 0; i < SI_NUM_SHADOWED_REG_RANGES; i++) + si_build_load_reg(sctx->screen, pm4, i, sctx->shadowed_regs); + + return pm4; +} + +void si_init_cp_reg_shadowing(struct si_context *sctx) +{ + if (sctx->screen->debug_flags & DBG(SHADOW_REGS)) { + sctx->shadowed_regs = + si_aligned_buffer_create(sctx->b.screen, + SI_RESOURCE_FLAG_UNMAPPABLE, + PIPE_USAGE_DEFAULT, + SI_SHADOWED_REG_BUFFER_SIZE, + 4096); + if (!sctx->shadowed_regs) + fprintf(stderr, "radeonsi: cannot create a shadowed_regs buffer\n"); + } + + si_init_cs_preamble_state(sctx, sctx->shadowed_regs != NULL); + + if (sctx->shadowed_regs) { + /* We need to clear the shadowed reg buffer. */ + si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, &sctx->shadowed_regs->b.b, + 0, sctx->shadowed_regs->bo_size, 0, 0, SI_COHERENCY_CP, + L2_BYPASS); + + /* Create the shadowing preamble. */ + struct si_pm4_state *shadowing_preamble = + si_create_shadowing_ib_preamble(sctx); + + /* Initialize shadowed registers as follows. */ + radeon_add_to_buffer_list(sctx, sctx->gfx_cs, sctx->shadowed_regs, + RADEON_USAGE_READWRITE, RADEON_PRIO_DESCRIPTORS); + si_pm4_emit(sctx, shadowing_preamble); + ac_emulate_clear_state(&sctx->screen->info, sctx->gfx_cs, + radeon_set_context_reg_seq_array); + si_pm4_emit(sctx, sctx->cs_preamble_state); + + /* The register values are shadowed, so we won't need to set them again. */ + si_pm4_free_state(sctx, sctx->cs_preamble_state, ~0); + + /* Execute the shadowing preamble as cs_preamble, which will + * load register values from memory. + */ + sctx->cs_preamble_state = shadowing_preamble; + + si_set_tracked_regs_to_clear_state(sctx); + } +} diff --git a/src/gallium/drivers/radeonsi/si_descriptors.c b/src/gallium/drivers/radeonsi/si_descriptors.c index 096234b3efb..ce5eba575b7 100644 --- a/src/gallium/drivers/radeonsi/si_descriptors.c +++ b/src/gallium/drivers/radeonsi/si_descriptors.c @@ -2086,6 +2086,13 @@ static void si_emit_global_shader_pointers(struct si_context *sctx, struct si_de si_emit_shader_pointer(sctx, descs, R_00B230_SPI_SHADER_USER_DATA_GS_0); si_emit_shader_pointer(sctx, descs, R_00B430_SPI_SHADER_USER_DATA_HS_0); return; + } else if (sctx->chip_class == GFX9 && sctx->shadowed_regs) { + /* We can't use the COMMON registers with register shadowing. */ + si_emit_shader_pointer(sctx, descs, R_00B030_SPI_SHADER_USER_DATA_PS_0); + si_emit_shader_pointer(sctx, descs, R_00B130_SPI_SHADER_USER_DATA_VS_0); + si_emit_shader_pointer(sctx, descs, R_00B330_SPI_SHADER_USER_DATA_ES_0); + si_emit_shader_pointer(sctx, descs, R_00B430_SPI_SHADER_USER_DATA_LS_0); + return; } else if (sctx->chip_class == GFX9) { /* Broadcast it to all shader stages. */ si_emit_shader_pointer(sctx, descs, R_00B530_SPI_SHADER_USER_DATA_COMMON_0); diff --git a/src/gallium/drivers/radeonsi/si_gfx_cs.c b/src/gallium/drivers/radeonsi/si_gfx_cs.c index fe175769f80..59f6c0141ca 100644 --- a/src/gallium/drivers/radeonsi/si_gfx_cs.c +++ b/src/gallium/drivers/radeonsi/si_gfx_cs.c @@ -405,6 +405,11 @@ void si_begin_new_gfx_cs(struct si_context *ctx) radeon_add_to_buffer_list(ctx, ctx->gfx_cs, ctx->border_color_buffer, RADEON_USAGE_READ, RADEON_PRIO_BORDER_COLORS); + if (ctx->shadowed_regs) { + radeon_add_to_buffer_list(ctx, ctx->gfx_cs, ctx->shadowed_regs, + RADEON_USAGE_READWRITE, + RADEON_PRIO_DESCRIPTORS); + } ctx->cs_shader_state.initialized = false; si_add_all_descriptors_to_bo_list(ctx); @@ -536,12 +541,14 @@ void si_begin_new_gfx_cs(struct si_context *ctx) ctx->index_ring_offset = 0; - if (has_clear_state) { - si_set_tracked_regs_to_clear_state(ctx); - } else { - /* Set all register values to unknown. */ - ctx->tracked_regs.reg_saved = 0; - ctx->last_gs_out_prim = -1; /* unknown */ + if (!ctx->shadowed_regs) { + if (has_clear_state) { + si_set_tracked_regs_to_clear_state(ctx); + } else { + /* Set all register values to unknown. */ + ctx->tracked_regs.reg_saved = 0; + ctx->last_gs_out_prim = -1; /* unknown */ + } } /* 0xffffffff is a impossible value to register SPI_PS_INPUT_CNTL_n */ diff --git a/src/gallium/drivers/radeonsi/si_pipe.c b/src/gallium/drivers/radeonsi/si_pipe.c index e366b6b5458..0e1cde2e25e 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.c +++ b/src/gallium/drivers/radeonsi/si_pipe.c @@ -90,6 +90,7 @@ static const struct debug_named_value debug_options[] = { {"check_vm", DBG(CHECK_VM), "Check VM faults and dump debug info."}, {"reserve_vmid", DBG(RESERVE_VMID), "Force VMID reservation per context."}, {"zerovram", DBG(ZERO_VRAM), "Clear VRAM allocations."}, + {"shadowregs", DBG(SHADOW_REGS), "Enable CP register shadowing."}, /* 3D engine options: */ {"nogfx", DBG(NO_GFX), "Disable graphics. Only multimedia compute paths can be used."}, @@ -297,6 +298,7 @@ static void si_destroy_context(struct pipe_context *context) si_resource_reference(&sctx->index_ring, NULL); si_resource_reference(&sctx->barrier_buf, NULL); si_resource_reference(&sctx->last_ib_barrier_buf, NULL); + si_resource_reference(&sctx->shadowed_regs, NULL); pb_reference(&sctx->gds, NULL); pb_reference(&sctx->gds_oa, NULL); @@ -552,7 +554,6 @@ static struct pipe_context *si_create_context(struct pipe_screen *screen, unsign si_init_msaa_functions(sctx); si_init_shader_functions(sctx); si_init_state_functions(sctx); - si_init_cs_preamble_state(sctx); si_init_streamout_functions(sctx); si_init_viewport_functions(sctx); @@ -669,6 +670,11 @@ static struct pipe_context *si_create_context(struct pipe_screen *screen, unsign /* The remainder of this function initializes the gfx CS and must be last. */ assert(sctx->gfx_cs->current.cdw == 0); + + if (sctx->has_graphics) { + si_init_cp_reg_shadowing(sctx); + } + si_begin_new_gfx_cs(sctx); assert(sctx->gfx_cs->current.cdw == sctx->initial_gfx_cs_size); diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h index 14b29db9b85..c59bbd5463b 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.h +++ b/src/gallium/drivers/radeonsi/si_pipe.h @@ -184,6 +184,7 @@ enum DBG_CHECK_VM, DBG_RESERVE_VMID, DBG_ZERO_VRAM, + DBG_SHADOW_REGS, /* 3D engine options: */ DBG_NO_GFX, @@ -912,6 +913,7 @@ struct si_context { struct u_log_context *log; void *query_result_shader; void *sh_query_result_shader; + struct si_resource *shadowed_regs; void (*emit_cache_flush)(struct si_context *ctx); @@ -1368,6 +1370,9 @@ void si_cp_copy_data(struct si_context *sctx, struct radeon_cmdbuf *cs, unsigned struct si_resource *dst, unsigned dst_offset, unsigned src_sel, struct si_resource *src, unsigned src_offset); +/* si_cp_reg_shadowing.c */ +void si_init_cp_reg_shadowing(struct si_context *sctx); + /* si_debug.c */ void si_save_cs(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, struct radeon_saved_cs *saved, bool get_buffer_list); diff --git a/src/gallium/drivers/radeonsi/si_state.c b/src/gallium/drivers/radeonsi/si_state.c index 40995c5e5c0..4ebcf5088a6 100644 --- a/src/gallium/drivers/radeonsi/si_state.c +++ b/src/gallium/drivers/radeonsi/si_state.c @@ -5108,7 +5108,7 @@ static void si_set_raster_config(struct si_context *sctx, struct si_pm4_state *p } } -void si_init_cs_preamble_state(struct si_context *sctx) +void si_init_cs_preamble_state(struct si_context *sctx, bool uses_reg_shadowing) { struct si_screen *sscreen = sctx->screen; uint64_t border_color_va = sctx->border_color_buffer->gpu_address; @@ -5118,13 +5118,15 @@ void si_init_cs_preamble_state(struct si_context *sctx) if (!pm4) return; - si_pm4_cmd_add(pm4, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); - si_pm4_cmd_add(pm4, CC0_UPDATE_LOAD_ENABLES(1)); - si_pm4_cmd_add(pm4, CC1_UPDATE_SHADOW_ENABLES(1)); + if (!uses_reg_shadowing) { + si_pm4_cmd_add(pm4, PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); + si_pm4_cmd_add(pm4, CC0_UPDATE_LOAD_ENABLES(1)); + si_pm4_cmd_add(pm4, CC1_UPDATE_SHADOW_ENABLES(1)); - if (has_clear_state) { - si_pm4_cmd_add(pm4, PKT3(PKT3_CLEAR_STATE, 0, 0)); - si_pm4_cmd_add(pm4, 0); + if (has_clear_state) { + si_pm4_cmd_add(pm4, PKT3(PKT3_CLEAR_STATE, 0, 0)); + si_pm4_cmd_add(pm4, 0); + } } /* CLEAR_STATE doesn't restore these correctly. */ diff --git a/src/gallium/drivers/radeonsi/si_state.h b/src/gallium/drivers/radeonsi/si_state.h index 62202de2cdb..911d3c90a62 100644 --- a/src/gallium/drivers/radeonsi/si_state.h +++ b/src/gallium/drivers/radeonsi/si_state.h @@ -527,7 +527,7 @@ void si_rebind_buffer(struct si_context *sctx, struct pipe_resource *buf); void si_init_state_compute_functions(struct si_context *sctx); void si_init_state_functions(struct si_context *sctx); void si_init_screen_state_functions(struct si_screen *sscreen); -void si_init_cs_preamble_state(struct si_context *sctx); +void si_init_cs_preamble_state(struct si_context *sctx, bool uses_reg_shadowing); void si_make_buffer_descriptor(struct si_screen *screen, struct si_resource *buf, enum pipe_format format, unsigned offset, unsigned size, uint32_t *state); diff --git a/src/gallium/drivers/radeonsi/si_state_draw.c b/src/gallium/drivers/radeonsi/si_state_draw.c index b123e610ffe..b6b789da0c2 100644 --- a/src/gallium/drivers/radeonsi/si_state_draw.c +++ b/src/gallium/drivers/radeonsi/si_state_draw.c @@ -761,7 +761,8 @@ static void si_emit_draw_packets(struct si_context *sctx, const struct pipe_draw /* draw packet */ if (index_size) { - if (index_size != sctx->last_index_size) { + /* Register shadowing doesn't shadow INDEX_TYPE. */ + if (index_size != sctx->last_index_size || sctx->shadowed_regs) { unsigned index_type; /* index type */ @@ -880,7 +881,9 @@ static void si_emit_draw_packets(struct si_context *sctx, const struct pipe_draw } else { int base_vertex; - if (sctx->last_instance_count == SI_INSTANCE_COUNT_UNKNOWN || + /* Register shadowing requires that we always emit PKT3_NUM_INSTANCES. */ + if (sctx->shadowed_regs || + sctx->last_instance_count == SI_INSTANCE_COUNT_UNKNOWN || sctx->last_instance_count != instance_count) { radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, 0)); radeon_emit(cs, instance_count); -- 2.30.2