From 79d4d2807ff60d571c377c7f4aa729601ab873e1 Mon Sep 17 00:00:00 2001 From: Samuel Pitoiset Date: Fri, 28 Feb 2020 17:55:54 +0100 Subject: [PATCH] radv/sqtt: add support for GFX10 All SQTT registers were moved to privileged space on GFX10, to emit them we need a workaround with COPY_DATA. Signed-off-by: Samuel Pitoiset Reviewed-by: Bas Nieuwenhuizen Tested-by: Marge Bot Part-of: --- src/amd/vulkan/radv_cs.h | 16 ++ src/amd/vulkan/radv_device.c | 8 +- src/amd/vulkan/radv_private.h | 5 +- src/amd/vulkan/radv_sqtt.c | 313 ++++++++++++++++++++++++---------- 4 files changed, 243 insertions(+), 99 deletions(-) diff --git a/src/amd/vulkan/radv_cs.h b/src/amd/vulkan/radv_cs.h index 16649c55e6a..1c89f1f8395 100644 --- a/src/amd/vulkan/radv_cs.h +++ b/src/amd/vulkan/radv_cs.h @@ -149,4 +149,20 @@ static inline void radeon_set_uconfig_reg_idx(const struct radv_physical_device radeon_emit(cs, value); } +static inline void radeon_set_privileged_config_reg(struct radeon_cmdbuf *cs, + unsigned reg, + unsigned value) +{ + assert(reg < CIK_UCONFIG_REG_OFFSET); + assert(cs->cdw + 6 <= cs->max_dw); + + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | + COPY_DATA_DST_SEL(COPY_DATA_PERF)); + radeon_emit(cs, value); + radeon_emit(cs, 0); /* unused */ + radeon_emit(cs, reg >> 2); + radeon_emit(cs, 0); /* unused */ +} + #endif /* RADV_CS_H */ diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index 12b326d37dc..0a5db57e715 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -2971,12 +2971,12 @@ VkResult radv_CreateDevice( int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1); if (radv_thread_trace >= 0) { - fprintf(stderr, "*****************************************************************************\n"); - fprintf(stderr, "* WARNING: Thread trace support is experimental and only supported on GFX9! *\n"); - fprintf(stderr, "*****************************************************************************\n"); + fprintf(stderr, "******************************************************************************\n"); + fprintf(stderr, "* WARNING: Thread trace support is experimental and only supported on GFX9+! *\n"); + fprintf(stderr, "******************************************************************************\n"); /* TODO: add support for more ASICs. */ - assert(device->physical_device->rad_info.chip_class == GFX9); + assert(device->physical_device->rad_info.chip_class >= GFX9); /* Default buffer size set to 1MB per SE. */ device->thread_trace_buffer_size = diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index 1f776c8f580..f9efa5b0a81 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -2375,7 +2375,10 @@ void radv_nir_shader_info_init(struct radv_shader_info *info); struct radv_thread_trace_info { uint32_t cur_offset; uint32_t trace_status; - uint32_t write_counter; + union { + uint32_t gfx9_write_counter; + uint32_t gfx10_dropped_cntr; + }; }; struct radv_thread_trace_se { diff --git a/src/amd/vulkan/radv_sqtt.c b/src/amd/vulkan/radv_sqtt.c index 324d0f897ab..c04a79f2842 100644 --- a/src/amd/vulkan/radv_sqtt.c +++ b/src/amd/vulkan/radv_sqtt.c @@ -69,7 +69,7 @@ radv_emit_thread_trace_start(struct radv_device *device, uint32_t shifted_size = device->thread_trace_buffer_size >> SQTT_BUFFER_ALIGN_SHIFT; unsigned max_se = device->physical_device->rad_info.max_se; - assert(device->physical_device->rad_info.chip_class == GFX9); + assert(device->physical_device->rad_info.chip_class >= GFX9); for (unsigned se = 0; se < max_se; se++) { uint64_t data_va = radv_thread_trace_get_data_va(device, se); @@ -81,61 +81,99 @@ radv_emit_thread_trace_start(struct radv_device *device, S_030800_SH_INDEX(0) | S_030800_INSTANCE_BROADCAST_WRITES(1)); - /* Order seems important for the following 4 registers. */ - radeon_set_uconfig_reg(cs, R_030CDC_SQ_THREAD_TRACE_BASE2, - S_030CDC_ADDR_HI(shifted_va >> 32)); - - radeon_set_uconfig_reg(cs, R_030CC0_SQ_THREAD_TRACE_BASE, - S_030CC0_ADDR(shifted_va)); - - radeon_set_uconfig_reg(cs, R_030CC4_SQ_THREAD_TRACE_SIZE, - S_030CC4_SIZE(shifted_size)); - - radeon_set_uconfig_reg(cs, R_030CD4_SQ_THREAD_TRACE_CTRL, - S_030CD4_RESET_BUFFER(1)); - - radeon_set_uconfig_reg(cs, R_030CC8_SQ_THREAD_TRACE_MASK, - S_030CC8_CU_SEL(2) | - S_030CC8_SH_SEL(0) | - S_030CC8_SIMD_EN(0xf) | - S_030CC8_VM_ID_MASK(0) | - S_030CC8_REG_STALL_EN(1) | - S_030CC8_SPI_STALL_EN(1) | - S_030CC8_SQ_STALL_EN(1)); - - /* Trace all tokens and registers. */ - radeon_set_uconfig_reg(cs, R_030CCC_SQ_THREAD_TRACE_TOKEN_MASK, - S_030CCC_TOKEN_MASK(0xbfff) | - S_030CCC_REG_MASK(0xff) | - S_030CCC_REG_DROP_ON_STALL(0)); - - /* Enable SQTT perf counters for all CUs. */ - radeon_set_uconfig_reg(cs, R_030CD0_SQ_THREAD_TRACE_PERF_MASK, - S_030CD0_SH0_MASK(0xffff) | - S_030CD0_SH1_MASK(0xffff)); - - radeon_set_uconfig_reg(cs, R_030CE0_SQ_THREAD_TRACE_TOKEN_MASK2, - S_030CE0_INST_MASK(0xffffffff)); - - radeon_set_uconfig_reg(cs, R_030CEC_SQ_THREAD_TRACE_HIWATER, - S_030CEC_HIWATER(4)); - - /* Reset thread trace status errors. */ - radeon_set_uconfig_reg(cs, R_030CE8_SQ_THREAD_TRACE_STATUS, - S_030CE8_UTC_ERROR(0)); - - /* Enable the thread trace mode. */ - radeon_set_uconfig_reg(cs, R_030CD8_SQ_THREAD_TRACE_MODE, - S_030CD8_MASK_PS(1) | - S_030CD8_MASK_VS(1) | - S_030CD8_MASK_GS(1) | - S_030CD8_MASK_ES(1) | - S_030CD8_MASK_HS(1) | - S_030CD8_MASK_LS(1) | - S_030CD8_MASK_CS(1) | - S_030CD8_AUTOFLUSH_EN(1) | /* periodically flush SQTT data to memory */ - S_030CD8_TC_PERF_EN(1) | /* count SQTT traffic in TCC perf counters */ - S_030CD8_MODE(1)); + if (device->physical_device->rad_info.chip_class == GFX9) { + /* Order seems important for the following 4 registers. */ + radeon_set_uconfig_reg(cs, R_030CDC_SQ_THREAD_TRACE_BASE2, + S_030CDC_ADDR_HI(shifted_va >> 32)); + + radeon_set_uconfig_reg(cs, R_030CC0_SQ_THREAD_TRACE_BASE, + S_030CC0_ADDR(shifted_va)); + + radeon_set_uconfig_reg(cs, R_030CC4_SQ_THREAD_TRACE_SIZE, + S_030CC4_SIZE(shifted_size)); + + radeon_set_uconfig_reg(cs, R_030CD4_SQ_THREAD_TRACE_CTRL, + S_030CD4_RESET_BUFFER(1)); + + radeon_set_uconfig_reg(cs, R_030CC8_SQ_THREAD_TRACE_MASK, + S_030CC8_CU_SEL(2) | + S_030CC8_SH_SEL(0) | + S_030CC8_SIMD_EN(0xf) | + S_030CC8_VM_ID_MASK(0) | + S_030CC8_REG_STALL_EN(1) | + S_030CC8_SPI_STALL_EN(1) | + S_030CC8_SQ_STALL_EN(1)); + + /* Trace all tokens and registers. */ + radeon_set_uconfig_reg(cs, R_030CCC_SQ_THREAD_TRACE_TOKEN_MASK, + S_030CCC_TOKEN_MASK(0xbfff) | + S_030CCC_REG_MASK(0xff) | + S_030CCC_REG_DROP_ON_STALL(0)); + + /* Enable SQTT perf counters for all CUs. */ + radeon_set_uconfig_reg(cs, R_030CD0_SQ_THREAD_TRACE_PERF_MASK, + S_030CD0_SH0_MASK(0xffff) | + S_030CD0_SH1_MASK(0xffff)); + + radeon_set_uconfig_reg(cs, R_030CE0_SQ_THREAD_TRACE_TOKEN_MASK2, + S_030CE0_INST_MASK(0xffffffff)); + + radeon_set_uconfig_reg(cs, R_030CEC_SQ_THREAD_TRACE_HIWATER, + S_030CEC_HIWATER(4)); + + /* Reset thread trace status errors. */ + radeon_set_uconfig_reg(cs, R_030CE8_SQ_THREAD_TRACE_STATUS, + S_030CE8_UTC_ERROR(0)); + + /* Enable the thread trace mode. */ + radeon_set_uconfig_reg(cs, R_030CD8_SQ_THREAD_TRACE_MODE, + S_030CD8_MASK_PS(1) | + S_030CD8_MASK_VS(1) | + S_030CD8_MASK_GS(1) | + S_030CD8_MASK_ES(1) | + S_030CD8_MASK_HS(1) | + S_030CD8_MASK_LS(1) | + S_030CD8_MASK_CS(1) | + S_030CD8_AUTOFLUSH_EN(1) | /* periodically flush SQTT data to memory */ + S_030CD8_TC_PERF_EN(1) | /* count SQTT traffic in TCC perf counters */ + S_030CD8_MODE(1)); + } else { + /* Order seems important for the following 2 registers. */ + radeon_set_privileged_config_reg(cs, R_008D04_SQ_THREAD_TRACE_BUF0_SIZE, + S_008D04_SIZE(shifted_size) | + S_008D04_BASE_HI(shifted_va >> 32)); + + radeon_set_privileged_config_reg(cs, R_008D00_SQ_THREAD_TRACE_BUF0_BASE, + S_008D00_BASE_LO(shifted_va)); + + radeon_set_privileged_config_reg(cs, R_008D14_SQ_THREAD_TRACE_MASK, + S_008D14_WTYPE_INCLUDE(0x7f) | /* all shader stages */ + S_008D14_SA_SEL(0) | + S_008D14_WGP_SEL(0) | + S_008D14_SIMD_SEL(0)); + + radeon_set_privileged_config_reg(cs, R_008D18_SQ_THREAD_TRACE_TOKEN_MASK, + S_008D18_REG_INCLUDE(V_008D18_REG_INCLUDE_SQDEC | + V_008D18_REG_INCLUDE_SHDEC | + V_008D18_REG_INCLUDE_GFXUDEC | + V_008D18_REG_INCLUDE_CONTEXT | + V_008D18_REG_INCLUDE_COMP | + V_008D18_REG_INCLUDE_CONTEXT | + V_008D18_REG_INCLUDE_CONFIG) | + S_008D18_TOKEN_EXCLUDE(V_008D18_TOKEN_EXCLUDE_PERF)); + + /* Should be emitted last (it enables thread traces). */ + radeon_set_privileged_config_reg(cs, R_008D1C_SQ_THREAD_TRACE_CTRL, + S_008D1C_MODE(1) | + S_008D1C_HIWATER(5) | + S_008D1C_UTIL_TIMER(1) | + S_008D1C_RT_FREQ(2) | /* 4096 clk */ + S_008D1C_DRAW_EVENT_EN(1) | + S_008D1C_REG_STALL_EN(1) | + S_008D1C_SPI_STALL_EN(1) | + S_008D1C_SQ_STALL_EN(1) | + S_008D1C_REG_DROP_ON_STALL(0)); + } } /* Restore global broadcasting. */ @@ -155,13 +193,20 @@ radv_emit_thread_trace_start(struct radv_device *device, } } -static const uint32_t thread_trace_info_regs[] = +static const uint32_t gfx9_thread_trace_info_regs[] = { R_030CE4_SQ_THREAD_TRACE_WPTR, R_030CE8_SQ_THREAD_TRACE_STATUS, R_030CF0_SQ_THREAD_TRACE_CNTR, }; +static const uint32_t gfx10_thread_trace_info_regs[] = +{ + R_008D10_SQ_THREAD_TRACE_WPTR, + R_008D20_SQ_THREAD_TRACE_STATUS, + R_008D24_SQ_THREAD_TRACE_DROPPED_CNTR, +}; + static void radv_emit_thread_trace_stop(struct radv_device *device, struct radeon_cmdbuf *cs, @@ -169,7 +214,7 @@ radv_emit_thread_trace_stop(struct radv_device *device, { unsigned max_se = device->physical_device->rad_info.max_se; - assert(device->physical_device->rad_info.chip_class == GFX9); + assert(device->physical_device->rad_info.chip_class >= GFX9); /* Stop the thread trace with a different event based on the queue. */ if (queue_family_index == RADV_QUEUE_COMPUTE && @@ -191,32 +236,73 @@ radv_emit_thread_trace_stop(struct radv_device *device, S_030800_SH_INDEX(0) | S_030800_INSTANCE_BROADCAST_WRITES(1)); - /* Disable the thread trace mode. */ - radeon_set_uconfig_reg(cs, R_030CD8_SQ_THREAD_TRACE_MODE, - S_030CD8_MODE(0)); - - /* Wait for thread trace completion. */ - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ - radeon_emit(cs, R_030CE8_SQ_THREAD_TRACE_STATUS >> 2); /* register */ - radeon_emit(cs, 0); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, S_030CE8_BUSY(1)); /* mask */ - radeon_emit(cs, 4); /* poll interval */ - - /* Get the VA where the info struct is stored for this SE. */ - uint64_t info_va = radv_thread_trace_get_info_va(device, se); - - /* Copy back the info struct one DWORD at a time. */ - for (unsigned i = 0; i < ARRAY_SIZE(thread_trace_info_regs); i++) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | - COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | - COPY_DATA_WR_CONFIRM); - radeon_emit(cs, thread_trace_info_regs[i] >> 2); - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, (info_va + i * 4)); - radeon_emit(cs, (info_va + i * 4) >> 32); + if (device->physical_device->rad_info.chip_class == GFX9) { + /* Disable the thread trace mode. */ + radeon_set_uconfig_reg(cs, R_030CD8_SQ_THREAD_TRACE_MODE, + S_030CD8_MODE(0)); + + /* Wait for thread trace completion. */ + radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); + radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ + radeon_emit(cs, R_030CE8_SQ_THREAD_TRACE_STATUS >> 2); /* register */ + radeon_emit(cs, 0); + radeon_emit(cs, 0); /* reference value */ + radeon_emit(cs, S_030CE8_BUSY(1)); /* mask */ + radeon_emit(cs, 4); /* poll interval */ + + /* Get the VA where the info struct is stored for this SE. */ + uint64_t info_va = radv_thread_trace_get_info_va(device, se); + + /* Copy back the info struct one DWORD at a time. */ + for (unsigned i = 0; i < ARRAY_SIZE(gfx9_thread_trace_info_regs); i++) { + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | + COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | + COPY_DATA_WR_CONFIRM); + radeon_emit(cs, gfx9_thread_trace_info_regs[i] >> 2); + radeon_emit(cs, 0); /* unused */ + radeon_emit(cs, (info_va + i * 4)); + radeon_emit(cs, (info_va + i * 4) >> 32); + } + } else { + assert(device->physical_device->rad_info.chip_class == GFX10); + + /* Make sure to wait for the trace buffer. */ + radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); + radeon_emit(cs, WAIT_REG_MEM_NOT_EQUAL); /* wait until the register is equal to the reference value */ + radeon_emit(cs, R_008D20_SQ_THREAD_TRACE_STATUS >> 2); /* register */ + radeon_emit(cs, 0); + radeon_emit(cs, 0); /* reference value */ + radeon_emit(cs, S_008D20_FINISH_DONE(1)); /* mask */ + radeon_emit(cs, 4); /* poll interval */ + + /* Disable the thread trace mode. */ + radeon_set_privileged_config_reg(cs, R_008D1C_SQ_THREAD_TRACE_CTRL, + S_008D1C_MODE(0)); + + /* Wait for thread trace completion. */ + radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); + radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */ + radeon_emit(cs, R_008D20_SQ_THREAD_TRACE_STATUS >> 2); /* register */ + radeon_emit(cs, 0); + radeon_emit(cs, 0); /* reference value */ + radeon_emit(cs, S_008D20_BUSY(1)); /* mask */ + radeon_emit(cs, 4); /* poll interval */ + + /* Get the VA where the info struct is stored for this SE. */ + uint64_t info_va = radv_thread_trace_get_info_va(device, se); + + /* Copy back the info struct one DWORD at a time. */ + for (unsigned i = 0; i < ARRAY_SIZE(gfx10_thread_trace_info_regs); i++) { + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | + COPY_DATA_DST_SEL(COPY_DATA_TC_L2) | + COPY_DATA_WR_CONFIRM); + radeon_emit(cs, gfx10_thread_trace_info_regs[i] >> 2); + radeon_emit(cs, 0); /* unused */ + radeon_emit(cs, (info_va + i * 4)); + radeon_emit(cs, (info_va + i * 4) >> 32); + } } } @@ -228,13 +314,18 @@ radv_emit_thread_trace_stop(struct radv_device *device, } static void -radv_emit_spi_config_cntl(struct radeon_cmdbuf *cs, bool enable) +radv_emit_spi_config_cntl(struct radv_device *device, + struct radeon_cmdbuf *cs, bool enable) { - radeon_set_uconfig_reg(cs, R_031100_SPI_CONFIG_CNTL, - S_031100_GPR_WRITE_PRIORITY(0x2c688) | - S_031100_EXP_PRIORITY_ORDER(3) | - S_031100_ENABLE_SQG_TOP_EVENTS(enable) | - S_031100_ENABLE_SQG_BOP_EVENTS(enable)); + uint32_t spi_config_cntl = S_031100_GPR_WRITE_PRIORITY(0x2c688) | + S_031100_EXP_PRIORITY_ORDER(3) | + S_031100_ENABLE_SQG_TOP_EVENTS(enable) | + S_031100_ENABLE_SQG_BOP_EVENTS(enable); + + if (device->physical_device->rad_info.chip_class == GFX10) + spi_config_cntl |= S_031100_PS_PKR_PRIORITY_CNTL(3); + + radeon_set_uconfig_reg(cs, R_031100_SPI_CONFIG_CNTL, spi_config_cntl); } static void @@ -283,7 +374,9 @@ radv_thread_trace_init_cs(struct radv_device *device) family); /* Enable SQG events that collects thread trace data. */ - radv_emit_spi_config_cntl(device->thread_trace_start_cs[family], true); + radv_emit_spi_config_cntl(device, + device->thread_trace_start_cs[family], + true); radv_emit_thread_trace_start(device, device->thread_trace_start_cs[family], @@ -320,7 +413,9 @@ radv_thread_trace_init_cs(struct radv_device *device) family); /* Restore previous state by disabling SQG events. */ - radv_emit_spi_config_cntl(device->thread_trace_stop_cs[family], false); + radv_emit_spi_config_cntl(device, + device->thread_trace_stop_cs[family], + false); ws->cs_finalize(device->thread_trace_stop_cs[family]); } @@ -395,6 +490,36 @@ radv_end_thread_trace(struct radv_queue *queue) return radv_queue_internal_submit(queue, cs); } +static bool +radv_is_thread_trace_complete(struct radv_device *device, + const struct radv_thread_trace_info *info) +{ + if (device->physical_device->rad_info.chip_class == GFX10) { + /* GFX10 doesn't have THREAD_TRACE_CNTR but it reports the + * number of dropped bytes for all SEs via + * THREAD_TRACE_DROPPED_CNTR. + */ + return info->gfx10_dropped_cntr == 0; + } + + /* Otherwise, compare the current thread trace offset with the number + * of written bytes. + */ + return info->cur_offset < info->gfx9_write_counter; +} + +static uint32_t +radv_get_expected_buffer_size(struct radv_device *device, + const struct radv_thread_trace_info *info) +{ + if (device->physical_device->rad_info.chip_class == GFX10) { + uint32_t dropped_cntr_per_se = info->gfx10_dropped_cntr / device->physical_device->rad_info.max_se; + return ((info->cur_offset * 32) + dropped_cntr_per_se) / 1024; + } + + return (info->gfx9_write_counter * 32) / 1024; +} + bool radv_get_thread_trace(struct radv_queue *queue, struct radv_thread_trace *thread_trace) @@ -415,9 +540,9 @@ radv_get_thread_trace(struct radv_queue *queue, (struct radv_thread_trace_info *)info_ptr; struct radv_thread_trace_se thread_trace_se = {}; - if (info->cur_offset < info->write_counter) { + if (!radv_is_thread_trace_complete(device, info)) { uint32_t expected_size = - (info->write_counter * 32) / 1024; + radv_get_expected_buffer_size(device, info); uint32_t available_size = (info->cur_offset * 32) / 1024; -- 2.30.2