X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_perfcounter.c;h=d2f0db81d2c88ac2e3d1866786d25101fe88e5cf;hb=e2e700f6053d0b16ba46e4d5c5b20e965fb2224e;hp=24855e4e6f22b387d11e3169cb43262e94563edb;hpb=156e81f305b19f62b4d6ea98bf44a951e89dc947;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/si_perfcounter.c b/src/gallium/drivers/radeonsi/si_perfcounter.c index 24855e4e6f2..d2f0db81d2c 100644 --- a/src/gallium/drivers/radeonsi/si_perfcounter.c +++ b/src/gallium/drivers/radeonsi/si_perfcounter.c @@ -1,5 +1,6 @@ /* * Copyright 2015 Advanced Micro Devices, Inc. + * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -19,326 +20,569 @@ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * Authors: - * Nicolai Hähnle - * */ -#include "radeon/r600_cs.h" -#include "radeon/r600_query.h" -#include "radeon/r600_pipe_common.h" +#include "si_build_pm4.h" +#include "si_query.h" #include "util/u_memory.h" -#include "si_pipe.h" -#include "sid.h" - -enum si_pc_reg_layout { - /* All secondary selector dwords follow as one block after the primary - * selector dwords for the counters that have secondary selectors. - */ - SI_PC_MULTI_BLOCK = 0, - - /* Each secondary selector dword follows immediately afters the - * corresponding primary. - */ - SI_PC_MULTI_ALTERNATE = 1, +enum si_pc_block_flags +{ + /* This block is part of the shader engine */ + SI_PC_BLOCK_SE = (1 << 0), - /* All secondary selector dwords follow as one block after all primary - * selector dwords. - */ - SI_PC_MULTI_TAIL = 2, + /* Expose per-instance groups instead of summing all instances (within + * an SE). */ + SI_PC_BLOCK_INSTANCE_GROUPS = (1 << 1), - /* Free-form arrangement of selector registers. */ - SI_PC_MULTI_CUSTOM = 3, + /* Expose per-SE groups instead of summing instances across SEs. */ + SI_PC_BLOCK_SE_GROUPS = (1 << 2), - SI_PC_MULTI_MASK = 3, + /* Shader block */ + SI_PC_BLOCK_SHADER = (1 << 3), - /* Registers are laid out in decreasing rather than increasing order. */ - SI_PC_REG_REVERSE = 4, + /* Non-shader block with perfcounters windowed by shaders. */ + SI_PC_BLOCK_SHADER_WINDOWED = (1 << 4), +}; - SI_PC_FAKE = 8, +enum si_pc_reg_layout +{ + /* All secondary selector dwords follow as one block after the primary + * selector dwords for the counters that have secondary selectors. + * + * Example: + * PERFCOUNTER0_SELECT + * PERFCOUNTER1_SELECT + * PERFCOUNTER0_SELECT1 + * PERFCOUNTER1_SELECT1 + * PERFCOUNTER2_SELECT + * PERFCOUNTER3_SELECT + */ + SI_PC_MULTI_BLOCK = 0, + + /* Each secondary selector dword follows immediately after the + * corresponding primary. + * + * Example: + * PERFCOUNTER0_SELECT + * PERFCOUNTER0_SELECT1 + * PERFCOUNTER1_SELECT + * PERFCOUNTER1_SELECT1 + * PERFCOUNTER2_SELECT + * PERFCOUNTER3_SELECT + */ + SI_PC_MULTI_ALTERNATE = 1, + + /* All secondary selector dwords follow as one block after all primary + * selector dwords. + * + * Example: + * PERFCOUNTER0_SELECT + * PERFCOUNTER1_SELECT + * PERFCOUNTER2_SELECT + * PERFCOUNTER3_SELECT + * PERFCOUNTER0_SELECT1 + * PERFCOUNTER1_SELECT1 + */ + SI_PC_MULTI_TAIL = 2, + + /* Free-form arrangement of selector registers. */ + SI_PC_MULTI_CUSTOM = 3, + + SI_PC_MULTI_MASK = 3, + + /* Registers are laid out in decreasing rather than increasing order. */ + SI_PC_REG_REVERSE = 4, + + SI_PC_FAKE = 8, }; struct si_pc_block_base { - const char *name; - unsigned num_counters; - unsigned flags; + const char *name; + unsigned num_counters; + unsigned flags; + + unsigned select_or; + unsigned select0; + unsigned counter0_lo; + unsigned *select; + unsigned *counters; + unsigned num_multi; + unsigned num_prelude; + unsigned layout; +}; - unsigned select_or; - unsigned select0; - unsigned counter0_lo; - unsigned *select; - unsigned *counters; - unsigned num_multi; - unsigned num_prelude; - unsigned layout; +struct si_pc_block_gfxdescr { + struct si_pc_block_base *b; + unsigned selectors; + unsigned instances; }; struct si_pc_block { - struct si_pc_block_base *b; - unsigned selectors; - unsigned instances; + const struct si_pc_block_gfxdescr *b; + unsigned num_instances; + + unsigned num_groups; + char *group_names; + unsigned group_name_stride; + + char *selector_names; + unsigned selector_name_stride; }; /* The order is chosen to be compatible with GPUPerfStudio's hardcoding of * performance counter group IDs. */ -static const char * const si_pc_shader_type_suffixes[] = { - "", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS" -}; +static const char *const si_pc_shader_type_suffixes[] = {"", "_ES", "_GS", "_VS", + "_PS", "_LS", "_HS", "_CS"}; static const unsigned si_pc_shader_type_bits[] = { - 0x7f, - S_036780_ES_EN(1), - S_036780_GS_EN(1), - S_036780_VS_EN(1), - S_036780_PS_EN(1), - S_036780_LS_EN(1), - S_036780_HS_EN(1), - S_036780_CS_EN(1), + 0x7f, + S_036780_ES_EN(1), + S_036780_GS_EN(1), + S_036780_VS_EN(1), + S_036780_PS_EN(1), + S_036780_LS_EN(1), + S_036780_HS_EN(1), + S_036780_CS_EN(1), }; -static struct si_pc_block_base cik_CB = { - .name = "CB", - .num_counters = 4, - .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS, +/* Max counters per HW block */ +#define SI_QUERY_MAX_COUNTERS 16 - .select0 = R_037000_CB_PERFCOUNTER_FILTER, - .counter0_lo = R_035018_CB_PERFCOUNTER0_LO, - .num_multi = 1, - .num_prelude = 1, - .layout = SI_PC_MULTI_ALTERNATE, +#define SI_PC_SHADERS_WINDOWING (1u << 31) + +struct si_query_group { + struct si_query_group *next; + struct si_pc_block *block; + unsigned sub_gid; /* only used during init */ + unsigned result_base; /* only used during init */ + int se; + int instance; + unsigned num_counters; + unsigned selectors[SI_QUERY_MAX_COUNTERS]; +}; + +struct si_query_counter { + unsigned base; + unsigned qwords; + unsigned stride; /* in uint64s */ +}; + +struct si_query_pc { + struct si_query b; + struct si_query_buffer buffer; + + /* Size of the results in memory, in bytes. */ + unsigned result_size; + + unsigned shaders; + unsigned num_counters; + struct si_query_counter *counters; + struct si_query_group *groups; +}; + +static struct si_pc_block_base cik_CB = { + .name = "CB", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_037000_CB_PERFCOUNTER_FILTER, + .counter0_lo = R_035018_CB_PERFCOUNTER0_LO, + .num_multi = 1, + .num_prelude = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; static unsigned cik_CPC_select[] = { - R_036024_CPC_PERFCOUNTER0_SELECT, - R_036010_CPC_PERFCOUNTER0_SELECT1, - R_03600C_CPC_PERFCOUNTER1_SELECT, + R_036024_CPC_PERFCOUNTER0_SELECT, + R_036010_CPC_PERFCOUNTER0_SELECT1, + R_03600C_CPC_PERFCOUNTER1_SELECT, }; static struct si_pc_block_base cik_CPC = { - .name = "CPC", - .num_counters = 2, + .name = "CPC", + .num_counters = 2, - .select = cik_CPC_select, - .counter0_lo = R_034018_CPC_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE, + .select = cik_CPC_select, + .counter0_lo = R_034018_CPC_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE, }; static struct si_pc_block_base cik_CPF = { - .name = "CPF", - .num_counters = 2, + .name = "CPF", + .num_counters = 2, - .select0 = R_03601C_CPF_PERFCOUNTER0_SELECT, - .counter0_lo = R_034028_CPF_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, + .select0 = R_03601C_CPF_PERFCOUNTER0_SELECT, + .counter0_lo = R_034028_CPF_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, }; static struct si_pc_block_base cik_CPG = { - .name = "CPG", - .num_counters = 2, + .name = "CPG", + .num_counters = 2, - .select0 = R_036008_CPG_PERFCOUNTER0_SELECT, - .counter0_lo = R_034008_CPG_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, + .select0 = R_036008_CPG_PERFCOUNTER0_SELECT, + .counter0_lo = R_034008_CPG_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, }; static struct si_pc_block_base cik_DB = { - .name = "DB", - .num_counters = 4, - .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_037100_DB_PERFCOUNTER0_SELECT, - .counter0_lo = R_035100_DB_PERFCOUNTER0_LO, - .num_multi = 3, // really only 2, but there's a gap between registers - .layout = SI_PC_MULTI_ALTERNATE, + .name = "DB", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_037100_DB_PERFCOUNTER0_SELECT, + .counter0_lo = R_035100_DB_PERFCOUNTER0_LO, + .num_multi = 3, // really only 2, but there's a gap between registers + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_GDS = { - .name = "GDS", - .num_counters = 4, + .name = "GDS", + .num_counters = 4, - .select0 = R_036A00_GDS_PERFCOUNTER0_SELECT, - .counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_TAIL, + .select0 = R_036A00_GDS_PERFCOUNTER0_SELECT, + .counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_TAIL, }; static unsigned cik_GRBM_counters[] = { - R_034100_GRBM_PERFCOUNTER0_LO, - R_03410C_GRBM_PERFCOUNTER1_LO, + R_034100_GRBM_PERFCOUNTER0_LO, + R_03410C_GRBM_PERFCOUNTER1_LO, }; static struct si_pc_block_base cik_GRBM = { - .name = "GRBM", - .num_counters = 2, + .name = "GRBM", + .num_counters = 2, - .select0 = R_036100_GRBM_PERFCOUNTER0_SELECT, - .counters = cik_GRBM_counters, + .select0 = R_036100_GRBM_PERFCOUNTER0_SELECT, + .counters = cik_GRBM_counters, }; static struct si_pc_block_base cik_GRBMSE = { - .name = "GRBMSE", - .num_counters = 4, + .name = "GRBMSE", + .num_counters = 4, - .select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT, - .counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO, + .select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT, + .counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO, }; static struct si_pc_block_base cik_IA = { - .name = "IA", - .num_counters = 4, + .name = "IA", + .num_counters = 4, - .select0 = R_036210_IA_PERFCOUNTER0_SELECT, - .counter0_lo = R_034220_IA_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_TAIL, + .select0 = R_036210_IA_PERFCOUNTER0_SELECT, + .counter0_lo = R_034220_IA_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_TAIL, }; static struct si_pc_block_base cik_PA_SC = { - .name = "PA_SC", - .num_counters = 8, - .flags = R600_PC_BLOCK_SE, - - .select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT, - .counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "PA_SC", + .num_counters = 8, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT, + .counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; +/* According to docs, PA_SU counters are only 48 bits wide. */ static struct si_pc_block_base cik_PA_SU = { - .name = "PA_SU", - .num_counters = 4, - .flags = R600_PC_BLOCK_SE, - - .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT, - .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "PA_SU", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT, + .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_SPI = { - .name = "SPI", - .num_counters = 6, - .flags = R600_PC_BLOCK_SE, - - .select0 = R_036600_SPI_PERFCOUNTER0_SELECT, - .counter0_lo = R_034604_SPI_PERFCOUNTER0_LO, - .num_multi = 4, - .layout = SI_PC_MULTI_BLOCK, + .name = "SPI", + .num_counters = 6, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036600_SPI_PERFCOUNTER0_SELECT, + .counter0_lo = R_034604_SPI_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_BLOCK, }; static struct si_pc_block_base cik_SQ = { - .name = "SQ", - .num_counters = 16, - .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_SHADER, + .name = "SQ", + .num_counters = 16, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER, - .select0 = R_036700_SQ_PERFCOUNTER0_SELECT, - .select_or = S_036700_SQC_BANK_MASK(15) | - S_036700_SQC_CLIENT_MASK(15) | - S_036700_SIMD_MASK(15), - .counter0_lo = R_034700_SQ_PERFCOUNTER0_LO, + .select0 = R_036700_SQ_PERFCOUNTER0_SELECT, + .select_or = S_036700_SQC_BANK_MASK(15) | S_036700_SQC_CLIENT_MASK(15) | S_036700_SIMD_MASK(15), + .counter0_lo = R_034700_SQ_PERFCOUNTER0_LO, }; static struct si_pc_block_base cik_SX = { - .name = "SX", - .num_counters = 4, - .flags = R600_PC_BLOCK_SE, - - .select0 = R_036900_SX_PERFCOUNTER0_SELECT, - .counter0_lo = R_034900_SX_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_TAIL, + .name = "SX", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036900_SX_PERFCOUNTER0_SELECT, + .counter0_lo = R_034900_SX_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_TAIL, }; static struct si_pc_block_base cik_TA = { - .name = "TA", - .num_counters = 2, - .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED, - - .select0 = R_036B00_TA_PERFCOUNTER0_SELECT, - .counter0_lo = R_034B00_TA_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TA", + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036B00_TA_PERFCOUNTER0_SELECT, + .counter0_lo = R_034B00_TA_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TD = { - .name = "TD", - .num_counters = 2, - .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED, - - .select0 = R_036C00_TD_PERFCOUNTER0_SELECT, - .counter0_lo = R_034C00_TD_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TD", + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036C00_TD_PERFCOUNTER0_SELECT, + .counter0_lo = R_034C00_TD_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TCA = { - .name = "TCA", - .num_counters = 4, - .flags = R600_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_036E40_TCA_PERFCOUNTER0_SELECT, - .counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TCA", + .num_counters = 4, + .flags = SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_036E40_TCA_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TCC = { - .name = "TCC", - .num_counters = 4, - .flags = R600_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_036E00_TCC_PERFCOUNTER0_SELECT, - .counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TCC", + .num_counters = 4, + .flags = SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_036E00_TCC_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TCP = { - .name = "TCP", - .num_counters = 4, - .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED, - - .select0 = R_036D00_TCP_PERFCOUNTER0_SELECT, - .counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TCP", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036D00_TCP_PERFCOUNTER0_SELECT, + .counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_VGT = { - .name = "VGT", - .num_counters = 4, - .flags = R600_PC_BLOCK_SE, - - .select0 = R_036230_VGT_PERFCOUNTER0_SELECT, - .counter0_lo = R_034240_VGT_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_TAIL, + .name = "VGT", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036230_VGT_PERFCOUNTER0_SELECT, + .counter0_lo = R_034240_VGT_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_TAIL, }; static struct si_pc_block_base cik_WD = { - .name = "WD", - .num_counters = 4, + .name = "WD", + .num_counters = 4, - .select0 = R_036200_WD_PERFCOUNTER0_SELECT, - .counter0_lo = R_034200_WD_PERFCOUNTER0_LO, + .select0 = R_036200_WD_PERFCOUNTER0_SELECT, + .counter0_lo = R_034200_WD_PERFCOUNTER0_LO, }; static struct si_pc_block_base cik_MC = { - .name = "MC", - .num_counters = 4, + .name = "MC", + .num_counters = 4, - .layout = SI_PC_FAKE, + .layout = SI_PC_FAKE, }; static struct si_pc_block_base cik_SRBM = { - .name = "SRBM", - .num_counters = 2, + .name = "SRBM", + .num_counters = 2, - .layout = SI_PC_FAKE, + .layout = SI_PC_FAKE, +}; + +static struct si_pc_block_base gfx10_CHA = { + .name = "CHA", + .num_counters = 4, + + .select0 = R_037780_CHA_PERFCOUNTER0_SELECT, + .counter0_lo = R_035800_CHA_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_CHCG = { + .name = "CHCG", + .num_counters = 4, + + .select0 = R_036F18_CHCG_PERFCOUNTER0_SELECT, + .counter0_lo = R_034F20_CHCG_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_CHC = { + .name = "CHC", + .num_counters = 4, + + .select0 = R_036F00_CHC_PERFCOUNTER0_SELECT, + .counter0_lo = R_034F00_CHC_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GCR = { + .name = "GCR", + .num_counters = 2, + + .select0 = R_037580_GCR_PERFCOUNTER0_SELECT, + .counter0_lo = R_035480_GCR_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GE = { + .name = "GE", + .num_counters = 12, + + .select0 = R_036200_GE_PERFCOUNTER0_SELECT, + .counter0_lo = R_034200_GE_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL1A = { + .name = "GL1A", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_037700_GL1A_PERFCOUNTER0_SELECT, + .counter0_lo = R_035700_GL1A_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL1C = { + .name = "GL1C", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036E80_GL1C_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E80_GL1C_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL2A = { + .name = "GL2A", + .num_counters = 4, + + .select0 = R_036E40_GL2A_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E40_GL2A_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL2C = { + .name = "GL2C", + .num_counters = 4, + + .select0 = R_036E00_GL2C_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E00_GL2C_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static unsigned gfx10_PA_PH_select[] = { + R_037600_PA_PH_PERFCOUNTER0_SELECT, + R_037604_PA_PH_PERFCOUNTER0_SELECT1, + R_037608_PA_PH_PERFCOUNTER1_SELECT, + R_037640_PA_PH_PERFCOUNTER1_SELECT1, + R_03760C_PA_PH_PERFCOUNTER2_SELECT, + R_037644_PA_PH_PERFCOUNTER2_SELECT1, + R_037610_PA_PH_PERFCOUNTER3_SELECT, + R_037648_PA_PH_PERFCOUNTER3_SELECT1, + R_037614_PA_PH_PERFCOUNTER4_SELECT, + R_037618_PA_PH_PERFCOUNTER5_SELECT, + R_03761C_PA_PH_PERFCOUNTER6_SELECT, + R_037620_PA_PH_PERFCOUNTER7_SELECT, +}; +static struct si_pc_block_base gfx10_PA_PH = { + .name = "PA_PH", + .num_counters = 8, + .flags = SI_PC_BLOCK_SE, + + .select = gfx10_PA_PH_select, + .counter0_lo = R_035600_PA_PH_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_CUSTOM, +}; + +static struct si_pc_block_base gfx10_PA_SU = { + .name = "PA_SU", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT, + .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_RLC = { + .name = "RLC", + .num_counters = 2, + + .select0 = R_037304_RLC_PERFCOUNTER0_SELECT, + .counter0_lo = R_035200_RLC_PERFCOUNTER0_LO, + .num_multi = 0, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_RMI = { + .name = "RMI", + /* Actually 4, but the 2nd counter is missing the secondary selector while + * the 3rd counter has it, which complicates the register layout. */ + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_037400_RMI_PERFCOUNTER0_SELECT, + .counter0_lo = R_035300_RMI_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_UTCL1 = { + .name = "UTCL1", + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_03758C_UTCL1_PERFCOUNTER0_SELECT, + .counter0_lo = R_035470_UTCL1_PERFCOUNTER0_LO, + .num_multi = 0, + .layout = SI_PC_MULTI_ALTERNATE, }; /* Both the number of instances and selectors varies between chips of the same @@ -349,411 +593,925 @@ static struct si_pc_block_base cik_SRBM = { * blindly once it believes it has identified the hardware, so the order of * blocks here matters. */ -static struct si_pc_block groups_CIK[] = { - { &cik_CB, 226, 4 }, - { &cik_CPF, 17 }, - { &cik_DB, 257, 4 }, - { &cik_GRBM, 34 }, - { &cik_GRBMSE, 15 }, - { &cik_PA_SU, 153 }, - { &cik_PA_SC, 395 }, - { &cik_SPI, 186 }, - { &cik_SQ, 252 }, - { &cik_SX, 32 }, - { &cik_TA, 111, 11 }, - { &cik_TCA, 39, 2 }, - { &cik_TCC, 160, 16 }, - { &cik_TD, 55, 11 }, - { &cik_TCP, 154, 11 }, - { &cik_GDS, 121 }, - { &cik_VGT, 140 }, - { &cik_IA, 22 }, - { &cik_MC, 22 }, - { &cik_SRBM, 19 }, - { &cik_WD, 22 }, - { &cik_CPG, 46 }, - { &cik_CPC, 22 }, - -}; - -static struct si_pc_block groups_VI[] = { - { &cik_CB, 396, 4 }, - { &cik_CPF, 19 }, - { &cik_DB, 257, 4 }, - { &cik_GRBM, 34 }, - { &cik_GRBMSE, 15 }, - { &cik_PA_SU, 153 }, - { &cik_PA_SC, 397 }, - { &cik_SPI, 197 }, - { &cik_SQ, 273 }, - { &cik_SX, 34 }, - { &cik_TA, 119, 16 }, - { &cik_TCA, 35, 2 }, - { &cik_TCC, 192, 16 }, - { &cik_TD, 55, 16 }, - { &cik_TCP, 180, 16 }, - { &cik_GDS, 121 }, - { &cik_VGT, 147 }, - { &cik_IA, 24 }, - { &cik_MC, 22 }, - { &cik_SRBM, 27 }, - { &cik_WD, 37 }, - { &cik_CPG, 48 }, - { &cik_CPC, 24 }, - -}; - -static void si_pc_get_size(struct r600_perfcounter_block *group, - unsigned count, unsigned *selectors, - unsigned *num_select_dw, unsigned *num_read_dw) +static struct si_pc_block_gfxdescr groups_CIK[] = { + {&cik_CB, 226}, {&cik_CPF, 17}, {&cik_DB, 257}, {&cik_GRBM, 34}, {&cik_GRBMSE, 15}, + {&cik_PA_SU, 153}, {&cik_PA_SC, 395}, {&cik_SPI, 186}, {&cik_SQ, 252}, {&cik_SX, 32}, + {&cik_TA, 111}, {&cik_TCA, 39, 2}, {&cik_TCC, 160}, {&cik_TD, 55}, {&cik_TCP, 154}, + {&cik_GDS, 121}, {&cik_VGT, 140}, {&cik_IA, 22}, {&cik_MC, 22}, {&cik_SRBM, 19}, + {&cik_WD, 22}, {&cik_CPG, 46}, {&cik_CPC, 22}, + +}; + +static struct si_pc_block_gfxdescr groups_VI[] = { + {&cik_CB, 405}, {&cik_CPF, 19}, {&cik_DB, 257}, {&cik_GRBM, 34}, {&cik_GRBMSE, 15}, + {&cik_PA_SU, 154}, {&cik_PA_SC, 397}, {&cik_SPI, 197}, {&cik_SQ, 273}, {&cik_SX, 34}, + {&cik_TA, 119}, {&cik_TCA, 35, 2}, {&cik_TCC, 192}, {&cik_TD, 55}, {&cik_TCP, 180}, + {&cik_GDS, 121}, {&cik_VGT, 147}, {&cik_IA, 24}, {&cik_MC, 22}, {&cik_SRBM, 27}, + {&cik_WD, 37}, {&cik_CPG, 48}, {&cik_CPC, 24}, + +}; + +static struct si_pc_block_gfxdescr groups_gfx9[] = { + {&cik_CB, 438}, {&cik_CPF, 32}, {&cik_DB, 328}, {&cik_GRBM, 38}, {&cik_GRBMSE, 16}, + {&cik_PA_SU, 292}, {&cik_PA_SC, 491}, {&cik_SPI, 196}, {&cik_SQ, 374}, {&cik_SX, 208}, + {&cik_TA, 119}, {&cik_TCA, 35, 2}, {&cik_TCC, 256}, {&cik_TD, 57}, {&cik_TCP, 85}, + {&cik_GDS, 121}, {&cik_VGT, 148}, {&cik_IA, 32}, {&cik_WD, 58}, {&cik_CPG, 59}, + {&cik_CPC, 35}, +}; + +static struct si_pc_block_gfxdescr groups_gfx10[] = { + {&cik_CB, 461}, + {&gfx10_CHA, 45}, + {&gfx10_CHCG, 35}, + {&gfx10_CHC, 35}, + {&cik_CPC, 47}, + {&cik_CPF, 40}, + {&cik_CPG, 82}, + {&cik_DB, 370}, + {&gfx10_GCR, 94}, + {&cik_GDS, 123}, + {&gfx10_GE, 315}, + {&gfx10_GL1A, 36}, + {&gfx10_GL1C, 64}, + {&gfx10_GL2A, 91}, + {&gfx10_GL2C, 235}, + {&cik_GRBM, 47}, + {&cik_GRBMSE, 19}, + {&gfx10_PA_PH, 960}, + {&cik_PA_SC, 552}, + {&gfx10_PA_SU, 266}, + {&gfx10_RLC, 7}, + {&gfx10_RMI, 258}, + {&cik_SPI, 329}, + {&cik_SQ, 509}, + {&cik_SX, 225}, + {&cik_TA, 226}, + {&cik_TCP, 77}, + {&cik_TD, 61}, + {&gfx10_UTCL1, 15}, +}; + +static bool si_pc_block_has_per_se_groups(const struct si_perfcounters *pc, + const struct si_pc_block *block) +{ + return block->b->b->flags & SI_PC_BLOCK_SE_GROUPS || + (block->b->b->flags & SI_PC_BLOCK_SE && pc->separate_se); +} + +static bool si_pc_block_has_per_instance_groups(const struct si_perfcounters *pc, + const struct si_pc_block *block) +{ + return block->b->b->flags & SI_PC_BLOCK_INSTANCE_GROUPS || + (block->num_instances > 1 && pc->separate_instance); +} + +static struct si_pc_block *lookup_counter(struct si_perfcounters *pc, unsigned index, + unsigned *base_gid, unsigned *sub_index) +{ + struct si_pc_block *block = pc->blocks; + unsigned bid; + + *base_gid = 0; + for (bid = 0; bid < pc->num_blocks; ++bid, ++block) { + unsigned total = block->num_groups * block->b->selectors; + + if (index < total) { + *sub_index = index; + return block; + } + + index -= total; + *base_gid += block->num_groups; + } + + return NULL; +} + +static struct si_pc_block *lookup_group(struct si_perfcounters *pc, unsigned *index) { - struct si_pc_block *sigroup = (struct si_pc_block *)group->data; - struct si_pc_block_base *regs = sigroup->b; - unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK; - - if (regs->layout & SI_PC_FAKE) { - *num_select_dw = 0; - } else if (layout_multi == SI_PC_MULTI_BLOCK) { - if (count < regs->num_multi) - *num_select_dw = 2 * (count + 2) + regs->num_prelude; - else - *num_select_dw = 2 + count + regs->num_multi + regs->num_prelude; - } else if (layout_multi == SI_PC_MULTI_TAIL) { - *num_select_dw = 4 + count + MIN2(count, regs->num_multi) + regs->num_prelude; - } else if (layout_multi == SI_PC_MULTI_CUSTOM) { - assert(regs->num_prelude == 0); - *num_select_dw = 3 * (count + MIN2(count, regs->num_multi)); - } else { - assert(layout_multi == SI_PC_MULTI_ALTERNATE); - - *num_select_dw = 2 + count + MIN2(count, regs->num_multi) + regs->num_prelude; - } - - *num_read_dw = 6 * count; + unsigned bid; + struct si_pc_block *block = pc->blocks; + + for (bid = 0; bid < pc->num_blocks; ++bid, ++block) { + if (*index < block->num_groups) + return block; + *index -= block->num_groups; + } + + return NULL; } -static void si_pc_emit_instance(struct r600_common_context *ctx, - int se, int instance) +static void si_pc_emit_instance(struct si_context *sctx, int se, int instance) { - struct radeon_winsys_cs *cs = ctx->gfx.cs; - unsigned value = S_030800_SH_BROADCAST_WRITES(1); - - if (se >= 0) { - value |= S_030800_SE_INDEX(se); - } else { - value |= S_030800_SE_BROADCAST_WRITES(1); - } - - if (instance >= 0) { - value |= S_030800_INSTANCE_INDEX(instance); - } else { - value |= S_030800_INSTANCE_BROADCAST_WRITES(1); - } - - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value); + struct radeon_cmdbuf *cs = sctx->gfx_cs; + unsigned value = S_030800_SH_BROADCAST_WRITES(1); + + if (se >= 0) { + value |= S_030800_SE_INDEX(se); + } else { + value |= S_030800_SE_BROADCAST_WRITES(1); + } + + if (sctx->chip_class >= GFX10) { + /* TODO: Expose counters from each shader array separately if needed. */ + value |= S_030800_SA_BROADCAST_WRITES(1); + } + + if (instance >= 0) { + value |= S_030800_INSTANCE_INDEX(instance); + } else { + value |= S_030800_INSTANCE_BROADCAST_WRITES(1); + } + + radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value); } -static void si_pc_emit_shaders(struct r600_common_context *ctx, - unsigned shaders) +static void si_pc_emit_shaders(struct si_context *sctx, unsigned shaders) { - struct radeon_winsys_cs *cs = ctx->gfx.cs; + struct radeon_cmdbuf *cs = sctx->gfx_cs; - radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2); - radeon_emit(cs, shaders & 0x7f); - radeon_emit(cs, 0xffffffff); + radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2); + radeon_emit(cs, shaders & 0x7f); + radeon_emit(cs, 0xffffffff); } -static void si_pc_emit_select(struct r600_common_context *ctx, - struct r600_perfcounter_block *group, - unsigned count, unsigned *selectors) +static void si_pc_emit_select(struct si_context *sctx, struct si_pc_block *block, unsigned count, + unsigned *selectors) { - struct si_pc_block *sigroup = (struct si_pc_block *)group->data; - struct si_pc_block_base *regs = sigroup->b; - struct radeon_winsys_cs *cs = ctx->gfx.cs; - unsigned idx; - unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK; - unsigned dw; - - assert(count <= regs->num_counters); - - if (regs->layout & SI_PC_FAKE) - return; - - if (layout_multi == SI_PC_MULTI_BLOCK) { - assert(!(regs->layout & SI_PC_REG_REVERSE)); - - dw = count + regs->num_prelude; - if (count >= regs->num_multi) - dw += regs->num_multi; - radeon_set_uconfig_reg_seq(cs, regs->select0, dw); - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) - radeon_emit(cs, selectors[idx] | regs->select_or); - - if (count < regs->num_multi) { - unsigned select1 = - regs->select0 + 4 * regs->num_multi; - radeon_set_uconfig_reg_seq(cs, select1, count); - } - - for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) - radeon_emit(cs, 0); - - if (count > regs->num_multi) { - for (idx = regs->num_multi; idx < count; ++idx) - radeon_emit(cs, selectors[idx] | regs->select_or); - } - } else if (layout_multi == SI_PC_MULTI_TAIL) { - unsigned select1, select1_count; - - assert(!(regs->layout & SI_PC_REG_REVERSE)); - - radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude); - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - for (idx = 0; idx < count; ++idx) - radeon_emit(cs, selectors[idx] | regs->select_or); - - select1 = regs->select0 + 4 * regs->num_counters; - select1_count = MIN2(count, regs->num_multi); - radeon_set_uconfig_reg_seq(cs, select1, select1_count); - for (idx = 0; idx < select1_count; ++idx) - radeon_emit(cs, 0); - } else if (layout_multi == SI_PC_MULTI_CUSTOM) { - unsigned *reg = regs->select; - for (idx = 0; idx < count; ++idx) { - radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or); - if (idx < regs->num_multi) - radeon_set_uconfig_reg(cs, *reg++, 0); - } - } else { - assert(layout_multi == SI_PC_MULTI_ALTERNATE); - - unsigned reg_base = regs->select0; - unsigned reg_count = count + MIN2(count, regs->num_multi); - reg_count += regs->num_prelude; - - if (!(regs->layout & SI_PC_REG_REVERSE)) { - radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); - - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - for (idx = 0; idx < count; ++idx) { - radeon_emit(cs, selectors[idx] | regs->select_or); - if (idx < regs->num_multi) - radeon_emit(cs, 0); - } - } else { - reg_base -= (reg_count - 1) * 4; - radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); - - for (idx = count; idx > 0; --idx) { - if (idx <= regs->num_multi) - radeon_emit(cs, 0); - radeon_emit(cs, selectors[idx - 1] | regs->select_or); - } - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - } - } + struct si_pc_block_base *regs = block->b->b; + struct radeon_cmdbuf *cs = sctx->gfx_cs; + unsigned idx; + unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK; + unsigned dw; + + assert(count <= regs->num_counters); + + if (regs->layout & SI_PC_FAKE) + return; + + if (layout_multi == SI_PC_MULTI_BLOCK) { + assert(!(regs->layout & SI_PC_REG_REVERSE)); + + dw = count + regs->num_prelude; + if (count >= regs->num_multi) + dw += regs->num_multi; + radeon_set_uconfig_reg_seq(cs, regs->select0, dw); + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) + radeon_emit(cs, selectors[idx] | regs->select_or); + + if (count < regs->num_multi) { + unsigned select1 = regs->select0 + 4 * regs->num_multi; + radeon_set_uconfig_reg_seq(cs, select1, count); + } + + for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) + radeon_emit(cs, 0); + + if (count > regs->num_multi) { + for (idx = regs->num_multi; idx < count; ++idx) + radeon_emit(cs, selectors[idx] | regs->select_or); + } + } else if (layout_multi == SI_PC_MULTI_TAIL) { + unsigned select1, select1_count; + + assert(!(regs->layout & SI_PC_REG_REVERSE)); + + radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude); + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + for (idx = 0; idx < count; ++idx) + radeon_emit(cs, selectors[idx] | regs->select_or); + + select1 = regs->select0 + 4 * regs->num_counters; + select1_count = MIN2(count, regs->num_multi); + radeon_set_uconfig_reg_seq(cs, select1, select1_count); + for (idx = 0; idx < select1_count; ++idx) + radeon_emit(cs, 0); + } else if (layout_multi == SI_PC_MULTI_CUSTOM) { + unsigned *reg = regs->select; + for (idx = 0; idx < count; ++idx) { + radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or); + if (idx < regs->num_multi) + radeon_set_uconfig_reg(cs, *reg++, 0); + } + } else { + assert(layout_multi == SI_PC_MULTI_ALTERNATE); + + unsigned reg_base = regs->select0; + unsigned reg_count = count + MIN2(count, regs->num_multi); + reg_count += regs->num_prelude; + + if (!(regs->layout & SI_PC_REG_REVERSE)) { + radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); + + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + for (idx = 0; idx < count; ++idx) { + radeon_emit(cs, selectors[idx] | regs->select_or); + if (idx < regs->num_multi) + radeon_emit(cs, 0); + } + } else { + reg_base -= (reg_count - 1) * 4; + radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); + + for (idx = count; idx > 0; --idx) { + if (idx <= regs->num_multi) + radeon_emit(cs, 0); + radeon_emit(cs, selectors[idx - 1] | regs->select_or); + } + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + } + } } -static void si_pc_emit_start(struct r600_common_context *ctx, - struct r600_resource *buffer, uint64_t va) +static void si_pc_emit_start(struct si_context *sctx, struct si_resource *buffer, uint64_t va) { - struct radeon_winsys_cs *cs = ctx->gfx.cs; - - radeon_add_to_buffer_list(ctx, &ctx->gfx, buffer, - RADEON_USAGE_WRITE, RADEON_PRIO_QUERY); - - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | - COPY_DATA_DST_SEL(COPY_DATA_MEM)); - radeon_emit(cs, 1); /* immediate */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, - S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET)); - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PERFCOUNTER_START) | EVENT_INDEX(0)); - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, - S_036020_PERFMON_STATE(V_036020_START_COUNTING)); + struct radeon_cmdbuf *cs = sctx->gfx_cs; + + si_cp_copy_data(sctx, sctx->gfx_cs, COPY_DATA_DST_MEM, buffer, va - buffer->gpu_address, + COPY_DATA_IMM, NULL, 1); + + radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, + S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET)); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_START) | EVENT_INDEX(0)); + radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, + S_036020_PERFMON_STATE(V_036020_START_COUNTING)); } /* Note: The buffer was already added in si_pc_emit_start, so we don't have to * do it again in here. */ -static void si_pc_emit_stop(struct r600_common_context *ctx, - struct r600_resource *buffer, uint64_t va) +static void si_pc_emit_stop(struct si_context *sctx, struct si_resource *buffer, uint64_t va) +{ + struct radeon_cmdbuf *cs = sctx->gfx_cs; + + si_cp_release_mem(sctx, cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE, + EOP_DATA_SEL_VALUE_32BIT, buffer, va, 0, SI_NOT_QUERY); + si_cp_wait_mem(sctx, cs, va, 0, 0xffffffff, WAIT_REG_MEM_EQUAL); + + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0)); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0)); + radeon_set_uconfig_reg( + cs, R_036020_CP_PERFMON_CNTL, + S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1)); +} + +static void si_pc_emit_read(struct si_context *sctx, struct si_pc_block *block, unsigned count, + uint64_t va) +{ + struct si_pc_block_base *regs = block->b->b; + struct radeon_cmdbuf *cs = sctx->gfx_cs; + unsigned idx; + unsigned reg = regs->counter0_lo; + unsigned reg_delta = 8; + + if (!(regs->layout & SI_PC_FAKE)) { + if (regs->layout & SI_PC_REG_REVERSE) + reg_delta = -reg_delta; + + for (idx = 0; idx < count; ++idx) { + if (regs->counters) + reg = regs->counters[idx]; + + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_COUNT_SEL); /* 64 bits */ + radeon_emit(cs, reg >> 2); + radeon_emit(cs, 0); /* unused */ + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + va += sizeof(uint64_t); + reg += reg_delta; + } + } else { + for (idx = 0; idx < count; ++idx) { + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_COUNT_SEL); + radeon_emit(cs, 0); /* immediate */ + radeon_emit(cs, 0); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + va += sizeof(uint64_t); + } + } +} + +static void si_pc_query_destroy(struct si_context *sctx, struct si_query *squery) +{ + struct si_query_pc *query = (struct si_query_pc *)squery; + + while (query->groups) { + struct si_query_group *group = query->groups; + query->groups = group->next; + FREE(group); + } + + FREE(query->counters); + + si_query_buffer_destroy(sctx->screen, &query->buffer); + FREE(query); +} + +static void si_inhibit_clockgating(struct si_context *sctx, bool inhibit) +{ + if (sctx->chip_class >= GFX10) { + radeon_set_uconfig_reg(sctx->gfx_cs, R_037390_RLC_PERFMON_CLK_CNTL, + S_037390_PERFMON_CLOCK_STATE(inhibit)); + } else if (sctx->chip_class >= GFX8) { + radeon_set_uconfig_reg(sctx->gfx_cs, R_0372FC_RLC_PERFMON_CLK_CNTL, + S_0372FC_PERFMON_CLOCK_STATE(inhibit)); + } +} + +static void si_pc_query_resume(struct si_context *sctx, struct si_query *squery) +/* + struct si_query_hw *hwquery, + struct si_resource *buffer, uint64_t va)*/ +{ + struct si_query_pc *query = (struct si_query_pc *)squery; + int current_se = -1; + int current_instance = -1; + + if (!si_query_buffer_alloc(sctx, &query->buffer, NULL, query->result_size)) + return; + si_need_gfx_cs_space(sctx); + + if (query->shaders) + si_pc_emit_shaders(sctx, query->shaders); + + si_inhibit_clockgating(sctx, true); + + for (struct si_query_group *group = query->groups; group; group = group->next) { + struct si_pc_block *block = group->block; + + if (group->se != current_se || group->instance != current_instance) { + current_se = group->se; + current_instance = group->instance; + si_pc_emit_instance(sctx, group->se, group->instance); + } + + si_pc_emit_select(sctx, block, group->num_counters, group->selectors); + } + + if (current_se != -1 || current_instance != -1) + si_pc_emit_instance(sctx, -1, -1); + + uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end; + si_pc_emit_start(sctx, query->buffer.buf, va); +} + +static void si_pc_query_suspend(struct si_context *sctx, struct si_query *squery) +{ + struct si_query_pc *query = (struct si_query_pc *)squery; + + if (!query->buffer.buf) + return; + + uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end; + query->buffer.results_end += query->result_size; + + si_pc_emit_stop(sctx, query->buffer.buf, va); + + for (struct si_query_group *group = query->groups; group; group = group->next) { + struct si_pc_block *block = group->block; + unsigned se = group->se >= 0 ? group->se : 0; + unsigned se_end = se + 1; + + if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0)) + se_end = sctx->screen->info.max_se; + + do { + unsigned instance = group->instance >= 0 ? group->instance : 0; + + do { + si_pc_emit_instance(sctx, se, instance); + si_pc_emit_read(sctx, block, group->num_counters, va); + va += sizeof(uint64_t) * group->num_counters; + } while (group->instance < 0 && ++instance < block->num_instances); + } while (++se < se_end); + } + + si_pc_emit_instance(sctx, -1, -1); + + si_inhibit_clockgating(sctx, false); +} + +static bool si_pc_query_begin(struct si_context *ctx, struct si_query *squery) +{ + struct si_query_pc *query = (struct si_query_pc *)squery; + + si_query_buffer_reset(ctx, &query->buffer); + + list_addtail(&query->b.active_list, &ctx->active_queries); + ctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend; + + si_pc_query_resume(ctx, squery); + + return true; +} + +static bool si_pc_query_end(struct si_context *ctx, struct si_query *squery) +{ + struct si_query_pc *query = (struct si_query_pc *)squery; + + si_pc_query_suspend(ctx, squery); + + list_del(&squery->active_list); + ctx->num_cs_dw_queries_suspend -= squery->num_cs_dw_suspend; + + return query->buffer.buf != NULL; +} + +static void si_pc_query_add_result(struct si_query_pc *query, void *buffer, + union pipe_query_result *result) +{ + uint64_t *results = buffer; + unsigned i, j; + + for (i = 0; i < query->num_counters; ++i) { + struct si_query_counter *counter = &query->counters[i]; + + for (j = 0; j < counter->qwords; ++j) { + uint32_t value = results[counter->base + j * counter->stride]; + result->batch[i].u64 += value; + } + } +} + +static bool si_pc_query_get_result(struct si_context *sctx, struct si_query *squery, bool wait, + union pipe_query_result *result) +{ + struct si_query_pc *query = (struct si_query_pc *)squery; + + memset(result, 0, sizeof(result->batch[0]) * query->num_counters); + + for (struct si_query_buffer *qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) { + unsigned usage = PIPE_TRANSFER_READ | (wait ? 0 : PIPE_TRANSFER_DONTBLOCK); + unsigned results_base = 0; + void *map; + + if (squery->b.flushed) + map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage); + else + map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage); + + if (!map) + return false; + + while (results_base != qbuf->results_end) { + si_pc_query_add_result(query, map + results_base, result); + results_base += query->result_size; + } + } + + return true; +} + +static const struct si_query_ops batch_query_ops = { + .destroy = si_pc_query_destroy, + .begin = si_pc_query_begin, + .end = si_pc_query_end, + .get_result = si_pc_query_get_result, + + .suspend = si_pc_query_suspend, + .resume = si_pc_query_resume, +}; + +static struct si_query_group *get_group_state(struct si_screen *screen, struct si_query_pc *query, + struct si_pc_block *block, unsigned sub_gid) +{ + struct si_query_group *group = query->groups; + + while (group) { + if (group->block == block && group->sub_gid == sub_gid) + return group; + group = group->next; + } + + group = CALLOC_STRUCT(si_query_group); + if (!group) + return NULL; + + group->block = block; + group->sub_gid = sub_gid; + + if (block->b->b->flags & SI_PC_BLOCK_SHADER) { + unsigned sub_gids = block->num_instances; + unsigned shader_id; + unsigned shaders; + unsigned query_shaders; + + if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) + sub_gids = sub_gids * screen->info.max_se; + shader_id = sub_gid / sub_gids; + sub_gid = sub_gid % sub_gids; + + shaders = si_pc_shader_type_bits[shader_id]; + + query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING; + if (query_shaders && query_shaders != shaders) { + fprintf(stderr, "si_perfcounter: incompatible shader groups\n"); + FREE(group); + return NULL; + } + query->shaders = shaders; + } + + if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) { + // A non-zero value in query->shaders ensures that the shader + // masking is reset unless the user explicitly requests one. + query->shaders = SI_PC_SHADERS_WINDOWING; + } + + if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) { + group->se = sub_gid / block->num_instances; + sub_gid = sub_gid % block->num_instances; + } else { + group->se = -1; + } + + if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) { + group->instance = sub_gid; + } else { + group->instance = -1; + } + + group->next = query->groups; + query->groups = group; + + return group; +} + +struct pipe_query *si_create_batch_query(struct pipe_context *ctx, unsigned num_queries, + unsigned *query_types) +{ + struct si_screen *screen = (struct si_screen *)ctx->screen; + struct si_perfcounters *pc = screen->perfcounters; + struct si_pc_block *block; + struct si_query_group *group; + struct si_query_pc *query; + unsigned base_gid, sub_gid, sub_index; + unsigned i, j; + + if (!pc) + return NULL; + + query = CALLOC_STRUCT(si_query_pc); + if (!query) + return NULL; + + query->b.ops = &batch_query_ops; + + query->num_counters = num_queries; + + /* Collect selectors per group */ + for (i = 0; i < num_queries; ++i) { + unsigned sub_gid; + + if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER) + goto error; + + block = + lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, &base_gid, &sub_index); + if (!block) + goto error; + + sub_gid = sub_index / block->b->selectors; + sub_index = sub_index % block->b->selectors; + + group = get_group_state(screen, query, block, sub_gid); + if (!group) + goto error; + + if (group->num_counters >= block->b->b->num_counters) { + fprintf(stderr, "perfcounter group %s: too many selected\n", block->b->b->name); + goto error; + } + group->selectors[group->num_counters] = sub_index; + ++group->num_counters; + } + + /* Compute result bases and CS size per group */ + query->b.num_cs_dw_suspend = pc->num_stop_cs_dwords; + query->b.num_cs_dw_suspend += pc->num_instance_cs_dwords; + + i = 0; + for (group = query->groups; group; group = group->next) { + struct si_pc_block *block = group->block; + unsigned read_dw; + unsigned instances = 1; + + if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0) + instances = screen->info.max_se; + if (group->instance < 0) + instances *= block->num_instances; + + group->result_base = i; + query->result_size += sizeof(uint64_t) * instances * group->num_counters; + i += instances * group->num_counters; + + read_dw = 6 * group->num_counters; + query->b.num_cs_dw_suspend += instances * read_dw; + query->b.num_cs_dw_suspend += instances * pc->num_instance_cs_dwords; + } + + if (query->shaders) { + if (query->shaders == SI_PC_SHADERS_WINDOWING) + query->shaders = 0xffffffff; + } + + /* Map user-supplied query array to result indices */ + query->counters = CALLOC(num_queries, sizeof(*query->counters)); + for (i = 0; i < num_queries; ++i) { + struct si_query_counter *counter = &query->counters[i]; + struct si_pc_block *block; + + block = + lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, &base_gid, &sub_index); + + sub_gid = sub_index / block->b->selectors; + sub_index = sub_index % block->b->selectors; + + group = get_group_state(screen, query, block, sub_gid); + assert(group != NULL); + + for (j = 0; j < group->num_counters; ++j) { + if (group->selectors[j] == sub_index) + break; + } + + counter->base = group->result_base + j; + counter->stride = group->num_counters; + + counter->qwords = 1; + if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0) + counter->qwords = screen->info.max_se; + if (group->instance < 0) + counter->qwords *= block->num_instances; + } + + return (struct pipe_query *)query; + +error: + si_pc_query_destroy((struct si_context *)ctx, &query->b); + return NULL; +} + +static bool si_init_block_names(struct si_screen *screen, struct si_pc_block *block) +{ + bool per_instance_groups = si_pc_block_has_per_instance_groups(screen->perfcounters, block); + bool per_se_groups = si_pc_block_has_per_se_groups(screen->perfcounters, block); + unsigned i, j, k; + unsigned groups_shader = 1, groups_se = 1, groups_instance = 1; + unsigned namelen; + char *groupname; + char *p; + + if (per_instance_groups) + groups_instance = block->num_instances; + if (per_se_groups) + groups_se = screen->info.max_se; + if (block->b->b->flags & SI_PC_BLOCK_SHADER) + groups_shader = ARRAY_SIZE(si_pc_shader_type_bits); + + namelen = strlen(block->b->b->name); + block->group_name_stride = namelen + 1; + if (block->b->b->flags & SI_PC_BLOCK_SHADER) + block->group_name_stride += 3; + if (per_se_groups) { + assert(groups_se <= 10); + block->group_name_stride += 1; + + if (per_instance_groups) + block->group_name_stride += 1; + } + if (per_instance_groups) { + assert(groups_instance <= 100); + block->group_name_stride += 2; + } + + block->group_names = MALLOC(block->num_groups * block->group_name_stride); + if (!block->group_names) + return false; + + groupname = block->group_names; + for (i = 0; i < groups_shader; ++i) { + const char *shader_suffix = si_pc_shader_type_suffixes[i]; + unsigned shaderlen = strlen(shader_suffix); + for (j = 0; j < groups_se; ++j) { + for (k = 0; k < groups_instance; ++k) { + strcpy(groupname, block->b->b->name); + p = groupname + namelen; + + if (block->b->b->flags & SI_PC_BLOCK_SHADER) { + strcpy(p, shader_suffix); + p += shaderlen; + } + + if (per_se_groups) { + p += sprintf(p, "%d", j); + if (per_instance_groups) + *p++ = '_'; + } + + if (per_instance_groups) + p += sprintf(p, "%d", k); + + groupname += block->group_name_stride; + } + } + } + + assert(block->b->selectors <= 1000); + block->selector_name_stride = block->group_name_stride + 4; + block->selector_names = + MALLOC(block->num_groups * block->b->selectors * block->selector_name_stride); + if (!block->selector_names) + return false; + + groupname = block->group_names; + p = block->selector_names; + for (i = 0; i < block->num_groups; ++i) { + for (j = 0; j < block->b->selectors; ++j) { + sprintf(p, "%s_%03d", groupname, j); + p += block->selector_name_stride; + } + groupname += block->group_name_stride; + } + + return true; +} + +int si_get_perfcounter_info(struct si_screen *screen, unsigned index, + struct pipe_driver_query_info *info) { - struct radeon_winsys_cs *cs = ctx->gfx.cs; - - if (ctx->screen->chip_class == CIK) { - /* Workaround for cache flush problems: send two EOP events. */ - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0)); - radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | - EVENT_INDEX(5)); - radeon_emit(cs, va); - radeon_emit(cs, (va >> 32) | EOP_DATA_SEL(1)); - radeon_emit(cs, 0); /* immediate data */ - radeon_emit(cs, 0); /* unused */ - } - - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0)); - radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | - EVENT_INDEX(5)); - radeon_emit(cs, va); - radeon_emit(cs, (va >> 32) | EOP_DATA_SEL(1)); - radeon_emit(cs, 0); /* immediate data */ - radeon_emit(cs, 0); /* unused */ - - radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); - radeon_emit(cs, WAIT_REG_MEM_EQUAL | WAIT_REG_MEM_MEM_SPACE(1)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, 0); /* reference value */ - radeon_emit(cs, 0xffffffff); /* mask */ - radeon_emit(cs, 4); /* poll interval */ - - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0)); - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PERFCOUNTER_STOP) | EVENT_INDEX(0)); - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, - S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) | - S_036020_PERFMON_SAMPLE_ENABLE(1)); + struct si_perfcounters *pc = screen->perfcounters; + struct si_pc_block *block; + unsigned base_gid, sub; + + if (!pc) + return 0; + + if (!info) { + unsigned bid, num_queries = 0; + + for (bid = 0; bid < pc->num_blocks; ++bid) { + num_queries += pc->blocks[bid].b->selectors * pc->blocks[bid].num_groups; + } + + return num_queries; + } + + block = lookup_counter(pc, index, &base_gid, &sub); + if (!block) + return 0; + + if (!block->selector_names) { + if (!si_init_block_names(screen, block)) + return 0; + } + info->name = block->selector_names + sub * block->selector_name_stride; + info->query_type = SI_QUERY_FIRST_PERFCOUNTER + index; + info->max_value.u64 = 0; + info->type = PIPE_DRIVER_QUERY_TYPE_UINT64; + info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE; + info->group_id = base_gid + sub / block->b->selectors; + info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH; + if (sub > 0 && sub + 1 < block->b->selectors * block->num_groups) + info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST; + return 1; } -static void si_pc_emit_read(struct r600_common_context *ctx, - struct r600_perfcounter_block *group, - unsigned count, unsigned *selectors, - struct r600_resource *buffer, uint64_t va) +int si_get_perfcounter_group_info(struct si_screen *screen, unsigned index, + struct pipe_driver_query_group_info *info) { - struct si_pc_block *sigroup = (struct si_pc_block *)group->data; - struct si_pc_block_base *regs = sigroup->b; - struct radeon_winsys_cs *cs = ctx->gfx.cs; - unsigned idx; - unsigned reg = regs->counter0_lo; - unsigned reg_delta = 8; - - if (!(regs->layout & SI_PC_FAKE)) { - if (regs->layout & SI_PC_REG_REVERSE) - reg_delta = -reg_delta; - - for (idx = 0; idx < count; ++idx) { - if (regs->counters) - reg = regs->counters[idx]; - - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | - COPY_DATA_DST_SEL(COPY_DATA_MEM)); - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - va += 4; - reg += reg_delta; - } - } else { - for (idx = 0; idx < count; ++idx) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | - COPY_DATA_DST_SEL(COPY_DATA_MEM)); - radeon_emit(cs, 0); /* immediate */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - va += 4; - } - } + struct si_perfcounters *pc = screen->perfcounters; + struct si_pc_block *block; + + if (!pc) + return 0; + + if (!info) + return pc->num_groups; + + block = lookup_group(pc, &index); + if (!block) + return 0; + + if (!block->group_names) { + if (!si_init_block_names(screen, block)) + return 0; + } + info->name = block->group_names + index * block->group_name_stride; + info->num_queries = block->b->selectors; + info->max_active_queries = block->b->b->num_counters; + return 1; } -static void si_pc_cleanup(struct r600_common_screen *rscreen) +void si_destroy_perfcounters(struct si_screen *screen) { - r600_perfcounters_do_destroy(rscreen->perfcounters); - rscreen->perfcounters = NULL; + struct si_perfcounters *pc = screen->perfcounters; + unsigned i; + + if (!pc) + return; + + for (i = 0; i < pc->num_blocks; ++i) { + FREE(pc->blocks[i].group_names); + FREE(pc->blocks[i].selector_names); + } + FREE(pc->blocks); + FREE(pc); + screen->perfcounters = NULL; } void si_init_perfcounters(struct si_screen *screen) { - struct r600_perfcounters *pc; - struct si_pc_block *blocks; - unsigned num_blocks; - unsigned i; - - switch (screen->b.chip_class) { - case CIK: - blocks = groups_CIK; - num_blocks = ARRAY_SIZE(groups_CIK); - break; - case VI: - blocks = groups_VI; - num_blocks = ARRAY_SIZE(groups_VI); - break; - case SI: - default: - return; /* not implemented */ - } - - if (screen->b.info.max_sh_per_se != 1) { - /* This should not happen on non-SI chips. */ - fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not " - "supported (inaccurate performance counters)\n", - screen->b.info.max_sh_per_se); - } - - pc = CALLOC_STRUCT(r600_perfcounters); - if (!pc) - return; - - pc->num_start_cs_dwords = 14; - pc->num_stop_cs_dwords = 20; - pc->num_instance_cs_dwords = 3; - pc->num_shaders_cs_dwords = 4; - - if (screen->b.chip_class == CIK) { - pc->num_stop_cs_dwords += 6; - } - - pc->num_shader_types = ARRAY_SIZE(si_pc_shader_type_bits); - pc->shader_type_suffixes = si_pc_shader_type_suffixes; - pc->shader_type_bits = si_pc_shader_type_bits; - - pc->get_size = si_pc_get_size; - pc->emit_instance = si_pc_emit_instance; - pc->emit_shaders = si_pc_emit_shaders; - pc->emit_select = si_pc_emit_select; - pc->emit_start = si_pc_emit_start; - pc->emit_stop = si_pc_emit_stop; - pc->emit_read = si_pc_emit_read; - pc->cleanup = si_pc_cleanup; - - if (!r600_perfcounters_init(pc, num_blocks)) - goto error; - - for (i = 0; i < num_blocks; ++i) { - struct si_pc_block *block = &blocks[i]; - unsigned instances = block->instances; - - if (!strcmp(block->b->name, "IA")) { - if (screen->b.info.max_se > 2) - instances = 2; - } - - r600_perfcounters_add_block(&screen->b, pc, - block->b->name, - block->b->flags, - block->b->num_counters, - block->selectors, - instances, - block); - } - - screen->b.perfcounters = pc; - return; + struct si_perfcounters *pc; + const struct si_pc_block_gfxdescr *blocks; + unsigned num_blocks; + unsigned i; + + switch (screen->info.chip_class) { + case GFX7: + blocks = groups_CIK; + num_blocks = ARRAY_SIZE(groups_CIK); + break; + case GFX8: + blocks = groups_VI; + num_blocks = ARRAY_SIZE(groups_VI); + break; + case GFX9: + blocks = groups_gfx9; + num_blocks = ARRAY_SIZE(groups_gfx9); + break; + case GFX10: + case GFX10_3: + blocks = groups_gfx10; + num_blocks = ARRAY_SIZE(groups_gfx10); + break; + case GFX6: + default: + return; /* not implemented */ + } + + screen->perfcounters = pc = CALLOC_STRUCT(si_perfcounters); + if (!pc) + return; + + pc->num_stop_cs_dwords = 14 + si_cp_write_fence_dwords(screen); + pc->num_instance_cs_dwords = 3; + + pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false); + pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false); + + pc->blocks = CALLOC(num_blocks, sizeof(struct si_pc_block)); + if (!pc->blocks) + goto error; + pc->num_blocks = num_blocks; + + for (i = 0; i < num_blocks; ++i) { + struct si_pc_block *block = &pc->blocks[i]; + block->b = &blocks[i]; + block->num_instances = MAX2(1, block->b->instances); + + if (!strcmp(block->b->b->name, "CB") || + !strcmp(block->b->b->name, "DB") || + !strcmp(block->b->b->name, "RMI")) + block->num_instances = screen->info.max_se; + else if (!strcmp(block->b->b->name, "TCC")) + block->num_instances = screen->info.num_tcc_blocks; + else if (!strcmp(block->b->b->name, "IA")) + block->num_instances = MAX2(1, screen->info.max_se / 2); + else if (!strcmp(block->b->b->name, "TA") || + !strcmp(block->b->b->name, "TCP") || + !strcmp(block->b->b->name, "TD")) { + block->num_instances = MAX2(1, screen->info.max_good_cu_per_sa); + } + + if (si_pc_block_has_per_instance_groups(pc, block)) { + block->num_groups = block->num_instances; + } else { + block->num_groups = 1; + } + + if (si_pc_block_has_per_se_groups(pc, block)) + block->num_groups *= screen->info.max_se; + if (block->b->b->flags & SI_PC_BLOCK_SHADER) + block->num_groups *= ARRAY_SIZE(si_pc_shader_type_bits); + + pc->num_groups += block->num_groups; + } + + return; error: - r600_perfcounters_do_destroy(pc); + si_destroy_perfcounters(screen); }