X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_perfcounter.c;h=d2f0db81d2c88ac2e3d1866786d25101fe88e5cf;hb=e2e700f6053d0b16ba46e4d5c5b20e965fb2224e;hp=f15f310dd11faed11004cb3d14e7ac4a1cf31361;hpb=54bc87469a4747ea626d1d680f39eb3d282ede5d;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/si_perfcounter.c b/src/gallium/drivers/radeonsi/si_perfcounter.c index f15f310dd11..d2f0db81d2c 100644 --- a/src/gallium/drivers/radeonsi/si_perfcounter.c +++ b/src/gallium/drivers/radeonsi/si_perfcounter.c @@ -26,380 +26,563 @@ #include "si_query.h" #include "util/u_memory.h" +enum si_pc_block_flags +{ + /* This block is part of the shader engine */ + SI_PC_BLOCK_SE = (1 << 0), -enum si_pc_block_flags { - /* This block is part of the shader engine */ - SI_PC_BLOCK_SE = (1 << 0), - - /* Expose per-instance groups instead of summing all instances (within - * an SE). */ - SI_PC_BLOCK_INSTANCE_GROUPS = (1 << 1), + /* Expose per-instance groups instead of summing all instances (within + * an SE). */ + SI_PC_BLOCK_INSTANCE_GROUPS = (1 << 1), - /* Expose per-SE groups instead of summing instances across SEs. */ - SI_PC_BLOCK_SE_GROUPS = (1 << 2), + /* Expose per-SE groups instead of summing instances across SEs. */ + SI_PC_BLOCK_SE_GROUPS = (1 << 2), - /* Shader block */ - SI_PC_BLOCK_SHADER = (1 << 3), + /* Shader block */ + SI_PC_BLOCK_SHADER = (1 << 3), - /* Non-shader block with perfcounters windowed by shaders. */ - SI_PC_BLOCK_SHADER_WINDOWED = (1 << 4), + /* Non-shader block with perfcounters windowed by shaders. */ + SI_PC_BLOCK_SHADER_WINDOWED = (1 << 4), }; -enum si_pc_reg_layout { - /* All secondary selector dwords follow as one block after the primary - * selector dwords for the counters that have secondary selectors. - */ - SI_PC_MULTI_BLOCK = 0, - - /* Each secondary selector dword follows immediately afters the - * corresponding primary. - */ - SI_PC_MULTI_ALTERNATE = 1, - - /* All secondary selector dwords follow as one block after all primary - * selector dwords. - */ - SI_PC_MULTI_TAIL = 2, - - /* Free-form arrangement of selector registers. */ - SI_PC_MULTI_CUSTOM = 3, - - SI_PC_MULTI_MASK = 3, - - /* Registers are laid out in decreasing rather than increasing order. */ - SI_PC_REG_REVERSE = 4, - - SI_PC_FAKE = 8, +enum si_pc_reg_layout +{ + /* All secondary selector dwords follow as one block after the primary + * selector dwords for the counters that have secondary selectors. + * + * Example: + * PERFCOUNTER0_SELECT + * PERFCOUNTER1_SELECT + * PERFCOUNTER0_SELECT1 + * PERFCOUNTER1_SELECT1 + * PERFCOUNTER2_SELECT + * PERFCOUNTER3_SELECT + */ + SI_PC_MULTI_BLOCK = 0, + + /* Each secondary selector dword follows immediately after the + * corresponding primary. + * + * Example: + * PERFCOUNTER0_SELECT + * PERFCOUNTER0_SELECT1 + * PERFCOUNTER1_SELECT + * PERFCOUNTER1_SELECT1 + * PERFCOUNTER2_SELECT + * PERFCOUNTER3_SELECT + */ + SI_PC_MULTI_ALTERNATE = 1, + + /* All secondary selector dwords follow as one block after all primary + * selector dwords. + * + * Example: + * PERFCOUNTER0_SELECT + * PERFCOUNTER1_SELECT + * PERFCOUNTER2_SELECT + * PERFCOUNTER3_SELECT + * PERFCOUNTER0_SELECT1 + * PERFCOUNTER1_SELECT1 + */ + SI_PC_MULTI_TAIL = 2, + + /* Free-form arrangement of selector registers. */ + SI_PC_MULTI_CUSTOM = 3, + + SI_PC_MULTI_MASK = 3, + + /* Registers are laid out in decreasing rather than increasing order. */ + SI_PC_REG_REVERSE = 4, + + SI_PC_FAKE = 8, }; struct si_pc_block_base { - const char *name; - unsigned num_counters; - unsigned flags; - - unsigned select_or; - unsigned select0; - unsigned counter0_lo; - unsigned *select; - unsigned *counters; - unsigned num_multi; - unsigned num_prelude; - unsigned layout; + const char *name; + unsigned num_counters; + unsigned flags; + + unsigned select_or; + unsigned select0; + unsigned counter0_lo; + unsigned *select; + unsigned *counters; + unsigned num_multi; + unsigned num_prelude; + unsigned layout; }; struct si_pc_block_gfxdescr { - struct si_pc_block_base *b; - unsigned selectors; - unsigned instances; + struct si_pc_block_base *b; + unsigned selectors; + unsigned instances; }; struct si_pc_block { - const struct si_pc_block_gfxdescr *b; - unsigned num_instances; + const struct si_pc_block_gfxdescr *b; + unsigned num_instances; - unsigned num_groups; - char *group_names; - unsigned group_name_stride; + unsigned num_groups; + char *group_names; + unsigned group_name_stride; - char *selector_names; - unsigned selector_name_stride; + char *selector_names; + unsigned selector_name_stride; }; /* The order is chosen to be compatible with GPUPerfStudio's hardcoding of * performance counter group IDs. */ -static const char * const si_pc_shader_type_suffixes[] = { - "", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS" -}; +static const char *const si_pc_shader_type_suffixes[] = {"", "_ES", "_GS", "_VS", + "_PS", "_LS", "_HS", "_CS"}; static const unsigned si_pc_shader_type_bits[] = { - 0x7f, - S_036780_ES_EN(1), - S_036780_GS_EN(1), - S_036780_VS_EN(1), - S_036780_PS_EN(1), - S_036780_LS_EN(1), - S_036780_HS_EN(1), - S_036780_CS_EN(1), + 0x7f, + S_036780_ES_EN(1), + S_036780_GS_EN(1), + S_036780_VS_EN(1), + S_036780_PS_EN(1), + S_036780_LS_EN(1), + S_036780_HS_EN(1), + S_036780_CS_EN(1), }; /* Max counters per HW block */ #define SI_QUERY_MAX_COUNTERS 16 -#define SI_PC_SHADERS_WINDOWING (1 << 31) +#define SI_PC_SHADERS_WINDOWING (1u << 31) struct si_query_group { - struct si_query_group *next; - struct si_pc_block *block; - unsigned sub_gid; /* only used during init */ - unsigned result_base; /* only used during init */ - int se; - int instance; - unsigned num_counters; - unsigned selectors[SI_QUERY_MAX_COUNTERS]; + struct si_query_group *next; + struct si_pc_block *block; + unsigned sub_gid; /* only used during init */ + unsigned result_base; /* only used during init */ + int se; + int instance; + unsigned num_counters; + unsigned selectors[SI_QUERY_MAX_COUNTERS]; }; struct si_query_counter { - unsigned base; - unsigned qwords; - unsigned stride; /* in uint64s */ + unsigned base; + unsigned qwords; + unsigned stride; /* in uint64s */ }; struct si_query_pc { - struct si_query b; - struct si_query_buffer buffer; + struct si_query b; + struct si_query_buffer buffer; - /* Size of the results in memory, in bytes. */ - unsigned result_size; + /* Size of the results in memory, in bytes. */ + unsigned result_size; - unsigned shaders; - unsigned num_counters; - struct si_query_counter *counters; - struct si_query_group *groups; + unsigned shaders; + unsigned num_counters; + struct si_query_counter *counters; + struct si_query_group *groups; }; - static struct si_pc_block_base cik_CB = { - .name = "CB", - .num_counters = 4, - .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_037000_CB_PERFCOUNTER_FILTER, - .counter0_lo = R_035018_CB_PERFCOUNTER0_LO, - .num_multi = 1, - .num_prelude = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "CB", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_037000_CB_PERFCOUNTER_FILTER, + .counter0_lo = R_035018_CB_PERFCOUNTER0_LO, + .num_multi = 1, + .num_prelude = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; static unsigned cik_CPC_select[] = { - R_036024_CPC_PERFCOUNTER0_SELECT, - R_036010_CPC_PERFCOUNTER0_SELECT1, - R_03600C_CPC_PERFCOUNTER1_SELECT, + R_036024_CPC_PERFCOUNTER0_SELECT, + R_036010_CPC_PERFCOUNTER0_SELECT1, + R_03600C_CPC_PERFCOUNTER1_SELECT, }; static struct si_pc_block_base cik_CPC = { - .name = "CPC", - .num_counters = 2, + .name = "CPC", + .num_counters = 2, - .select = cik_CPC_select, - .counter0_lo = R_034018_CPC_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE, + .select = cik_CPC_select, + .counter0_lo = R_034018_CPC_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE, }; static struct si_pc_block_base cik_CPF = { - .name = "CPF", - .num_counters = 2, + .name = "CPF", + .num_counters = 2, - .select0 = R_03601C_CPF_PERFCOUNTER0_SELECT, - .counter0_lo = R_034028_CPF_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, + .select0 = R_03601C_CPF_PERFCOUNTER0_SELECT, + .counter0_lo = R_034028_CPF_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, }; static struct si_pc_block_base cik_CPG = { - .name = "CPG", - .num_counters = 2, + .name = "CPG", + .num_counters = 2, - .select0 = R_036008_CPG_PERFCOUNTER0_SELECT, - .counter0_lo = R_034008_CPG_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, + .select0 = R_036008_CPG_PERFCOUNTER0_SELECT, + .counter0_lo = R_034008_CPG_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE, }; static struct si_pc_block_base cik_DB = { - .name = "DB", - .num_counters = 4, - .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_037100_DB_PERFCOUNTER0_SELECT, - .counter0_lo = R_035100_DB_PERFCOUNTER0_LO, - .num_multi = 3, // really only 2, but there's a gap between registers - .layout = SI_PC_MULTI_ALTERNATE, + .name = "DB", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_037100_DB_PERFCOUNTER0_SELECT, + .counter0_lo = R_035100_DB_PERFCOUNTER0_LO, + .num_multi = 3, // really only 2, but there's a gap between registers + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_GDS = { - .name = "GDS", - .num_counters = 4, + .name = "GDS", + .num_counters = 4, - .select0 = R_036A00_GDS_PERFCOUNTER0_SELECT, - .counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_TAIL, + .select0 = R_036A00_GDS_PERFCOUNTER0_SELECT, + .counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_TAIL, }; static unsigned cik_GRBM_counters[] = { - R_034100_GRBM_PERFCOUNTER0_LO, - R_03410C_GRBM_PERFCOUNTER1_LO, + R_034100_GRBM_PERFCOUNTER0_LO, + R_03410C_GRBM_PERFCOUNTER1_LO, }; static struct si_pc_block_base cik_GRBM = { - .name = "GRBM", - .num_counters = 2, + .name = "GRBM", + .num_counters = 2, - .select0 = R_036100_GRBM_PERFCOUNTER0_SELECT, - .counters = cik_GRBM_counters, + .select0 = R_036100_GRBM_PERFCOUNTER0_SELECT, + .counters = cik_GRBM_counters, }; static struct si_pc_block_base cik_GRBMSE = { - .name = "GRBMSE", - .num_counters = 4, + .name = "GRBMSE", + .num_counters = 4, - .select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT, - .counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO, + .select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT, + .counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO, }; static struct si_pc_block_base cik_IA = { - .name = "IA", - .num_counters = 4, + .name = "IA", + .num_counters = 4, - .select0 = R_036210_IA_PERFCOUNTER0_SELECT, - .counter0_lo = R_034220_IA_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_TAIL, + .select0 = R_036210_IA_PERFCOUNTER0_SELECT, + .counter0_lo = R_034220_IA_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_TAIL, }; static struct si_pc_block_base cik_PA_SC = { - .name = "PA_SC", - .num_counters = 8, - .flags = SI_PC_BLOCK_SE, - - .select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT, - .counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "PA_SC", + .num_counters = 8, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT, + .counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; /* According to docs, PA_SU counters are only 48 bits wide. */ static struct si_pc_block_base cik_PA_SU = { - .name = "PA_SU", - .num_counters = 4, - .flags = SI_PC_BLOCK_SE, - - .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT, - .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "PA_SU", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT, + .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_SPI = { - .name = "SPI", - .num_counters = 6, - .flags = SI_PC_BLOCK_SE, - - .select0 = R_036600_SPI_PERFCOUNTER0_SELECT, - .counter0_lo = R_034604_SPI_PERFCOUNTER0_LO, - .num_multi = 4, - .layout = SI_PC_MULTI_BLOCK, + .name = "SPI", + .num_counters = 6, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036600_SPI_PERFCOUNTER0_SELECT, + .counter0_lo = R_034604_SPI_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_BLOCK, }; static struct si_pc_block_base cik_SQ = { - .name = "SQ", - .num_counters = 16, - .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER, - - .select0 = R_036700_SQ_PERFCOUNTER0_SELECT, - .select_or = S_036700_SQC_BANK_MASK(15) | - S_036700_SQC_CLIENT_MASK(15) | - S_036700_SIMD_MASK(15), - .counter0_lo = R_034700_SQ_PERFCOUNTER0_LO, + .name = "SQ", + .num_counters = 16, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER, + + .select0 = R_036700_SQ_PERFCOUNTER0_SELECT, + .select_or = S_036700_SQC_BANK_MASK(15) | S_036700_SQC_CLIENT_MASK(15) | S_036700_SIMD_MASK(15), + .counter0_lo = R_034700_SQ_PERFCOUNTER0_LO, }; static struct si_pc_block_base cik_SX = { - .name = "SX", - .num_counters = 4, - .flags = SI_PC_BLOCK_SE, - - .select0 = R_036900_SX_PERFCOUNTER0_SELECT, - .counter0_lo = R_034900_SX_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_TAIL, + .name = "SX", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036900_SX_PERFCOUNTER0_SELECT, + .counter0_lo = R_034900_SX_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_TAIL, }; static struct si_pc_block_base cik_TA = { - .name = "TA", - .num_counters = 2, - .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, - - .select0 = R_036B00_TA_PERFCOUNTER0_SELECT, - .counter0_lo = R_034B00_TA_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TA", + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036B00_TA_PERFCOUNTER0_SELECT, + .counter0_lo = R_034B00_TA_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TD = { - .name = "TD", - .num_counters = 2, - .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, - - .select0 = R_036C00_TD_PERFCOUNTER0_SELECT, - .counter0_lo = R_034C00_TD_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TD", + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036C00_TD_PERFCOUNTER0_SELECT, + .counter0_lo = R_034C00_TD_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TCA = { - .name = "TCA", - .num_counters = 4, - .flags = SI_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_036E40_TCA_PERFCOUNTER0_SELECT, - .counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TCA", + .num_counters = 4, + .flags = SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_036E40_TCA_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TCC = { - .name = "TCC", - .num_counters = 4, - .flags = SI_PC_BLOCK_INSTANCE_GROUPS, - - .select0 = R_036E00_TCC_PERFCOUNTER0_SELECT, - .counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TCC", + .num_counters = 4, + .flags = SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_036E00_TCC_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_TCP = { - .name = "TCP", - .num_counters = 4, - .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, - - .select0 = R_036D00_TCP_PERFCOUNTER0_SELECT, - .counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO, - .num_multi = 2, - .layout = SI_PC_MULTI_ALTERNATE, + .name = "TCP", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036D00_TCP_PERFCOUNTER0_SELECT, + .counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, }; static struct si_pc_block_base cik_VGT = { - .name = "VGT", - .num_counters = 4, - .flags = SI_PC_BLOCK_SE, - - .select0 = R_036230_VGT_PERFCOUNTER0_SELECT, - .counter0_lo = R_034240_VGT_PERFCOUNTER0_LO, - .num_multi = 1, - .layout = SI_PC_MULTI_TAIL, + .name = "VGT", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036230_VGT_PERFCOUNTER0_SELECT, + .counter0_lo = R_034240_VGT_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_TAIL, }; static struct si_pc_block_base cik_WD = { - .name = "WD", - .num_counters = 4, + .name = "WD", + .num_counters = 4, - .select0 = R_036200_WD_PERFCOUNTER0_SELECT, - .counter0_lo = R_034200_WD_PERFCOUNTER0_LO, + .select0 = R_036200_WD_PERFCOUNTER0_SELECT, + .counter0_lo = R_034200_WD_PERFCOUNTER0_LO, }; static struct si_pc_block_base cik_MC = { - .name = "MC", - .num_counters = 4, + .name = "MC", + .num_counters = 4, - .layout = SI_PC_FAKE, + .layout = SI_PC_FAKE, }; static struct si_pc_block_base cik_SRBM = { - .name = "SRBM", - .num_counters = 2, + .name = "SRBM", + .num_counters = 2, + + .layout = SI_PC_FAKE, +}; + +static struct si_pc_block_base gfx10_CHA = { + .name = "CHA", + .num_counters = 4, + + .select0 = R_037780_CHA_PERFCOUNTER0_SELECT, + .counter0_lo = R_035800_CHA_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_CHCG = { + .name = "CHCG", + .num_counters = 4, + + .select0 = R_036F18_CHCG_PERFCOUNTER0_SELECT, + .counter0_lo = R_034F20_CHCG_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_CHC = { + .name = "CHC", + .num_counters = 4, + + .select0 = R_036F00_CHC_PERFCOUNTER0_SELECT, + .counter0_lo = R_034F00_CHC_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GCR = { + .name = "GCR", + .num_counters = 2, + + .select0 = R_037580_GCR_PERFCOUNTER0_SELECT, + .counter0_lo = R_035480_GCR_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GE = { + .name = "GE", + .num_counters = 12, + + .select0 = R_036200_GE_PERFCOUNTER0_SELECT, + .counter0_lo = R_034200_GE_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL1A = { + .name = "GL1A", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_037700_GL1A_PERFCOUNTER0_SELECT, + .counter0_lo = R_035700_GL1A_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL1C = { + .name = "GL1C", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_036E80_GL1C_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E80_GL1C_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL2A = { + .name = "GL2A", + .num_counters = 4, + + .select0 = R_036E40_GL2A_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E40_GL2A_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_GL2C = { + .name = "GL2C", + .num_counters = 4, - .layout = SI_PC_FAKE, + .select0 = R_036E00_GL2C_PERFCOUNTER0_SELECT, + .counter0_lo = R_034E00_GL2C_PERFCOUNTER0_LO, + .num_multi = 2, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static unsigned gfx10_PA_PH_select[] = { + R_037600_PA_PH_PERFCOUNTER0_SELECT, + R_037604_PA_PH_PERFCOUNTER0_SELECT1, + R_037608_PA_PH_PERFCOUNTER1_SELECT, + R_037640_PA_PH_PERFCOUNTER1_SELECT1, + R_03760C_PA_PH_PERFCOUNTER2_SELECT, + R_037644_PA_PH_PERFCOUNTER2_SELECT1, + R_037610_PA_PH_PERFCOUNTER3_SELECT, + R_037648_PA_PH_PERFCOUNTER3_SELECT1, + R_037614_PA_PH_PERFCOUNTER4_SELECT, + R_037618_PA_PH_PERFCOUNTER5_SELECT, + R_03761C_PA_PH_PERFCOUNTER6_SELECT, + R_037620_PA_PH_PERFCOUNTER7_SELECT, +}; +static struct si_pc_block_base gfx10_PA_PH = { + .name = "PA_PH", + .num_counters = 8, + .flags = SI_PC_BLOCK_SE, + + .select = gfx10_PA_PH_select, + .counter0_lo = R_035600_PA_PH_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_CUSTOM, +}; + +static struct si_pc_block_base gfx10_PA_SU = { + .name = "PA_SU", + .num_counters = 4, + .flags = SI_PC_BLOCK_SE, + + .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT, + .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO, + .num_multi = 4, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_RLC = { + .name = "RLC", + .num_counters = 2, + + .select0 = R_037304_RLC_PERFCOUNTER0_SELECT, + .counter0_lo = R_035200_RLC_PERFCOUNTER0_LO, + .num_multi = 0, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_RMI = { + .name = "RMI", + /* Actually 4, but the 2nd counter is missing the secondary selector while + * the 3rd counter has it, which complicates the register layout. */ + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS, + + .select0 = R_037400_RMI_PERFCOUNTER0_SELECT, + .counter0_lo = R_035300_RMI_PERFCOUNTER0_LO, + .num_multi = 1, + .layout = SI_PC_MULTI_ALTERNATE, +}; + +static struct si_pc_block_base gfx10_UTCL1 = { + .name = "UTCL1", + .num_counters = 2, + .flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER_WINDOWED, + + .select0 = R_03758C_UTCL1_PERFCOUNTER0_SELECT, + .counter0_lo = R_035470_UTCL1_PERFCOUNTER0_LO, + .num_multi = 0, + .layout = SI_PC_MULTI_ALTERNATE, }; /* Both the number of instances and selectors varies between chips of the same @@ -411,954 +594,924 @@ static struct si_pc_block_base cik_SRBM = { * blocks here matters. */ static struct si_pc_block_gfxdescr groups_CIK[] = { - { &cik_CB, 226}, - { &cik_CPF, 17 }, - { &cik_DB, 257}, - { &cik_GRBM, 34 }, - { &cik_GRBMSE, 15 }, - { &cik_PA_SU, 153 }, - { &cik_PA_SC, 395 }, - { &cik_SPI, 186 }, - { &cik_SQ, 252 }, - { &cik_SX, 32 }, - { &cik_TA, 111, 11 }, - { &cik_TCA, 39, 2 }, - { &cik_TCC, 160}, - { &cik_TD, 55, 11 }, - { &cik_TCP, 154, 11 }, - { &cik_GDS, 121 }, - { &cik_VGT, 140 }, - { &cik_IA, 22 }, - { &cik_MC, 22 }, - { &cik_SRBM, 19 }, - { &cik_WD, 22 }, - { &cik_CPG, 46 }, - { &cik_CPC, 22 }, + {&cik_CB, 226}, {&cik_CPF, 17}, {&cik_DB, 257}, {&cik_GRBM, 34}, {&cik_GRBMSE, 15}, + {&cik_PA_SU, 153}, {&cik_PA_SC, 395}, {&cik_SPI, 186}, {&cik_SQ, 252}, {&cik_SX, 32}, + {&cik_TA, 111}, {&cik_TCA, 39, 2}, {&cik_TCC, 160}, {&cik_TD, 55}, {&cik_TCP, 154}, + {&cik_GDS, 121}, {&cik_VGT, 140}, {&cik_IA, 22}, {&cik_MC, 22}, {&cik_SRBM, 19}, + {&cik_WD, 22}, {&cik_CPG, 46}, {&cik_CPC, 22}, }; static struct si_pc_block_gfxdescr groups_VI[] = { - { &cik_CB, 405}, - { &cik_CPF, 19 }, - { &cik_DB, 257}, - { &cik_GRBM, 34 }, - { &cik_GRBMSE, 15 }, - { &cik_PA_SU, 154 }, - { &cik_PA_SC, 397 }, - { &cik_SPI, 197 }, - { &cik_SQ, 273 }, - { &cik_SX, 34 }, - { &cik_TA, 119, 16 }, - { &cik_TCA, 35, 2 }, - { &cik_TCC, 192}, - { &cik_TD, 55, 16 }, - { &cik_TCP, 180, 16 }, - { &cik_GDS, 121 }, - { &cik_VGT, 147 }, - { &cik_IA, 24 }, - { &cik_MC, 22 }, - { &cik_SRBM, 27 }, - { &cik_WD, 37 }, - { &cik_CPG, 48 }, - { &cik_CPC, 24 }, + {&cik_CB, 405}, {&cik_CPF, 19}, {&cik_DB, 257}, {&cik_GRBM, 34}, {&cik_GRBMSE, 15}, + {&cik_PA_SU, 154}, {&cik_PA_SC, 397}, {&cik_SPI, 197}, {&cik_SQ, 273}, {&cik_SX, 34}, + {&cik_TA, 119}, {&cik_TCA, 35, 2}, {&cik_TCC, 192}, {&cik_TD, 55}, {&cik_TCP, 180}, + {&cik_GDS, 121}, {&cik_VGT, 147}, {&cik_IA, 24}, {&cik_MC, 22}, {&cik_SRBM, 27}, + {&cik_WD, 37}, {&cik_CPG, 48}, {&cik_CPC, 24}, }; static struct si_pc_block_gfxdescr groups_gfx9[] = { - { &cik_CB, 438}, - { &cik_CPF, 32 }, - { &cik_DB, 328}, - { &cik_GRBM, 38 }, - { &cik_GRBMSE, 16 }, - { &cik_PA_SU, 292 }, - { &cik_PA_SC, 491 }, - { &cik_SPI, 196 }, - { &cik_SQ, 374 }, - { &cik_SX, 208 }, - { &cik_TA, 119, 16 }, - { &cik_TCA, 35, 2 }, - { &cik_TCC, 256}, - { &cik_TD, 57, 16 }, - { &cik_TCP, 85, 16 }, - { &cik_GDS, 121 }, - { &cik_VGT, 148 }, - { &cik_IA, 32 }, - { &cik_WD, 58 }, - { &cik_CPG, 59 }, - { &cik_CPC, 35 }, + {&cik_CB, 438}, {&cik_CPF, 32}, {&cik_DB, 328}, {&cik_GRBM, 38}, {&cik_GRBMSE, 16}, + {&cik_PA_SU, 292}, {&cik_PA_SC, 491}, {&cik_SPI, 196}, {&cik_SQ, 374}, {&cik_SX, 208}, + {&cik_TA, 119}, {&cik_TCA, 35, 2}, {&cik_TCC, 256}, {&cik_TD, 57}, {&cik_TCP, 85}, + {&cik_GDS, 121}, {&cik_VGT, 148}, {&cik_IA, 32}, {&cik_WD, 58}, {&cik_CPG, 59}, + {&cik_CPC, 35}, +}; + +static struct si_pc_block_gfxdescr groups_gfx10[] = { + {&cik_CB, 461}, + {&gfx10_CHA, 45}, + {&gfx10_CHCG, 35}, + {&gfx10_CHC, 35}, + {&cik_CPC, 47}, + {&cik_CPF, 40}, + {&cik_CPG, 82}, + {&cik_DB, 370}, + {&gfx10_GCR, 94}, + {&cik_GDS, 123}, + {&gfx10_GE, 315}, + {&gfx10_GL1A, 36}, + {&gfx10_GL1C, 64}, + {&gfx10_GL2A, 91}, + {&gfx10_GL2C, 235}, + {&cik_GRBM, 47}, + {&cik_GRBMSE, 19}, + {&gfx10_PA_PH, 960}, + {&cik_PA_SC, 552}, + {&gfx10_PA_SU, 266}, + {&gfx10_RLC, 7}, + {&gfx10_RMI, 258}, + {&cik_SPI, 329}, + {&cik_SQ, 509}, + {&cik_SX, 225}, + {&cik_TA, 226}, + {&cik_TCP, 77}, + {&cik_TD, 61}, + {&gfx10_UTCL1, 15}, }; static bool si_pc_block_has_per_se_groups(const struct si_perfcounters *pc, - const struct si_pc_block *block) + const struct si_pc_block *block) { - return block->b->b->flags & SI_PC_BLOCK_SE_GROUPS || - (block->b->b->flags & SI_PC_BLOCK_SE && pc->separate_se); + return block->b->b->flags & SI_PC_BLOCK_SE_GROUPS || + (block->b->b->flags & SI_PC_BLOCK_SE && pc->separate_se); } static bool si_pc_block_has_per_instance_groups(const struct si_perfcounters *pc, - const struct si_pc_block *block) + const struct si_pc_block *block) { - return block->b->b->flags & SI_PC_BLOCK_INSTANCE_GROUPS || - (block->num_instances > 1 && pc->separate_instance); + return block->b->b->flags & SI_PC_BLOCK_INSTANCE_GROUPS || + (block->num_instances > 1 && pc->separate_instance); } -static struct si_pc_block * -lookup_counter(struct si_perfcounters *pc, unsigned index, - unsigned *base_gid, unsigned *sub_index) +static struct si_pc_block *lookup_counter(struct si_perfcounters *pc, unsigned index, + unsigned *base_gid, unsigned *sub_index) { - struct si_pc_block *block = pc->blocks; - unsigned bid; + struct si_pc_block *block = pc->blocks; + unsigned bid; - *base_gid = 0; - for (bid = 0; bid < pc->num_blocks; ++bid, ++block) { - unsigned total = block->num_groups * block->b->selectors; + *base_gid = 0; + for (bid = 0; bid < pc->num_blocks; ++bid, ++block) { + unsigned total = block->num_groups * block->b->selectors; - if (index < total) { - *sub_index = index; - return block; - } + if (index < total) { + *sub_index = index; + return block; + } - index -= total; - *base_gid += block->num_groups; - } + index -= total; + *base_gid += block->num_groups; + } - return NULL; + return NULL; } -static struct si_pc_block * -lookup_group(struct si_perfcounters *pc, unsigned *index) +static struct si_pc_block *lookup_group(struct si_perfcounters *pc, unsigned *index) { - unsigned bid; - struct si_pc_block *block = pc->blocks; + unsigned bid; + struct si_pc_block *block = pc->blocks; - for (bid = 0; bid < pc->num_blocks; ++bid, ++block) { - if (*index < block->num_groups) - return block; - *index -= block->num_groups; - } + for (bid = 0; bid < pc->num_blocks; ++bid, ++block) { + if (*index < block->num_groups) + return block; + *index -= block->num_groups; + } - return NULL; + return NULL; } -static void si_pc_emit_instance(struct si_context *sctx, - int se, int instance) +static void si_pc_emit_instance(struct si_context *sctx, int se, int instance) { - struct radeon_cmdbuf *cs = sctx->gfx_cs; - unsigned value = S_030800_SH_BROADCAST_WRITES(1); - - if (se >= 0) { - value |= S_030800_SE_INDEX(se); - } else { - value |= S_030800_SE_BROADCAST_WRITES(1); - } - - if (instance >= 0) { - value |= S_030800_INSTANCE_INDEX(instance); - } else { - value |= S_030800_INSTANCE_BROADCAST_WRITES(1); - } - - radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value); + struct radeon_cmdbuf *cs = sctx->gfx_cs; + unsigned value = S_030800_SH_BROADCAST_WRITES(1); + + if (se >= 0) { + value |= S_030800_SE_INDEX(se); + } else { + value |= S_030800_SE_BROADCAST_WRITES(1); + } + + if (sctx->chip_class >= GFX10) { + /* TODO: Expose counters from each shader array separately if needed. */ + value |= S_030800_SA_BROADCAST_WRITES(1); + } + + if (instance >= 0) { + value |= S_030800_INSTANCE_INDEX(instance); + } else { + value |= S_030800_INSTANCE_BROADCAST_WRITES(1); + } + + radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value); } -static void si_pc_emit_shaders(struct si_context *sctx, - unsigned shaders) +static void si_pc_emit_shaders(struct si_context *sctx, unsigned shaders) { - struct radeon_cmdbuf *cs = sctx->gfx_cs; + struct radeon_cmdbuf *cs = sctx->gfx_cs; - radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2); - radeon_emit(cs, shaders & 0x7f); - radeon_emit(cs, 0xffffffff); + radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2); + radeon_emit(cs, shaders & 0x7f); + radeon_emit(cs, 0xffffffff); } -static void si_pc_emit_select(struct si_context *sctx, - struct si_pc_block *block, - unsigned count, unsigned *selectors) +static void si_pc_emit_select(struct si_context *sctx, struct si_pc_block *block, unsigned count, + unsigned *selectors) { - struct si_pc_block_base *regs = block->b->b; - struct radeon_cmdbuf *cs = sctx->gfx_cs; - unsigned idx; - unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK; - unsigned dw; - - assert(count <= regs->num_counters); - - if (regs->layout & SI_PC_FAKE) - return; - - if (layout_multi == SI_PC_MULTI_BLOCK) { - assert(!(regs->layout & SI_PC_REG_REVERSE)); - - dw = count + regs->num_prelude; - if (count >= regs->num_multi) - dw += regs->num_multi; - radeon_set_uconfig_reg_seq(cs, regs->select0, dw); - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) - radeon_emit(cs, selectors[idx] | regs->select_or); - - if (count < regs->num_multi) { - unsigned select1 = - regs->select0 + 4 * regs->num_multi; - radeon_set_uconfig_reg_seq(cs, select1, count); - } - - for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) - radeon_emit(cs, 0); - - if (count > regs->num_multi) { - for (idx = regs->num_multi; idx < count; ++idx) - radeon_emit(cs, selectors[idx] | regs->select_or); - } - } else if (layout_multi == SI_PC_MULTI_TAIL) { - unsigned select1, select1_count; - - assert(!(regs->layout & SI_PC_REG_REVERSE)); - - radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude); - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - for (idx = 0; idx < count; ++idx) - radeon_emit(cs, selectors[idx] | regs->select_or); - - select1 = regs->select0 + 4 * regs->num_counters; - select1_count = MIN2(count, regs->num_multi); - radeon_set_uconfig_reg_seq(cs, select1, select1_count); - for (idx = 0; idx < select1_count; ++idx) - radeon_emit(cs, 0); - } else if (layout_multi == SI_PC_MULTI_CUSTOM) { - unsigned *reg = regs->select; - for (idx = 0; idx < count; ++idx) { - radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or); - if (idx < regs->num_multi) - radeon_set_uconfig_reg(cs, *reg++, 0); - } - } else { - assert(layout_multi == SI_PC_MULTI_ALTERNATE); - - unsigned reg_base = regs->select0; - unsigned reg_count = count + MIN2(count, regs->num_multi); - reg_count += regs->num_prelude; - - if (!(regs->layout & SI_PC_REG_REVERSE)) { - radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); - - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - for (idx = 0; idx < count; ++idx) { - radeon_emit(cs, selectors[idx] | regs->select_or); - if (idx < regs->num_multi) - radeon_emit(cs, 0); - } - } else { - reg_base -= (reg_count - 1) * 4; - radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); - - for (idx = count; idx > 0; --idx) { - if (idx <= regs->num_multi) - radeon_emit(cs, 0); - radeon_emit(cs, selectors[idx - 1] | regs->select_or); - } - for (idx = 0; idx < regs->num_prelude; ++idx) - radeon_emit(cs, 0); - } - } + struct si_pc_block_base *regs = block->b->b; + struct radeon_cmdbuf *cs = sctx->gfx_cs; + unsigned idx; + unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK; + unsigned dw; + + assert(count <= regs->num_counters); + + if (regs->layout & SI_PC_FAKE) + return; + + if (layout_multi == SI_PC_MULTI_BLOCK) { + assert(!(regs->layout & SI_PC_REG_REVERSE)); + + dw = count + regs->num_prelude; + if (count >= regs->num_multi) + dw += regs->num_multi; + radeon_set_uconfig_reg_seq(cs, regs->select0, dw); + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) + radeon_emit(cs, selectors[idx] | regs->select_or); + + if (count < regs->num_multi) { + unsigned select1 = regs->select0 + 4 * regs->num_multi; + radeon_set_uconfig_reg_seq(cs, select1, count); + } + + for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx) + radeon_emit(cs, 0); + + if (count > regs->num_multi) { + for (idx = regs->num_multi; idx < count; ++idx) + radeon_emit(cs, selectors[idx] | regs->select_or); + } + } else if (layout_multi == SI_PC_MULTI_TAIL) { + unsigned select1, select1_count; + + assert(!(regs->layout & SI_PC_REG_REVERSE)); + + radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude); + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + for (idx = 0; idx < count; ++idx) + radeon_emit(cs, selectors[idx] | regs->select_or); + + select1 = regs->select0 + 4 * regs->num_counters; + select1_count = MIN2(count, regs->num_multi); + radeon_set_uconfig_reg_seq(cs, select1, select1_count); + for (idx = 0; idx < select1_count; ++idx) + radeon_emit(cs, 0); + } else if (layout_multi == SI_PC_MULTI_CUSTOM) { + unsigned *reg = regs->select; + for (idx = 0; idx < count; ++idx) { + radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or); + if (idx < regs->num_multi) + radeon_set_uconfig_reg(cs, *reg++, 0); + } + } else { + assert(layout_multi == SI_PC_MULTI_ALTERNATE); + + unsigned reg_base = regs->select0; + unsigned reg_count = count + MIN2(count, regs->num_multi); + reg_count += regs->num_prelude; + + if (!(regs->layout & SI_PC_REG_REVERSE)) { + radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); + + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + for (idx = 0; idx < count; ++idx) { + radeon_emit(cs, selectors[idx] | regs->select_or); + if (idx < regs->num_multi) + radeon_emit(cs, 0); + } + } else { + reg_base -= (reg_count - 1) * 4; + radeon_set_uconfig_reg_seq(cs, reg_base, reg_count); + + for (idx = count; idx > 0; --idx) { + if (idx <= regs->num_multi) + radeon_emit(cs, 0); + radeon_emit(cs, selectors[idx - 1] | regs->select_or); + } + for (idx = 0; idx < regs->num_prelude; ++idx) + radeon_emit(cs, 0); + } + } } -static void si_pc_emit_start(struct si_context *sctx, - struct r600_resource *buffer, uint64_t va) +static void si_pc_emit_start(struct si_context *sctx, struct si_resource *buffer, uint64_t va) { - struct radeon_cmdbuf *cs = sctx->gfx_cs; - - radeon_add_to_buffer_list(sctx, sctx->gfx_cs, buffer, - RADEON_USAGE_WRITE, RADEON_PRIO_QUERY); - - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | - COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM)); - radeon_emit(cs, 1); /* immediate */ - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, - S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET)); - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_START) | EVENT_INDEX(0)); - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, - S_036020_PERFMON_STATE(V_036020_START_COUNTING)); + struct radeon_cmdbuf *cs = sctx->gfx_cs; + + si_cp_copy_data(sctx, sctx->gfx_cs, COPY_DATA_DST_MEM, buffer, va - buffer->gpu_address, + COPY_DATA_IMM, NULL, 1); + + radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, + S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET)); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_START) | EVENT_INDEX(0)); + radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, + S_036020_PERFMON_STATE(V_036020_START_COUNTING)); } /* Note: The buffer was already added in si_pc_emit_start, so we don't have to * do it again in here. */ -static void si_pc_emit_stop(struct si_context *sctx, - struct r600_resource *buffer, uint64_t va) +static void si_pc_emit_stop(struct si_context *sctx, struct si_resource *buffer, uint64_t va) { - struct radeon_cmdbuf *cs = sctx->gfx_cs; - - si_cp_release_mem(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0, - EOP_DST_SEL_MEM, EOP_INT_SEL_NONE, - EOP_DATA_SEL_VALUE_32BIT, - buffer, va, 0, SI_NOT_QUERY); - si_cp_wait_mem(sctx, cs, va, 0, 0xffffffff, WAIT_REG_MEM_EQUAL); - - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0)); - radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0)); - radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL, - S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) | - S_036020_PERFMON_SAMPLE_ENABLE(1)); + struct radeon_cmdbuf *cs = sctx->gfx_cs; + + si_cp_release_mem(sctx, cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE, + EOP_DATA_SEL_VALUE_32BIT, buffer, va, 0, SI_NOT_QUERY); + si_cp_wait_mem(sctx, cs, va, 0, 0xffffffff, WAIT_REG_MEM_EQUAL); + + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0)); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0)); + radeon_set_uconfig_reg( + cs, R_036020_CP_PERFMON_CNTL, + S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1)); } -static void si_pc_emit_read(struct si_context *sctx, - struct si_pc_block *block, - unsigned count, uint64_t va) +static void si_pc_emit_read(struct si_context *sctx, struct si_pc_block *block, unsigned count, + uint64_t va) { - struct si_pc_block_base *regs = block->b->b; - struct radeon_cmdbuf *cs = sctx->gfx_cs; - unsigned idx; - unsigned reg = regs->counter0_lo; - unsigned reg_delta = 8; - - if (!(regs->layout & SI_PC_FAKE)) { - if (regs->layout & SI_PC_REG_REVERSE) - reg_delta = -reg_delta; - - for (idx = 0; idx < count; ++idx) { - if (regs->counters) - reg = regs->counters[idx]; - - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | - COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM) | - COPY_DATA_COUNT_SEL); /* 64 bits */ - radeon_emit(cs, reg >> 2); - radeon_emit(cs, 0); /* unused */ - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - va += sizeof(uint64_t); - reg += reg_delta; - } - } else { - for (idx = 0; idx < count; ++idx) { - radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | - COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM) | - COPY_DATA_COUNT_SEL); - radeon_emit(cs, 0); /* immediate */ - radeon_emit(cs, 0); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - va += sizeof(uint64_t); - } - } + struct si_pc_block_base *regs = block->b->b; + struct radeon_cmdbuf *cs = sctx->gfx_cs; + unsigned idx; + unsigned reg = regs->counter0_lo; + unsigned reg_delta = 8; + + if (!(regs->layout & SI_PC_FAKE)) { + if (regs->layout & SI_PC_REG_REVERSE) + reg_delta = -reg_delta; + + for (idx = 0; idx < count; ++idx) { + if (regs->counters) + reg = regs->counters[idx]; + + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_COUNT_SEL); /* 64 bits */ + radeon_emit(cs, reg >> 2); + radeon_emit(cs, 0); /* unused */ + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + va += sizeof(uint64_t); + reg += reg_delta; + } + } else { + for (idx = 0; idx < count; ++idx) { + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) | + COPY_DATA_COUNT_SEL); + radeon_emit(cs, 0); /* immediate */ + radeon_emit(cs, 0); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + va += sizeof(uint64_t); + } + } } -static void si_pc_query_destroy(struct si_screen *sscreen, - struct si_query *rquery) +static void si_pc_query_destroy(struct si_context *sctx, struct si_query *squery) { - struct si_query_pc *query = (struct si_query_pc *)rquery; + struct si_query_pc *query = (struct si_query_pc *)squery; - while (query->groups) { - struct si_query_group *group = query->groups; - query->groups = group->next; - FREE(group); - } + while (query->groups) { + struct si_query_group *group = query->groups; + query->groups = group->next; + FREE(group); + } - FREE(query->counters); + FREE(query->counters); - si_query_buffer_destroy(sscreen, &query->buffer); - FREE(query); + si_query_buffer_destroy(sctx->screen, &query->buffer); + FREE(query); } -static void si_pc_query_resume(struct si_context *sctx, struct si_query *rquery) +static void si_inhibit_clockgating(struct si_context *sctx, bool inhibit) +{ + if (sctx->chip_class >= GFX10) { + radeon_set_uconfig_reg(sctx->gfx_cs, R_037390_RLC_PERFMON_CLK_CNTL, + S_037390_PERFMON_CLOCK_STATE(inhibit)); + } else if (sctx->chip_class >= GFX8) { + radeon_set_uconfig_reg(sctx->gfx_cs, R_0372FC_RLC_PERFMON_CLK_CNTL, + S_0372FC_PERFMON_CLOCK_STATE(inhibit)); + } +} + +static void si_pc_query_resume(struct si_context *sctx, struct si_query *squery) /* - struct si_query_hw *hwquery, - struct r600_resource *buffer, uint64_t va)*/ + struct si_query_hw *hwquery, + struct si_resource *buffer, uint64_t va)*/ { - struct si_query_pc *query = (struct si_query_pc *)rquery; - int current_se = -1; - int current_instance = -1; + struct si_query_pc *query = (struct si_query_pc *)squery; + int current_se = -1; + int current_instance = -1; + + if (!si_query_buffer_alloc(sctx, &query->buffer, NULL, query->result_size)) + return; + si_need_gfx_cs_space(sctx); - if (!si_query_buffer_alloc(sctx, &query->buffer, NULL, query->result_size)) - return; - si_need_gfx_cs_space(sctx); + if (query->shaders) + si_pc_emit_shaders(sctx, query->shaders); - if (query->shaders) - si_pc_emit_shaders(sctx, query->shaders); + si_inhibit_clockgating(sctx, true); - for (struct si_query_group *group = query->groups; group; group = group->next) { - struct si_pc_block *block = group->block; + for (struct si_query_group *group = query->groups; group; group = group->next) { + struct si_pc_block *block = group->block; - if (group->se != current_se || group->instance != current_instance) { - current_se = group->se; - current_instance = group->instance; - si_pc_emit_instance(sctx, group->se, group->instance); - } + if (group->se != current_se || group->instance != current_instance) { + current_se = group->se; + current_instance = group->instance; + si_pc_emit_instance(sctx, group->se, group->instance); + } - si_pc_emit_select(sctx, block, group->num_counters, group->selectors); - } + si_pc_emit_select(sctx, block, group->num_counters, group->selectors); + } - if (current_se != -1 || current_instance != -1) - si_pc_emit_instance(sctx, -1, -1); + if (current_se != -1 || current_instance != -1) + si_pc_emit_instance(sctx, -1, -1); - uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end; - si_pc_emit_start(sctx, query->buffer.buf, va); + uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end; + si_pc_emit_start(sctx, query->buffer.buf, va); } -static void si_pc_query_suspend(struct si_context *sctx, struct si_query *rquery) +static void si_pc_query_suspend(struct si_context *sctx, struct si_query *squery) { - struct si_query_pc *query = (struct si_query_pc *)rquery; + struct si_query_pc *query = (struct si_query_pc *)squery; + + if (!query->buffer.buf) + return; - if (!query->buffer.buf) - return; + uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end; + query->buffer.results_end += query->result_size; - uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end; - query->buffer.results_end += query->result_size; + si_pc_emit_stop(sctx, query->buffer.buf, va); - si_pc_emit_stop(sctx, query->buffer.buf, va); + for (struct si_query_group *group = query->groups; group; group = group->next) { + struct si_pc_block *block = group->block; + unsigned se = group->se >= 0 ? group->se : 0; + unsigned se_end = se + 1; - for (struct si_query_group *group = query->groups; group; group = group->next) { - struct si_pc_block *block = group->block; - unsigned se = group->se >= 0 ? group->se : 0; - unsigned se_end = se + 1; + if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0)) + se_end = sctx->screen->info.max_se; - if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0)) - se_end = sctx->screen->info.max_se; + do { + unsigned instance = group->instance >= 0 ? group->instance : 0; - do { - unsigned instance = group->instance >= 0 ? group->instance : 0; + do { + si_pc_emit_instance(sctx, se, instance); + si_pc_emit_read(sctx, block, group->num_counters, va); + va += sizeof(uint64_t) * group->num_counters; + } while (group->instance < 0 && ++instance < block->num_instances); + } while (++se < se_end); + } - do { - si_pc_emit_instance(sctx, se, instance); - si_pc_emit_read(sctx, block, group->num_counters, va); - va += sizeof(uint64_t) * group->num_counters; - } while (group->instance < 0 && ++instance < block->num_instances); - } while (++se < se_end); - } + si_pc_emit_instance(sctx, -1, -1); - si_pc_emit_instance(sctx, -1, -1); + si_inhibit_clockgating(sctx, false); } -static bool si_pc_query_begin(struct si_context *ctx, struct si_query *rquery) +static bool si_pc_query_begin(struct si_context *ctx, struct si_query *squery) { - struct si_query_pc *query = (struct si_query_pc *)rquery; + struct si_query_pc *query = (struct si_query_pc *)squery; - si_query_buffer_reset(ctx, &query->buffer); + si_query_buffer_reset(ctx, &query->buffer); - LIST_ADDTAIL(&query->b.active_list, &ctx->active_queries); - ctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend; + list_addtail(&query->b.active_list, &ctx->active_queries); + ctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend; - si_pc_query_resume(ctx, rquery); + si_pc_query_resume(ctx, squery); - return true; + return true; } -static bool si_pc_query_end(struct si_context *ctx, struct si_query *rquery) +static bool si_pc_query_end(struct si_context *ctx, struct si_query *squery) { - struct si_query_pc *query = (struct si_query_pc *)rquery; + struct si_query_pc *query = (struct si_query_pc *)squery; - si_pc_query_suspend(ctx, rquery); + si_pc_query_suspend(ctx, squery); - LIST_DEL(&rquery->active_list); - ctx->num_cs_dw_queries_suspend -= rquery->num_cs_dw_suspend; + list_del(&squery->active_list); + ctx->num_cs_dw_queries_suspend -= squery->num_cs_dw_suspend; - return query->buffer.buf != NULL; + return query->buffer.buf != NULL; } -static void si_pc_query_add_result(struct si_query_pc *query, - void *buffer, - union pipe_query_result *result) +static void si_pc_query_add_result(struct si_query_pc *query, void *buffer, + union pipe_query_result *result) { - uint64_t *results = buffer; - unsigned i, j; + uint64_t *results = buffer; + unsigned i, j; - for (i = 0; i < query->num_counters; ++i) { - struct si_query_counter *counter = &query->counters[i]; + for (i = 0; i < query->num_counters; ++i) { + struct si_query_counter *counter = &query->counters[i]; - for (j = 0; j < counter->qwords; ++j) { - uint32_t value = results[counter->base + j * counter->stride]; - result->batch[i].u64 += value; - } - } + for (j = 0; j < counter->qwords; ++j) { + uint32_t value = results[counter->base + j * counter->stride]; + result->batch[i].u64 += value; + } + } } -static bool si_pc_query_get_result(struct si_context *sctx, struct si_query *rquery, - bool wait, union pipe_query_result *result) +static bool si_pc_query_get_result(struct si_context *sctx, struct si_query *squery, bool wait, + union pipe_query_result *result) { - struct si_query_pc *query = (struct si_query_pc *)rquery; + struct si_query_pc *query = (struct si_query_pc *)squery; - memset(result, 0, sizeof(result->batch[0]) * query->num_counters); + memset(result, 0, sizeof(result->batch[0]) * query->num_counters); - for (struct si_query_buffer *qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) { - unsigned usage = PIPE_TRANSFER_READ | - (wait ? 0 : PIPE_TRANSFER_DONTBLOCK); - unsigned results_base = 0; - void *map; + for (struct si_query_buffer *qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) { + unsigned usage = PIPE_TRANSFER_READ | (wait ? 0 : PIPE_TRANSFER_DONTBLOCK); + unsigned results_base = 0; + void *map; - if (rquery->b.flushed) - map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage); - else - map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage); + if (squery->b.flushed) + map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage); + else + map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage); - if (!map) - return false; + if (!map) + return false; - while (results_base != qbuf->results_end) { - si_pc_query_add_result(query, map + results_base, result); - results_base += query->result_size; - } - } + while (results_base != qbuf->results_end) { + si_pc_query_add_result(query, map + results_base, result); + results_base += query->result_size; + } + } - return true; + return true; } static const struct si_query_ops batch_query_ops = { - .destroy = si_pc_query_destroy, - .begin = si_pc_query_begin, - .end = si_pc_query_end, - .get_result = si_pc_query_get_result, + .destroy = si_pc_query_destroy, + .begin = si_pc_query_begin, + .end = si_pc_query_end, + .get_result = si_pc_query_get_result, - .suspend = si_pc_query_suspend, - .resume = si_pc_query_resume, + .suspend = si_pc_query_suspend, + .resume = si_pc_query_resume, }; -static struct si_query_group *get_group_state(struct si_screen *screen, - struct si_query_pc *query, - struct si_pc_block *block, - unsigned sub_gid) +static struct si_query_group *get_group_state(struct si_screen *screen, struct si_query_pc *query, + struct si_pc_block *block, unsigned sub_gid) { - struct si_query_group *group = query->groups; - - while (group) { - if (group->block == block && group->sub_gid == sub_gid) - return group; - group = group->next; - } - - group = CALLOC_STRUCT(si_query_group); - if (!group) - return NULL; - - group->block = block; - group->sub_gid = sub_gid; - - if (block->b->b->flags & SI_PC_BLOCK_SHADER) { - unsigned sub_gids = block->num_instances; - unsigned shader_id; - unsigned shaders; - unsigned query_shaders; - - if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) - sub_gids = sub_gids * screen->info.max_se; - shader_id = sub_gid / sub_gids; - sub_gid = sub_gid % sub_gids; - - shaders = si_pc_shader_type_bits[shader_id]; - - query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING; - if (query_shaders && query_shaders != shaders) { - fprintf(stderr, "si_perfcounter: incompatible shader groups\n"); - FREE(group); - return NULL; - } - query->shaders = shaders; - } - - if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) { - // A non-zero value in query->shaders ensures that the shader - // masking is reset unless the user explicitly requests one. - query->shaders = SI_PC_SHADERS_WINDOWING; - } - - if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) { - group->se = sub_gid / block->num_instances; - sub_gid = sub_gid % block->num_instances; - } else { - group->se = -1; - } - - if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) { - group->instance = sub_gid; - } else { - group->instance = -1; - } - - group->next = query->groups; - query->groups = group; - - return group; + struct si_query_group *group = query->groups; + + while (group) { + if (group->block == block && group->sub_gid == sub_gid) + return group; + group = group->next; + } + + group = CALLOC_STRUCT(si_query_group); + if (!group) + return NULL; + + group->block = block; + group->sub_gid = sub_gid; + + if (block->b->b->flags & SI_PC_BLOCK_SHADER) { + unsigned sub_gids = block->num_instances; + unsigned shader_id; + unsigned shaders; + unsigned query_shaders; + + if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) + sub_gids = sub_gids * screen->info.max_se; + shader_id = sub_gid / sub_gids; + sub_gid = sub_gid % sub_gids; + + shaders = si_pc_shader_type_bits[shader_id]; + + query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING; + if (query_shaders && query_shaders != shaders) { + fprintf(stderr, "si_perfcounter: incompatible shader groups\n"); + FREE(group); + return NULL; + } + query->shaders = shaders; + } + + if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) { + // A non-zero value in query->shaders ensures that the shader + // masking is reset unless the user explicitly requests one. + query->shaders = SI_PC_SHADERS_WINDOWING; + } + + if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) { + group->se = sub_gid / block->num_instances; + sub_gid = sub_gid % block->num_instances; + } else { + group->se = -1; + } + + if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) { + group->instance = sub_gid; + } else { + group->instance = -1; + } + + group->next = query->groups; + query->groups = group; + + return group; } -struct pipe_query *si_create_batch_query(struct pipe_context *ctx, - unsigned num_queries, - unsigned *query_types) +struct pipe_query *si_create_batch_query(struct pipe_context *ctx, unsigned num_queries, + unsigned *query_types) { - struct si_screen *screen = - (struct si_screen *)ctx->screen; - struct si_perfcounters *pc = screen->perfcounters; - struct si_pc_block *block; - struct si_query_group *group; - struct si_query_pc *query; - unsigned base_gid, sub_gid, sub_index; - unsigned i, j; - - if (!pc) - return NULL; - - query = CALLOC_STRUCT(si_query_pc); - if (!query) - return NULL; - - query->b.ops = &batch_query_ops; - - query->num_counters = num_queries; - - /* Collect selectors per group */ - for (i = 0; i < num_queries; ++i) { - unsigned sub_gid; - - if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER) - goto error; - - block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, - &base_gid, &sub_index); - if (!block) - goto error; - - sub_gid = sub_index / block->b->selectors; - sub_index = sub_index % block->b->selectors; - - group = get_group_state(screen, query, block, sub_gid); - if (!group) - goto error; - - if (group->num_counters >= block->b->b->num_counters) { - fprintf(stderr, - "perfcounter group %s: too many selected\n", - block->b->b->name); - goto error; - } - group->selectors[group->num_counters] = sub_index; - ++group->num_counters; - } - - /* Compute result bases and CS size per group */ - query->b.num_cs_dw_suspend = pc->num_stop_cs_dwords; - query->b.num_cs_dw_suspend += pc->num_instance_cs_dwords; - - i = 0; - for (group = query->groups; group; group = group->next) { - struct si_pc_block *block = group->block; - unsigned read_dw; - unsigned instances = 1; - - if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0) - instances = screen->info.max_se; - if (group->instance < 0) - instances *= block->num_instances; - - group->result_base = i; - query->result_size += sizeof(uint64_t) * instances * group->num_counters; - i += instances * group->num_counters; - - read_dw = 6 * group->num_counters; - query->b.num_cs_dw_suspend += instances * read_dw; - query->b.num_cs_dw_suspend += instances * pc->num_instance_cs_dwords; - } - - if (query->shaders) { - if (query->shaders == SI_PC_SHADERS_WINDOWING) - query->shaders = 0xffffffff; - } - - /* Map user-supplied query array to result indices */ - query->counters = CALLOC(num_queries, sizeof(*query->counters)); - for (i = 0; i < num_queries; ++i) { - struct si_query_counter *counter = &query->counters[i]; - struct si_pc_block *block; - - block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, - &base_gid, &sub_index); - - sub_gid = sub_index / block->b->selectors; - sub_index = sub_index % block->b->selectors; - - group = get_group_state(screen, query, block, sub_gid); - assert(group != NULL); - - for (j = 0; j < group->num_counters; ++j) { - if (group->selectors[j] == sub_index) - break; - } - - counter->base = group->result_base + j; - counter->stride = group->num_counters; - - counter->qwords = 1; - if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0) - counter->qwords = screen->info.max_se; - if (group->instance < 0) - counter->qwords *= block->num_instances; - } + struct si_screen *screen = (struct si_screen *)ctx->screen; + struct si_perfcounters *pc = screen->perfcounters; + struct si_pc_block *block; + struct si_query_group *group; + struct si_query_pc *query; + unsigned base_gid, sub_gid, sub_index; + unsigned i, j; + + if (!pc) + return NULL; + + query = CALLOC_STRUCT(si_query_pc); + if (!query) + return NULL; + + query->b.ops = &batch_query_ops; + + query->num_counters = num_queries; + + /* Collect selectors per group */ + for (i = 0; i < num_queries; ++i) { + unsigned sub_gid; + + if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER) + goto error; + + block = + lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, &base_gid, &sub_index); + if (!block) + goto error; + + sub_gid = sub_index / block->b->selectors; + sub_index = sub_index % block->b->selectors; + + group = get_group_state(screen, query, block, sub_gid); + if (!group) + goto error; + + if (group->num_counters >= block->b->b->num_counters) { + fprintf(stderr, "perfcounter group %s: too many selected\n", block->b->b->name); + goto error; + } + group->selectors[group->num_counters] = sub_index; + ++group->num_counters; + } + + /* Compute result bases and CS size per group */ + query->b.num_cs_dw_suspend = pc->num_stop_cs_dwords; + query->b.num_cs_dw_suspend += pc->num_instance_cs_dwords; + + i = 0; + for (group = query->groups; group; group = group->next) { + struct si_pc_block *block = group->block; + unsigned read_dw; + unsigned instances = 1; + + if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0) + instances = screen->info.max_se; + if (group->instance < 0) + instances *= block->num_instances; + + group->result_base = i; + query->result_size += sizeof(uint64_t) * instances * group->num_counters; + i += instances * group->num_counters; + + read_dw = 6 * group->num_counters; + query->b.num_cs_dw_suspend += instances * read_dw; + query->b.num_cs_dw_suspend += instances * pc->num_instance_cs_dwords; + } + + if (query->shaders) { + if (query->shaders == SI_PC_SHADERS_WINDOWING) + query->shaders = 0xffffffff; + } + + /* Map user-supplied query array to result indices */ + query->counters = CALLOC(num_queries, sizeof(*query->counters)); + for (i = 0; i < num_queries; ++i) { + struct si_query_counter *counter = &query->counters[i]; + struct si_pc_block *block; + + block = + lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, &base_gid, &sub_index); + + sub_gid = sub_index / block->b->selectors; + sub_index = sub_index % block->b->selectors; + + group = get_group_state(screen, query, block, sub_gid); + assert(group != NULL); + + for (j = 0; j < group->num_counters; ++j) { + if (group->selectors[j] == sub_index) + break; + } + + counter->base = group->result_base + j; + counter->stride = group->num_counters; + + counter->qwords = 1; + if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0) + counter->qwords = screen->info.max_se; + if (group->instance < 0) + counter->qwords *= block->num_instances; + } - return (struct pipe_query *)query; + return (struct pipe_query *)query; error: - si_pc_query_destroy(screen, &query->b); - return NULL; + si_pc_query_destroy((struct si_context *)ctx, &query->b); + return NULL; } -static bool si_init_block_names(struct si_screen *screen, - struct si_pc_block *block) +static bool si_init_block_names(struct si_screen *screen, struct si_pc_block *block) { - bool per_instance_groups = si_pc_block_has_per_instance_groups(screen->perfcounters, block); - bool per_se_groups = si_pc_block_has_per_se_groups(screen->perfcounters, block); - unsigned i, j, k; - unsigned groups_shader = 1, groups_se = 1, groups_instance = 1; - unsigned namelen; - char *groupname; - char *p; - - if (per_instance_groups) - groups_instance = block->num_instances; - if (per_se_groups) - groups_se = screen->info.max_se; - if (block->b->b->flags & SI_PC_BLOCK_SHADER) - groups_shader = ARRAY_SIZE(si_pc_shader_type_bits); - - namelen = strlen(block->b->b->name); - block->group_name_stride = namelen + 1; - if (block->b->b->flags & SI_PC_BLOCK_SHADER) - block->group_name_stride += 3; - if (per_se_groups) { - assert(groups_se <= 10); - block->group_name_stride += 1; - - if (per_instance_groups) - block->group_name_stride += 1; - } - if (per_instance_groups) { - assert(groups_instance <= 100); - block->group_name_stride += 2; - } - - block->group_names = MALLOC(block->num_groups * block->group_name_stride); - if (!block->group_names) - return false; - - groupname = block->group_names; - for (i = 0; i < groups_shader; ++i) { - const char *shader_suffix = si_pc_shader_type_suffixes[i]; - unsigned shaderlen = strlen(shader_suffix); - for (j = 0; j < groups_se; ++j) { - for (k = 0; k < groups_instance; ++k) { - strcpy(groupname, block->b->b->name); - p = groupname + namelen; - - if (block->b->b->flags & SI_PC_BLOCK_SHADER) { - strcpy(p, shader_suffix); - p += shaderlen; - } - - if (per_se_groups) { - p += sprintf(p, "%d", j); - if (per_instance_groups) - *p++ = '_'; - } - - if (per_instance_groups) - p += sprintf(p, "%d", k); - - groupname += block->group_name_stride; - } - } - } - - assert(block->b->selectors <= 1000); - block->selector_name_stride = block->group_name_stride + 4; - block->selector_names = MALLOC(block->num_groups * block->b->selectors * - block->selector_name_stride); - if (!block->selector_names) - return false; - - groupname = block->group_names; - p = block->selector_names; - for (i = 0; i < block->num_groups; ++i) { - for (j = 0; j < block->b->selectors; ++j) { - sprintf(p, "%s_%03d", groupname, j); - p += block->selector_name_stride; - } - groupname += block->group_name_stride; - } - - return true; + bool per_instance_groups = si_pc_block_has_per_instance_groups(screen->perfcounters, block); + bool per_se_groups = si_pc_block_has_per_se_groups(screen->perfcounters, block); + unsigned i, j, k; + unsigned groups_shader = 1, groups_se = 1, groups_instance = 1; + unsigned namelen; + char *groupname; + char *p; + + if (per_instance_groups) + groups_instance = block->num_instances; + if (per_se_groups) + groups_se = screen->info.max_se; + if (block->b->b->flags & SI_PC_BLOCK_SHADER) + groups_shader = ARRAY_SIZE(si_pc_shader_type_bits); + + namelen = strlen(block->b->b->name); + block->group_name_stride = namelen + 1; + if (block->b->b->flags & SI_PC_BLOCK_SHADER) + block->group_name_stride += 3; + if (per_se_groups) { + assert(groups_se <= 10); + block->group_name_stride += 1; + + if (per_instance_groups) + block->group_name_stride += 1; + } + if (per_instance_groups) { + assert(groups_instance <= 100); + block->group_name_stride += 2; + } + + block->group_names = MALLOC(block->num_groups * block->group_name_stride); + if (!block->group_names) + return false; + + groupname = block->group_names; + for (i = 0; i < groups_shader; ++i) { + const char *shader_suffix = si_pc_shader_type_suffixes[i]; + unsigned shaderlen = strlen(shader_suffix); + for (j = 0; j < groups_se; ++j) { + for (k = 0; k < groups_instance; ++k) { + strcpy(groupname, block->b->b->name); + p = groupname + namelen; + + if (block->b->b->flags & SI_PC_BLOCK_SHADER) { + strcpy(p, shader_suffix); + p += shaderlen; + } + + if (per_se_groups) { + p += sprintf(p, "%d", j); + if (per_instance_groups) + *p++ = '_'; + } + + if (per_instance_groups) + p += sprintf(p, "%d", k); + + groupname += block->group_name_stride; + } + } + } + + assert(block->b->selectors <= 1000); + block->selector_name_stride = block->group_name_stride + 4; + block->selector_names = + MALLOC(block->num_groups * block->b->selectors * block->selector_name_stride); + if (!block->selector_names) + return false; + + groupname = block->group_names; + p = block->selector_names; + for (i = 0; i < block->num_groups; ++i) { + for (j = 0; j < block->b->selectors; ++j) { + sprintf(p, "%s_%03d", groupname, j); + p += block->selector_name_stride; + } + groupname += block->group_name_stride; + } + + return true; } -int si_get_perfcounter_info(struct si_screen *screen, - unsigned index, - struct pipe_driver_query_info *info) +int si_get_perfcounter_info(struct si_screen *screen, unsigned index, + struct pipe_driver_query_info *info) { - struct si_perfcounters *pc = screen->perfcounters; - struct si_pc_block *block; - unsigned base_gid, sub; - - if (!pc) - return 0; - - if (!info) { - unsigned bid, num_queries = 0; - - for (bid = 0; bid < pc->num_blocks; ++bid) { - num_queries += pc->blocks[bid].b->selectors * - pc->blocks[bid].num_groups; - } - - return num_queries; - } - - block = lookup_counter(pc, index, &base_gid, &sub); - if (!block) - return 0; - - if (!block->selector_names) { - if (!si_init_block_names(screen, block)) - return 0; - } - info->name = block->selector_names + sub * block->selector_name_stride; - info->query_type = SI_QUERY_FIRST_PERFCOUNTER + index; - info->max_value.u64 = 0; - info->type = PIPE_DRIVER_QUERY_TYPE_UINT64; - info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE; - info->group_id = base_gid + sub / block->b->selectors; - info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH; - if (sub > 0 && sub + 1 < block->b->selectors * block->num_groups) - info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST; - return 1; + struct si_perfcounters *pc = screen->perfcounters; + struct si_pc_block *block; + unsigned base_gid, sub; + + if (!pc) + return 0; + + if (!info) { + unsigned bid, num_queries = 0; + + for (bid = 0; bid < pc->num_blocks; ++bid) { + num_queries += pc->blocks[bid].b->selectors * pc->blocks[bid].num_groups; + } + + return num_queries; + } + + block = lookup_counter(pc, index, &base_gid, &sub); + if (!block) + return 0; + + if (!block->selector_names) { + if (!si_init_block_names(screen, block)) + return 0; + } + info->name = block->selector_names + sub * block->selector_name_stride; + info->query_type = SI_QUERY_FIRST_PERFCOUNTER + index; + info->max_value.u64 = 0; + info->type = PIPE_DRIVER_QUERY_TYPE_UINT64; + info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE; + info->group_id = base_gid + sub / block->b->selectors; + info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH; + if (sub > 0 && sub + 1 < block->b->selectors * block->num_groups) + info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST; + return 1; } -int si_get_perfcounter_group_info(struct si_screen *screen, - unsigned index, - struct pipe_driver_query_group_info *info) +int si_get_perfcounter_group_info(struct si_screen *screen, unsigned index, + struct pipe_driver_query_group_info *info) { - struct si_perfcounters *pc = screen->perfcounters; - struct si_pc_block *block; - - if (!pc) - return 0; - - if (!info) - return pc->num_groups; - - block = lookup_group(pc, &index); - if (!block) - return 0; - - if (!block->group_names) { - if (!si_init_block_names(screen, block)) - return 0; - } - info->name = block->group_names + index * block->group_name_stride; - info->num_queries = block->b->selectors; - info->max_active_queries = block->b->b->num_counters; - return 1; + struct si_perfcounters *pc = screen->perfcounters; + struct si_pc_block *block; + + if (!pc) + return 0; + + if (!info) + return pc->num_groups; + + block = lookup_group(pc, &index); + if (!block) + return 0; + + if (!block->group_names) { + if (!si_init_block_names(screen, block)) + return 0; + } + info->name = block->group_names + index * block->group_name_stride; + info->num_queries = block->b->selectors; + info->max_active_queries = block->b->b->num_counters; + return 1; } void si_destroy_perfcounters(struct si_screen *screen) { - struct si_perfcounters *pc = screen->perfcounters; - unsigned i; - - if (!pc) - return; - - for (i = 0; i < pc->num_blocks; ++i) { - FREE(pc->blocks[i].group_names); - FREE(pc->blocks[i].selector_names); - } - FREE(pc->blocks); - FREE(pc); - screen->perfcounters = NULL; + struct si_perfcounters *pc = screen->perfcounters; + unsigned i; + + if (!pc) + return; + + for (i = 0; i < pc->num_blocks; ++i) { + FREE(pc->blocks[i].group_names); + FREE(pc->blocks[i].selector_names); + } + FREE(pc->blocks); + FREE(pc); + screen->perfcounters = NULL; } void si_init_perfcounters(struct si_screen *screen) { - struct si_perfcounters *pc; - const struct si_pc_block_gfxdescr *blocks; - unsigned num_blocks; - unsigned i; - - switch (screen->info.chip_class) { - case CIK: - blocks = groups_CIK; - num_blocks = ARRAY_SIZE(groups_CIK); - break; - case VI: - blocks = groups_VI; - num_blocks = ARRAY_SIZE(groups_VI); - break; - case GFX9: - blocks = groups_gfx9; - num_blocks = ARRAY_SIZE(groups_gfx9); - break; - case SI: - default: - return; /* not implemented */ - } - - if (screen->info.max_sh_per_se != 1) { - /* This should not happen on non-SI chips. */ - fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not " - "supported (inaccurate performance counters)\n", - screen->info.max_sh_per_se); - } - - screen->perfcounters = pc = CALLOC_STRUCT(si_perfcounters); - if (!pc) - return; - - pc->num_stop_cs_dwords = 14 + si_cp_write_fence_dwords(screen); - pc->num_instance_cs_dwords = 3; - - pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false); - pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false); - - pc->blocks = CALLOC(num_blocks, sizeof(struct si_pc_block)); - if (!pc->blocks) - goto error; - pc->num_blocks = num_blocks; - - for (i = 0; i < num_blocks; ++i) { - struct si_pc_block *block = &pc->blocks[i]; - block->b = &blocks[i]; - block->num_instances = block->b->instances; - - if (!strcmp(block->b->b->name, "CB") || - !strcmp(block->b->b->name, "DB")) - block->num_instances = screen->info.max_se; - else if (!strcmp(block->b->b->name, "TCC")) - block->num_instances = screen->info.num_tcc_blocks; - else if (!strcmp(block->b->b->name, "IA")) - block->num_instances = MAX2(1, screen->info.max_se / 2); - - if (si_pc_block_has_per_instance_groups(pc, block)) { - block->num_groups = block->num_instances; - } else { - block->num_groups = 1; - } - - if (si_pc_block_has_per_se_groups(pc, block)) - block->num_groups *= screen->info.max_se; - if (block->b->b->flags & SI_PC_BLOCK_SHADER) - block->num_groups *= ARRAY_SIZE(si_pc_shader_type_bits); - - pc->num_groups += block->num_groups; - } - - return; + struct si_perfcounters *pc; + const struct si_pc_block_gfxdescr *blocks; + unsigned num_blocks; + unsigned i; + + switch (screen->info.chip_class) { + case GFX7: + blocks = groups_CIK; + num_blocks = ARRAY_SIZE(groups_CIK); + break; + case GFX8: + blocks = groups_VI; + num_blocks = ARRAY_SIZE(groups_VI); + break; + case GFX9: + blocks = groups_gfx9; + num_blocks = ARRAY_SIZE(groups_gfx9); + break; + case GFX10: + case GFX10_3: + blocks = groups_gfx10; + num_blocks = ARRAY_SIZE(groups_gfx10); + break; + case GFX6: + default: + return; /* not implemented */ + } + + screen->perfcounters = pc = CALLOC_STRUCT(si_perfcounters); + if (!pc) + return; + + pc->num_stop_cs_dwords = 14 + si_cp_write_fence_dwords(screen); + pc->num_instance_cs_dwords = 3; + + pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false); + pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false); + + pc->blocks = CALLOC(num_blocks, sizeof(struct si_pc_block)); + if (!pc->blocks) + goto error; + pc->num_blocks = num_blocks; + + for (i = 0; i < num_blocks; ++i) { + struct si_pc_block *block = &pc->blocks[i]; + block->b = &blocks[i]; + block->num_instances = MAX2(1, block->b->instances); + + if (!strcmp(block->b->b->name, "CB") || + !strcmp(block->b->b->name, "DB") || + !strcmp(block->b->b->name, "RMI")) + block->num_instances = screen->info.max_se; + else if (!strcmp(block->b->b->name, "TCC")) + block->num_instances = screen->info.num_tcc_blocks; + else if (!strcmp(block->b->b->name, "IA")) + block->num_instances = MAX2(1, screen->info.max_se / 2); + else if (!strcmp(block->b->b->name, "TA") || + !strcmp(block->b->b->name, "TCP") || + !strcmp(block->b->b->name, "TD")) { + block->num_instances = MAX2(1, screen->info.max_good_cu_per_sa); + } + + if (si_pc_block_has_per_instance_groups(pc, block)) { + block->num_groups = block->num_instances; + } else { + block->num_groups = 1; + } + + if (si_pc_block_has_per_se_groups(pc, block)) + block->num_groups *= screen->info.max_se; + if (block->b->b->flags & SI_PC_BLOCK_SHADER) + block->num_groups *= ARRAY_SIZE(si_pc_shader_type_bits); + + pc->num_groups += block->num_groups; + } + + return; error: - si_destroy_perfcounters(screen); + si_destroy_perfcounters(screen); }