2 * Copyright 2015 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25 #include "si_build_pm4.h"
27 #include "util/u_memory.h"
30 enum si_pc_reg_layout
{
31 /* All secondary selector dwords follow as one block after the primary
32 * selector dwords for the counters that have secondary selectors.
34 SI_PC_MULTI_BLOCK
= 0,
36 /* Each secondary selector dword follows immediately afters the
37 * corresponding primary.
39 SI_PC_MULTI_ALTERNATE
= 1,
41 /* All secondary selector dwords follow as one block after all primary
46 /* Free-form arrangement of selector registers. */
47 SI_PC_MULTI_CUSTOM
= 3,
51 /* Registers are laid out in decreasing rather than increasing order. */
52 SI_PC_REG_REVERSE
= 4,
57 struct si_pc_block_base
{
59 unsigned num_counters
;
73 struct si_pc_block_base
*b
;
78 /* The order is chosen to be compatible with GPUPerfStudio's hardcoding of
79 * performance counter group IDs.
81 static const char * const si_pc_shader_type_suffixes
[] = {
82 "", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS"
85 static const unsigned si_pc_shader_type_bits
[] = {
96 static struct si_pc_block_base cik_CB
= {
99 .flags
= SI_PC_BLOCK_SE
| SI_PC_BLOCK_INSTANCE_GROUPS
,
101 .select0
= R_037000_CB_PERFCOUNTER_FILTER
,
102 .counter0_lo
= R_035018_CB_PERFCOUNTER0_LO
,
105 .layout
= SI_PC_MULTI_ALTERNATE
,
108 static unsigned cik_CPC_select
[] = {
109 R_036024_CPC_PERFCOUNTER0_SELECT
,
110 R_036010_CPC_PERFCOUNTER0_SELECT1
,
111 R_03600C_CPC_PERFCOUNTER1_SELECT
,
113 static struct si_pc_block_base cik_CPC
= {
117 .select
= cik_CPC_select
,
118 .counter0_lo
= R_034018_CPC_PERFCOUNTER0_LO
,
120 .layout
= SI_PC_MULTI_CUSTOM
| SI_PC_REG_REVERSE
,
123 static struct si_pc_block_base cik_CPF
= {
127 .select0
= R_03601C_CPF_PERFCOUNTER0_SELECT
,
128 .counter0_lo
= R_034028_CPF_PERFCOUNTER0_LO
,
130 .layout
= SI_PC_MULTI_ALTERNATE
| SI_PC_REG_REVERSE
,
133 static struct si_pc_block_base cik_CPG
= {
137 .select0
= R_036008_CPG_PERFCOUNTER0_SELECT
,
138 .counter0_lo
= R_034008_CPG_PERFCOUNTER0_LO
,
140 .layout
= SI_PC_MULTI_ALTERNATE
| SI_PC_REG_REVERSE
,
143 static struct si_pc_block_base cik_DB
= {
146 .flags
= SI_PC_BLOCK_SE
| SI_PC_BLOCK_INSTANCE_GROUPS
,
148 .select0
= R_037100_DB_PERFCOUNTER0_SELECT
,
149 .counter0_lo
= R_035100_DB_PERFCOUNTER0_LO
,
150 .num_multi
= 3, // really only 2, but there's a gap between registers
151 .layout
= SI_PC_MULTI_ALTERNATE
,
154 static struct si_pc_block_base cik_GDS
= {
158 .select0
= R_036A00_GDS_PERFCOUNTER0_SELECT
,
159 .counter0_lo
= R_034A00_GDS_PERFCOUNTER0_LO
,
161 .layout
= SI_PC_MULTI_TAIL
,
164 static unsigned cik_GRBM_counters
[] = {
165 R_034100_GRBM_PERFCOUNTER0_LO
,
166 R_03410C_GRBM_PERFCOUNTER1_LO
,
168 static struct si_pc_block_base cik_GRBM
= {
172 .select0
= R_036100_GRBM_PERFCOUNTER0_SELECT
,
173 .counters
= cik_GRBM_counters
,
176 static struct si_pc_block_base cik_GRBMSE
= {
180 .select0
= R_036108_GRBM_SE0_PERFCOUNTER_SELECT
,
181 .counter0_lo
= R_034114_GRBM_SE0_PERFCOUNTER_LO
,
184 static struct si_pc_block_base cik_IA
= {
188 .select0
= R_036210_IA_PERFCOUNTER0_SELECT
,
189 .counter0_lo
= R_034220_IA_PERFCOUNTER0_LO
,
191 .layout
= SI_PC_MULTI_TAIL
,
194 static struct si_pc_block_base cik_PA_SC
= {
197 .flags
= SI_PC_BLOCK_SE
,
199 .select0
= R_036500_PA_SC_PERFCOUNTER0_SELECT
,
200 .counter0_lo
= R_034500_PA_SC_PERFCOUNTER0_LO
,
202 .layout
= SI_PC_MULTI_ALTERNATE
,
205 /* According to docs, PA_SU counters are only 48 bits wide. */
206 static struct si_pc_block_base cik_PA_SU
= {
209 .flags
= SI_PC_BLOCK_SE
,
211 .select0
= R_036400_PA_SU_PERFCOUNTER0_SELECT
,
212 .counter0_lo
= R_034400_PA_SU_PERFCOUNTER0_LO
,
214 .layout
= SI_PC_MULTI_ALTERNATE
,
217 static struct si_pc_block_base cik_SPI
= {
220 .flags
= SI_PC_BLOCK_SE
,
222 .select0
= R_036600_SPI_PERFCOUNTER0_SELECT
,
223 .counter0_lo
= R_034604_SPI_PERFCOUNTER0_LO
,
225 .layout
= SI_PC_MULTI_BLOCK
,
228 static struct si_pc_block_base cik_SQ
= {
231 .flags
= SI_PC_BLOCK_SE
| SI_PC_BLOCK_SHADER
,
233 .select0
= R_036700_SQ_PERFCOUNTER0_SELECT
,
234 .select_or
= S_036700_SQC_BANK_MASK(15) |
235 S_036700_SQC_CLIENT_MASK(15) |
236 S_036700_SIMD_MASK(15),
237 .counter0_lo
= R_034700_SQ_PERFCOUNTER0_LO
,
240 static struct si_pc_block_base cik_SX
= {
243 .flags
= SI_PC_BLOCK_SE
,
245 .select0
= R_036900_SX_PERFCOUNTER0_SELECT
,
246 .counter0_lo
= R_034900_SX_PERFCOUNTER0_LO
,
248 .layout
= SI_PC_MULTI_TAIL
,
251 static struct si_pc_block_base cik_TA
= {
254 .flags
= SI_PC_BLOCK_SE
| SI_PC_BLOCK_INSTANCE_GROUPS
| SI_PC_BLOCK_SHADER_WINDOWED
,
256 .select0
= R_036B00_TA_PERFCOUNTER0_SELECT
,
257 .counter0_lo
= R_034B00_TA_PERFCOUNTER0_LO
,
259 .layout
= SI_PC_MULTI_ALTERNATE
,
262 static struct si_pc_block_base cik_TD
= {
265 .flags
= SI_PC_BLOCK_SE
| SI_PC_BLOCK_INSTANCE_GROUPS
| SI_PC_BLOCK_SHADER_WINDOWED
,
267 .select0
= R_036C00_TD_PERFCOUNTER0_SELECT
,
268 .counter0_lo
= R_034C00_TD_PERFCOUNTER0_LO
,
270 .layout
= SI_PC_MULTI_ALTERNATE
,
273 static struct si_pc_block_base cik_TCA
= {
276 .flags
= SI_PC_BLOCK_INSTANCE_GROUPS
,
278 .select0
= R_036E40_TCA_PERFCOUNTER0_SELECT
,
279 .counter0_lo
= R_034E40_TCA_PERFCOUNTER0_LO
,
281 .layout
= SI_PC_MULTI_ALTERNATE
,
284 static struct si_pc_block_base cik_TCC
= {
287 .flags
= SI_PC_BLOCK_INSTANCE_GROUPS
,
289 .select0
= R_036E00_TCC_PERFCOUNTER0_SELECT
,
290 .counter0_lo
= R_034E00_TCC_PERFCOUNTER0_LO
,
292 .layout
= SI_PC_MULTI_ALTERNATE
,
295 static struct si_pc_block_base cik_TCP
= {
298 .flags
= SI_PC_BLOCK_SE
| SI_PC_BLOCK_INSTANCE_GROUPS
| SI_PC_BLOCK_SHADER_WINDOWED
,
300 .select0
= R_036D00_TCP_PERFCOUNTER0_SELECT
,
301 .counter0_lo
= R_034D00_TCP_PERFCOUNTER0_LO
,
303 .layout
= SI_PC_MULTI_ALTERNATE
,
306 static struct si_pc_block_base cik_VGT
= {
309 .flags
= SI_PC_BLOCK_SE
,
311 .select0
= R_036230_VGT_PERFCOUNTER0_SELECT
,
312 .counter0_lo
= R_034240_VGT_PERFCOUNTER0_LO
,
314 .layout
= SI_PC_MULTI_TAIL
,
317 static struct si_pc_block_base cik_WD
= {
321 .select0
= R_036200_WD_PERFCOUNTER0_SELECT
,
322 .counter0_lo
= R_034200_WD_PERFCOUNTER0_LO
,
325 static struct si_pc_block_base cik_MC
= {
329 .layout
= SI_PC_FAKE
,
332 static struct si_pc_block_base cik_SRBM
= {
336 .layout
= SI_PC_FAKE
,
339 /* Both the number of instances and selectors varies between chips of the same
340 * class. We only differentiate by class here and simply expose the maximum
341 * number over all chips in a class.
343 * Unfortunately, GPUPerfStudio uses the order of performance counter groups
344 * blindly once it believes it has identified the hardware, so the order of
345 * blocks here matters.
347 static struct si_pc_block groups_CIK
[] = {
358 { &cik_TA
, 111, 11 },
362 { &cik_TCP
, 154, 11 },
374 static struct si_pc_block groups_VI
[] = {
385 { &cik_TA
, 119, 16 },
389 { &cik_TCP
, 180, 16 },
401 static struct si_pc_block groups_gfx9
[] = {
412 { &cik_TA
, 119, 16 },
416 { &cik_TCP
, 85, 16 },
425 static void si_pc_emit_instance(struct si_context
*sctx
,
426 int se
, int instance
)
428 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
429 unsigned value
= S_030800_SH_BROADCAST_WRITES(1);
432 value
|= S_030800_SE_INDEX(se
);
434 value
|= S_030800_SE_BROADCAST_WRITES(1);
438 value
|= S_030800_INSTANCE_INDEX(instance
);
440 value
|= S_030800_INSTANCE_BROADCAST_WRITES(1);
443 radeon_set_uconfig_reg(cs
, R_030800_GRBM_GFX_INDEX
, value
);
446 static void si_pc_emit_shaders(struct si_context
*sctx
,
449 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
451 radeon_set_uconfig_reg_seq(cs
, R_036780_SQ_PERFCOUNTER_CTRL
, 2);
452 radeon_emit(cs
, shaders
& 0x7f);
453 radeon_emit(cs
, 0xffffffff);
456 static void si_pc_emit_select(struct si_context
*sctx
,
457 struct si_perfcounter_block
*group
,
458 unsigned count
, unsigned *selectors
)
460 struct si_pc_block
*sigroup
= (struct si_pc_block
*)group
->data
;
461 struct si_pc_block_base
*regs
= sigroup
->b
;
462 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
464 unsigned layout_multi
= regs
->layout
& SI_PC_MULTI_MASK
;
467 assert(count
<= regs
->num_counters
);
469 if (regs
->layout
& SI_PC_FAKE
)
472 if (layout_multi
== SI_PC_MULTI_BLOCK
) {
473 assert(!(regs
->layout
& SI_PC_REG_REVERSE
));
475 dw
= count
+ regs
->num_prelude
;
476 if (count
>= regs
->num_multi
)
477 dw
+= regs
->num_multi
;
478 radeon_set_uconfig_reg_seq(cs
, regs
->select0
, dw
);
479 for (idx
= 0; idx
< regs
->num_prelude
; ++idx
)
481 for (idx
= 0; idx
< MIN2(count
, regs
->num_multi
); ++idx
)
482 radeon_emit(cs
, selectors
[idx
] | regs
->select_or
);
484 if (count
< regs
->num_multi
) {
486 regs
->select0
+ 4 * regs
->num_multi
;
487 radeon_set_uconfig_reg_seq(cs
, select1
, count
);
490 for (idx
= 0; idx
< MIN2(count
, regs
->num_multi
); ++idx
)
493 if (count
> regs
->num_multi
) {
494 for (idx
= regs
->num_multi
; idx
< count
; ++idx
)
495 radeon_emit(cs
, selectors
[idx
] | regs
->select_or
);
497 } else if (layout_multi
== SI_PC_MULTI_TAIL
) {
498 unsigned select1
, select1_count
;
500 assert(!(regs
->layout
& SI_PC_REG_REVERSE
));
502 radeon_set_uconfig_reg_seq(cs
, regs
->select0
, count
+ regs
->num_prelude
);
503 for (idx
= 0; idx
< regs
->num_prelude
; ++idx
)
505 for (idx
= 0; idx
< count
; ++idx
)
506 radeon_emit(cs
, selectors
[idx
] | regs
->select_or
);
508 select1
= regs
->select0
+ 4 * regs
->num_counters
;
509 select1_count
= MIN2(count
, regs
->num_multi
);
510 radeon_set_uconfig_reg_seq(cs
, select1
, select1_count
);
511 for (idx
= 0; idx
< select1_count
; ++idx
)
513 } else if (layout_multi
== SI_PC_MULTI_CUSTOM
) {
514 unsigned *reg
= regs
->select
;
515 for (idx
= 0; idx
< count
; ++idx
) {
516 radeon_set_uconfig_reg(cs
, *reg
++, selectors
[idx
] | regs
->select_or
);
517 if (idx
< regs
->num_multi
)
518 radeon_set_uconfig_reg(cs
, *reg
++, 0);
521 assert(layout_multi
== SI_PC_MULTI_ALTERNATE
);
523 unsigned reg_base
= regs
->select0
;
524 unsigned reg_count
= count
+ MIN2(count
, regs
->num_multi
);
525 reg_count
+= regs
->num_prelude
;
527 if (!(regs
->layout
& SI_PC_REG_REVERSE
)) {
528 radeon_set_uconfig_reg_seq(cs
, reg_base
, reg_count
);
530 for (idx
= 0; idx
< regs
->num_prelude
; ++idx
)
532 for (idx
= 0; idx
< count
; ++idx
) {
533 radeon_emit(cs
, selectors
[idx
] | regs
->select_or
);
534 if (idx
< regs
->num_multi
)
538 reg_base
-= (reg_count
- 1) * 4;
539 radeon_set_uconfig_reg_seq(cs
, reg_base
, reg_count
);
541 for (idx
= count
; idx
> 0; --idx
) {
542 if (idx
<= regs
->num_multi
)
544 radeon_emit(cs
, selectors
[idx
- 1] | regs
->select_or
);
546 for (idx
= 0; idx
< regs
->num_prelude
; ++idx
)
552 static void si_pc_emit_start(struct si_context
*sctx
,
553 struct r600_resource
*buffer
, uint64_t va
)
555 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
557 radeon_add_to_buffer_list(sctx
, sctx
->gfx_cs
, buffer
,
558 RADEON_USAGE_WRITE
, RADEON_PRIO_QUERY
);
560 radeon_emit(cs
, PKT3(PKT3_COPY_DATA
, 4, 0));
561 radeon_emit(cs
, COPY_DATA_SRC_SEL(COPY_DATA_IMM
) |
562 COPY_DATA_DST_SEL(COPY_DATA_MEM
));
563 radeon_emit(cs
, 1); /* immediate */
564 radeon_emit(cs
, 0); /* unused */
566 radeon_emit(cs
, va
>> 32);
568 radeon_set_uconfig_reg(cs
, R_036020_CP_PERFMON_CNTL
,
569 S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET
));
570 radeon_emit(cs
, PKT3(PKT3_EVENT_WRITE
, 0, 0));
571 radeon_emit(cs
, EVENT_TYPE(V_028A90_PERFCOUNTER_START
) | EVENT_INDEX(0));
572 radeon_set_uconfig_reg(cs
, R_036020_CP_PERFMON_CNTL
,
573 S_036020_PERFMON_STATE(V_036020_START_COUNTING
));
576 /* Note: The buffer was already added in si_pc_emit_start, so we don't have to
577 * do it again in here. */
578 static void si_pc_emit_stop(struct si_context
*sctx
,
579 struct r600_resource
*buffer
, uint64_t va
)
581 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
583 si_gfx_write_event_eop(sctx
, V_028A90_BOTTOM_OF_PIPE_TS
, 0,
584 EOP_DATA_SEL_VALUE_32BIT
,
585 buffer
, va
, 0, SI_NOT_QUERY
);
586 si_gfx_wait_fence(sctx
, va
, 0, 0xffffffff);
588 radeon_emit(cs
, PKT3(PKT3_EVENT_WRITE
, 0, 0));
589 radeon_emit(cs
, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE
) | EVENT_INDEX(0));
590 radeon_emit(cs
, PKT3(PKT3_EVENT_WRITE
, 0, 0));
591 radeon_emit(cs
, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP
) | EVENT_INDEX(0));
592 radeon_set_uconfig_reg(cs
, R_036020_CP_PERFMON_CNTL
,
593 S_036020_PERFMON_STATE(V_036020_STOP_COUNTING
) |
594 S_036020_PERFMON_SAMPLE_ENABLE(1));
597 static void si_pc_emit_read(struct si_context
*sctx
,
598 struct si_perfcounter_block
*group
,
599 unsigned count
, unsigned *selectors
,
600 struct r600_resource
*buffer
, uint64_t va
)
602 struct si_pc_block
*sigroup
= (struct si_pc_block
*)group
->data
;
603 struct si_pc_block_base
*regs
= sigroup
->b
;
604 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
606 unsigned reg
= regs
->counter0_lo
;
607 unsigned reg_delta
= 8;
609 if (!(regs
->layout
& SI_PC_FAKE
)) {
610 if (regs
->layout
& SI_PC_REG_REVERSE
)
611 reg_delta
= -reg_delta
;
613 for (idx
= 0; idx
< count
; ++idx
) {
615 reg
= regs
->counters
[idx
];
617 radeon_emit(cs
, PKT3(PKT3_COPY_DATA
, 4, 0));
618 radeon_emit(cs
, COPY_DATA_SRC_SEL(COPY_DATA_PERF
) |
619 COPY_DATA_DST_SEL(COPY_DATA_MEM
) |
620 COPY_DATA_COUNT_SEL
); /* 64 bits */
621 radeon_emit(cs
, reg
>> 2);
622 radeon_emit(cs
, 0); /* unused */
624 radeon_emit(cs
, va
>> 32);
625 va
+= sizeof(uint64_t);
629 for (idx
= 0; idx
< count
; ++idx
) {
630 radeon_emit(cs
, PKT3(PKT3_COPY_DATA
, 4, 0));
631 radeon_emit(cs
, COPY_DATA_SRC_SEL(COPY_DATA_IMM
) |
632 COPY_DATA_DST_SEL(COPY_DATA_MEM
) |
633 COPY_DATA_COUNT_SEL
);
634 radeon_emit(cs
, 0); /* immediate */
637 radeon_emit(cs
, va
>> 32);
638 va
+= sizeof(uint64_t);
643 static void si_pc_cleanup(struct si_screen
*sscreen
)
645 si_perfcounters_do_destroy(sscreen
->perfcounters
);
646 sscreen
->perfcounters
= NULL
;
649 void si_init_perfcounters(struct si_screen
*screen
)
651 struct si_perfcounters
*pc
;
652 struct si_pc_block
*blocks
;
656 switch (screen
->info
.chip_class
) {
659 num_blocks
= ARRAY_SIZE(groups_CIK
);
663 num_blocks
= ARRAY_SIZE(groups_VI
);
666 blocks
= groups_gfx9
;
667 num_blocks
= ARRAY_SIZE(groups_gfx9
);
671 return; /* not implemented */
674 if (screen
->info
.max_sh_per_se
!= 1) {
675 /* This should not happen on non-SI chips. */
676 fprintf(stderr
, "si_init_perfcounters: max_sh_per_se = %d not "
677 "supported (inaccurate performance counters)\n",
678 screen
->info
.max_sh_per_se
);
681 pc
= CALLOC_STRUCT(si_perfcounters
);
685 pc
->num_stop_cs_dwords
= 14 + si_gfx_write_fence_dwords(screen
);
686 pc
->num_instance_cs_dwords
= 3;
688 pc
->num_shader_types
= ARRAY_SIZE(si_pc_shader_type_bits
);
689 pc
->shader_type_suffixes
= si_pc_shader_type_suffixes
;
690 pc
->shader_type_bits
= si_pc_shader_type_bits
;
692 pc
->emit_instance
= si_pc_emit_instance
;
693 pc
->emit_shaders
= si_pc_emit_shaders
;
694 pc
->emit_select
= si_pc_emit_select
;
695 pc
->emit_start
= si_pc_emit_start
;
696 pc
->emit_stop
= si_pc_emit_stop
;
697 pc
->emit_read
= si_pc_emit_read
;
698 pc
->cleanup
= si_pc_cleanup
;
700 if (!si_perfcounters_init(pc
, num_blocks
))
703 for (i
= 0; i
< num_blocks
; ++i
) {
704 struct si_pc_block
*block
= &blocks
[i
];
705 unsigned instances
= block
->instances
;
707 if (!strcmp(block
->b
->name
, "CB") ||
708 !strcmp(block
->b
->name
, "DB"))
709 instances
= screen
->info
.max_se
;
710 else if (!strcmp(block
->b
->name
, "TCC"))
711 instances
= screen
->info
.num_tcc_blocks
;
712 else if (!strcmp(block
->b
->name
, "IA"))
713 instances
= MAX2(1, screen
->info
.max_se
/ 2);
715 si_perfcounters_add_block(screen
, pc
,
718 block
->b
->num_counters
,
724 screen
->perfcounters
= pc
;
728 si_perfcounters_do_destroy(pc
);