src/gallium/drivers/radeonsi/si_perfcounter.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #include "radeon/r600_cs.h"
  25 #include "radeon/r600_query.h"
  26 #include "util/u_memory.h"
  27
  28 #include "si_pipe.h"
  29 #include "sid.h"
  30
  31 enum si_pc_reg_layout {
  32         /* All secondary selector dwords follow as one block after the primary
  33          * selector dwords for the counters that have secondary selectors.
  34          */
  35         SI_PC_MULTI_BLOCK = 0,
  36
  37         /* Each secondary selector dword follows immediately afters the
  38          * corresponding primary.
  39          */
  40         SI_PC_MULTI_ALTERNATE = 1,
  41
  42         /* All secondary selector dwords follow as one block after all primary
  43          * selector dwords.
  44          */
  45         SI_PC_MULTI_TAIL = 2,
  46
  47         /* Free-form arrangement of selector registers. */
  48         SI_PC_MULTI_CUSTOM = 3,
  49
  50         SI_PC_MULTI_MASK = 3,
  51
  52         /* Registers are laid out in decreasing rather than increasing order. */
  53         SI_PC_REG_REVERSE = 4,
  54
  55         SI_PC_FAKE = 8,
  56 };
  57
  58 struct si_pc_block_base {
  59         const char *name;
  60         unsigned num_counters;
  61         unsigned flags;
  62
  63         unsigned select_or;
  64         unsigned select0;
  65         unsigned counter0_lo;
  66         unsigned *select;
  67         unsigned *counters;
  68         unsigned num_multi;
  69         unsigned num_prelude;
  70         unsigned layout;
  71 };
  72
  73 struct si_pc_block {
  74         struct si_pc_block_base *b;
  75         unsigned selectors;
  76         unsigned instances;
  77 };
  78
  79 /* The order is chosen to be compatible with GPUPerfStudio's hardcoding of
  80  * performance counter group IDs.
  81  */
  82 static const char * const si_pc_shader_type_suffixes[] = {
  83         "", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS"
  84 };
  85
  86 static const unsigned si_pc_shader_type_bits[] = {
  87         0x7f,
  88         S_036780_ES_EN(1),
  89         S_036780_GS_EN(1),
  90         S_036780_VS_EN(1),
  91         S_036780_PS_EN(1),
  92         S_036780_LS_EN(1),
  93         S_036780_HS_EN(1),
  94         S_036780_CS_EN(1),
  95 };
  96
  97 static struct si_pc_block_base cik_CB = {
  98         .name = "CB",
  99         .num_counters = 4,
 100         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS,
 101
 102         .select0 = R_037000_CB_PERFCOUNTER_FILTER,
 103         .counter0_lo = R_035018_CB_PERFCOUNTER0_LO,
 104         .num_multi = 1,
 105         .num_prelude = 1,
 106         .layout = SI_PC_MULTI_ALTERNATE,
 107 };
 108
 109 static unsigned cik_CPC_select[] = {
 110         R_036024_CPC_PERFCOUNTER0_SELECT,
 111         R_036010_CPC_PERFCOUNTER0_SELECT1,
 112         R_03600C_CPC_PERFCOUNTER1_SELECT,
 113 };
 114 static struct si_pc_block_base cik_CPC = {
 115         .name = "CPC",
 116         .num_counters = 2,
 117
 118         .select = cik_CPC_select,
 119         .counter0_lo = R_034018_CPC_PERFCOUNTER0_LO,
 120         .num_multi = 1,
 121         .layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE,
 122 };
 123
 124 static struct si_pc_block_base cik_CPF = {
 125         .name = "CPF",
 126         .num_counters = 2,
 127
 128         .select0 = R_03601C_CPF_PERFCOUNTER0_SELECT,
 129         .counter0_lo = R_034028_CPF_PERFCOUNTER0_LO,
 130         .num_multi = 1,
 131         .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
 132 };
 133
 134 static struct si_pc_block_base cik_CPG = {
 135         .name = "CPG",
 136         .num_counters = 2,
 137
 138         .select0 = R_036008_CPG_PERFCOUNTER0_SELECT,
 139         .counter0_lo = R_034008_CPG_PERFCOUNTER0_LO,
 140         .num_multi = 1,
 141         .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
 142 };
 143
 144 static struct si_pc_block_base cik_DB = {
 145         .name = "DB",
 146         .num_counters = 4,
 147         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS,
 148
 149         .select0 = R_037100_DB_PERFCOUNTER0_SELECT,
 150         .counter0_lo = R_035100_DB_PERFCOUNTER0_LO,
 151         .num_multi = 3, // really only 2, but there's a gap between registers
 152         .layout = SI_PC_MULTI_ALTERNATE,
 153 };
 154
 155 static struct si_pc_block_base cik_GDS = {
 156         .name = "GDS",
 157         .num_counters = 4,
 158
 159         .select0 = R_036A00_GDS_PERFCOUNTER0_SELECT,
 160         .counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO,
 161         .num_multi = 1,
 162         .layout = SI_PC_MULTI_TAIL,
 163 };
 164
 165 static unsigned cik_GRBM_counters[] = {
 166         R_034100_GRBM_PERFCOUNTER0_LO,
 167         R_03410C_GRBM_PERFCOUNTER1_LO,
 168 };
 169 static struct si_pc_block_base cik_GRBM = {
 170         .name = "GRBM",
 171         .num_counters = 2,
 172
 173         .select0 = R_036100_GRBM_PERFCOUNTER0_SELECT,
 174         .counters = cik_GRBM_counters,
 175 };
 176
 177 static struct si_pc_block_base cik_GRBMSE = {
 178         .name = "GRBMSE",
 179         .num_counters = 4,
 180
 181         .select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT,
 182         .counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO,
 183 };
 184
 185 static struct si_pc_block_base cik_IA = {
 186         .name = "IA",
 187         .num_counters = 4,
 188
 189         .select0 = R_036210_IA_PERFCOUNTER0_SELECT,
 190         .counter0_lo = R_034220_IA_PERFCOUNTER0_LO,
 191         .num_multi = 1,
 192         .layout = SI_PC_MULTI_TAIL,
 193 };
 194
 195 static struct si_pc_block_base cik_PA_SC = {
 196         .name = "PA_SC",
 197         .num_counters = 8,
 198         .flags = R600_PC_BLOCK_SE,
 199
 200         .select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT,
 201         .counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO,
 202         .num_multi = 1,
 203         .layout = SI_PC_MULTI_ALTERNATE,
 204 };
 205
 206 /* According to docs, PA_SU counters are only 48 bits wide. */
 207 static struct si_pc_block_base cik_PA_SU = {
 208         .name = "PA_SU",
 209         .num_counters = 4,
 210         .flags = R600_PC_BLOCK_SE,
 211
 212         .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT,
 213         .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO,
 214         .num_multi = 2,
 215         .layout = SI_PC_MULTI_ALTERNATE,
 216 };
 217
 218 static struct si_pc_block_base cik_SPI = {
 219         .name = "SPI",
 220         .num_counters = 6,
 221         .flags = R600_PC_BLOCK_SE,
 222
 223         .select0 = R_036600_SPI_PERFCOUNTER0_SELECT,
 224         .counter0_lo = R_034604_SPI_PERFCOUNTER0_LO,
 225         .num_multi = 4,
 226         .layout = SI_PC_MULTI_BLOCK,
 227 };
 228
 229 static struct si_pc_block_base cik_SQ = {
 230         .name = "SQ",
 231         .num_counters = 16,
 232         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_SHADER,
 233
 234         .select0 = R_036700_SQ_PERFCOUNTER0_SELECT,
 235         .select_or = S_036700_SQC_BANK_MASK(15) |
 236                         S_036700_SQC_CLIENT_MASK(15) |
 237                         S_036700_SIMD_MASK(15),
 238         .counter0_lo = R_034700_SQ_PERFCOUNTER0_LO,
 239 };
 240
 241 static struct si_pc_block_base cik_SX = {
 242         .name = "SX",
 243         .num_counters = 4,
 244         .flags = R600_PC_BLOCK_SE,
 245
 246         .select0 = R_036900_SX_PERFCOUNTER0_SELECT,
 247         .counter0_lo = R_034900_SX_PERFCOUNTER0_LO,
 248         .num_multi = 2,
 249         .layout = SI_PC_MULTI_TAIL,
 250 };
 251
 252 static struct si_pc_block_base cik_TA = {
 253         .name = "TA",
 254         .num_counters = 2,
 255         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
 256
 257         .select0 = R_036B00_TA_PERFCOUNTER0_SELECT,
 258         .counter0_lo = R_034B00_TA_PERFCOUNTER0_LO,
 259         .num_multi = 1,
 260         .layout = SI_PC_MULTI_ALTERNATE,
 261 };
 262
 263 static struct si_pc_block_base cik_TD = {
 264         .name = "TD",
 265         .num_counters = 2,
 266         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
 267
 268         .select0 = R_036C00_TD_PERFCOUNTER0_SELECT,
 269         .counter0_lo = R_034C00_TD_PERFCOUNTER0_LO,
 270         .num_multi = 1,
 271         .layout = SI_PC_MULTI_ALTERNATE,
 272 };
 273
 274 static struct si_pc_block_base cik_TCA = {
 275         .name = "TCA",
 276         .num_counters = 4,
 277         .flags = R600_PC_BLOCK_INSTANCE_GROUPS,
 278
 279         .select0 = R_036E40_TCA_PERFCOUNTER0_SELECT,
 280         .counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO,
 281         .num_multi = 2,
 282         .layout = SI_PC_MULTI_ALTERNATE,
 283 };
 284
 285 static struct si_pc_block_base cik_TCC = {
 286         .name = "TCC",
 287         .num_counters = 4,
 288         .flags = R600_PC_BLOCK_INSTANCE_GROUPS,
 289
 290         .select0 = R_036E00_TCC_PERFCOUNTER0_SELECT,
 291         .counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO,
 292         .num_multi = 2,
 293         .layout = SI_PC_MULTI_ALTERNATE,
 294 };
 295
 296 static struct si_pc_block_base cik_TCP = {
 297         .name = "TCP",
 298         .num_counters = 4,
 299         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
 300
 301         .select0 = R_036D00_TCP_PERFCOUNTER0_SELECT,
 302         .counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO,
 303         .num_multi = 2,
 304         .layout = SI_PC_MULTI_ALTERNATE,
 305 };
 306
 307 static struct si_pc_block_base cik_VGT = {
 308         .name = "VGT",
 309         .num_counters = 4,
 310         .flags = R600_PC_BLOCK_SE,
 311
 312         .select0 = R_036230_VGT_PERFCOUNTER0_SELECT,
 313         .counter0_lo = R_034240_VGT_PERFCOUNTER0_LO,
 314         .num_multi = 1,
 315         .layout = SI_PC_MULTI_TAIL,
 316 };
 317
 318 static struct si_pc_block_base cik_WD = {
 319         .name = "WD",
 320         .num_counters = 4,
 321
 322         .select0 = R_036200_WD_PERFCOUNTER0_SELECT,
 323         .counter0_lo = R_034200_WD_PERFCOUNTER0_LO,
 324 };
 325
 326 static struct si_pc_block_base cik_MC = {
 327         .name = "MC",
 328         .num_counters = 4,
 329
 330         .layout = SI_PC_FAKE,
 331 };
 332
 333 static struct si_pc_block_base cik_SRBM = {
 334         .name = "SRBM",
 335         .num_counters = 2,
 336
 337         .layout = SI_PC_FAKE,
 338 };
 339
 340 /* Both the number of instances and selectors varies between chips of the same
 341  * class. We only differentiate by class here and simply expose the maximum
 342  * number over all chips in a class.
 343  *
 344  * Unfortunately, GPUPerfStudio uses the order of performance counter groups
 345  * blindly once it believes it has identified the hardware, so the order of
 346  * blocks here matters.
 347  */
 348 static struct si_pc_block groups_CIK[] = {
 349         { &cik_CB, 226, 4 },
 350         { &cik_CPF, 17 },
 351         { &cik_DB, 257, 4 },
 352         { &cik_GRBM, 34 },
 353         { &cik_GRBMSE, 15 },
 354         { &cik_PA_SU, 153 },
 355         { &cik_PA_SC, 395 },
 356         { &cik_SPI, 186 },
 357         { &cik_SQ, 252 },
 358         { &cik_SX, 32 },
 359         { &cik_TA, 111, 11 },
 360         { &cik_TCA, 39, 2 },
 361         { &cik_TCC, 160, 16 },
 362         { &cik_TD, 55, 11 },
 363         { &cik_TCP, 154, 11 },
 364         { &cik_GDS, 121 },
 365         { &cik_VGT, 140 },
 366         { &cik_IA, 22 },
 367         { &cik_MC, 22 },
 368         { &cik_SRBM, 19 },
 369         { &cik_WD, 22 },
 370         { &cik_CPG, 46 },
 371         { &cik_CPC, 22 },
 372
 373 };
 374
 375 static struct si_pc_block groups_VI[] = {
 376         { &cik_CB, 405, 4 },
 377         { &cik_CPF, 19 },
 378         { &cik_DB, 257, 4 },
 379         { &cik_GRBM, 34 },
 380         { &cik_GRBMSE, 15 },
 381         { &cik_PA_SU, 153 },
 382         { &cik_PA_SC, 397 },
 383         { &cik_SPI, 197 },
 384         { &cik_SQ, 273 },
 385         { &cik_SX, 34 },
 386         { &cik_TA, 119, 16 },
 387         { &cik_TCA, 35, 2 },
 388         { &cik_TCC, 192, 16 },
 389         { &cik_TD, 55, 16 },
 390         { &cik_TCP, 180, 16 },
 391         { &cik_GDS, 121 },
 392         { &cik_VGT, 147 },
 393         { &cik_IA, 24 },
 394         { &cik_MC, 22 },
 395         { &cik_SRBM, 27 },
 396         { &cik_WD, 37 },
 397         { &cik_CPG, 48 },
 398         { &cik_CPC, 24 },
 399
 400 };
 401
 402 static struct si_pc_block groups_gfx9[] = {
 403         { &cik_CB, 438, 4 },
 404         { &cik_CPF, 32 },
 405         { &cik_DB, 328, 4 },
 406         { &cik_GRBM, 38 },
 407         { &cik_GRBMSE, 16 },
 408         { &cik_PA_SU, 292 },
 409         { &cik_PA_SC, 491 },
 410         { &cik_SPI, 196 },
 411         { &cik_SQ, 374 },
 412         { &cik_SX, 208 },
 413         { &cik_TA, 119, 16 },
 414         { &cik_TCA, 35, 2 },
 415         { &cik_TCC, 256, 16 },
 416         { &cik_TD, 57, 16 },
 417         { &cik_TCP, 85, 16 },
 418         { &cik_GDS, 121 },
 419         { &cik_VGT, 148 },
 420         { &cik_IA, 32 },
 421         { &cik_WD, 58 },
 422         { &cik_CPG, 59 },
 423         { &cik_CPC, 35 },
 424 };
 425
 426 static void si_pc_emit_instance(struct si_context *sctx,
 427                                 int se, int instance)
 428 {
 429         struct radeon_winsys_cs *cs = sctx->b.gfx_cs;
 430         unsigned value = S_030800_SH_BROADCAST_WRITES(1);
 431
 432         if (se >= 0) {
 433                 value |= S_030800_SE_INDEX(se);
 434         } else {
 435                 value |= S_030800_SE_BROADCAST_WRITES(1);
 436         }
 437
 438         if (instance >= 0) {
 439                 value |= S_030800_INSTANCE_INDEX(instance);
 440         } else {
 441                 value |= S_030800_INSTANCE_BROADCAST_WRITES(1);
 442         }
 443
 444         radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value);
 445 }
 446
 447 static void si_pc_emit_shaders(struct si_context *sctx,
 448                                unsigned shaders)
 449 {
 450         struct radeon_winsys_cs *cs = sctx->b.gfx_cs;
 451
 452         radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2);
 453         radeon_emit(cs, shaders & 0x7f);
 454         radeon_emit(cs, 0xffffffff);
 455 }
 456
 457 static void si_pc_emit_select(struct si_context *sctx,
 458                         struct r600_perfcounter_block *group,
 459                         unsigned count, unsigned *selectors)
 460 {
 461         struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
 462         struct si_pc_block_base *regs = sigroup->b;
 463         struct radeon_winsys_cs *cs = sctx->b.gfx_cs;
 464         unsigned idx;
 465         unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
 466         unsigned dw;
 467
 468         assert(count <= regs->num_counters);
 469
 470         if (regs->layout & SI_PC_FAKE)
 471                 return;
 472
 473         if (layout_multi == SI_PC_MULTI_BLOCK) {
 474                 assert(!(regs->layout & SI_PC_REG_REVERSE));
 475
 476                 dw = count + regs->num_prelude;
 477                 if (count >= regs->num_multi)
 478                         dw += regs->num_multi;
 479                 radeon_set_uconfig_reg_seq(cs, regs->select0, dw);
 480                 for (idx = 0; idx < regs->num_prelude; ++idx)
 481                         radeon_emit(cs, 0);
 482                 for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
 483                         radeon_emit(cs, selectors[idx] | regs->select_or);
 484
 485                 if (count < regs->num_multi) {
 486                         unsigned select1 =
 487                                 regs->select0 + 4 * regs->num_multi;
 488                         radeon_set_uconfig_reg_seq(cs, select1, count);
 489                 }
 490
 491                 for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
 492                         radeon_emit(cs, 0);
 493
 494                 if (count > regs->num_multi) {
 495                         for (idx = regs->num_multi; idx < count; ++idx)
 496                                 radeon_emit(cs, selectors[idx] | regs->select_or);
 497                 }
 498         } else if (layout_multi == SI_PC_MULTI_TAIL) {
 499                 unsigned select1, select1_count;
 500
 501                 assert(!(regs->layout & SI_PC_REG_REVERSE));
 502
 503                 radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude);
 504                 for (idx = 0; idx < regs->num_prelude; ++idx)
 505                         radeon_emit(cs, 0);
 506                 for (idx = 0; idx < count; ++idx)
 507                         radeon_emit(cs, selectors[idx] | regs->select_or);
 508
 509                 select1 = regs->select0 + 4 * regs->num_counters;
 510                 select1_count = MIN2(count, regs->num_multi);
 511                 radeon_set_uconfig_reg_seq(cs, select1, select1_count);
 512                 for (idx = 0; idx < select1_count; ++idx)
 513                         radeon_emit(cs, 0);
 514         } else if (layout_multi == SI_PC_MULTI_CUSTOM) {
 515                 unsigned *reg = regs->select;
 516                 for (idx = 0; idx < count; ++idx) {
 517                         radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or);
 518                         if (idx < regs->num_multi)
 519                                 radeon_set_uconfig_reg(cs, *reg++, 0);
 520                 }
 521         } else {
 522                 assert(layout_multi == SI_PC_MULTI_ALTERNATE);
 523
 524                 unsigned reg_base = regs->select0;
 525                 unsigned reg_count = count + MIN2(count, regs->num_multi);
 526                 reg_count += regs->num_prelude;
 527
 528                 if (!(regs->layout & SI_PC_REG_REVERSE)) {
 529                         radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);
 530
 531                         for (idx = 0; idx < regs->num_prelude; ++idx)
 532                                 radeon_emit(cs, 0);
 533                         for (idx = 0; idx < count; ++idx) {
 534                                 radeon_emit(cs, selectors[idx] | regs->select_or);
 535                                 if (idx < regs->num_multi)
 536                                         radeon_emit(cs, 0);
 537                         }
 538                 } else {
 539                         reg_base -= (reg_count - 1) * 4;
 540                         radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);
 541
 542                         for (idx = count; idx > 0; --idx) {
 543                                 if (idx <= regs->num_multi)
 544                                         radeon_emit(cs, 0);
 545                                 radeon_emit(cs, selectors[idx - 1] | regs->select_or);
 546                         }
 547                         for (idx = 0; idx < regs->num_prelude; ++idx)
 548                                 radeon_emit(cs, 0);
 549                 }
 550         }
 551 }
 552
 553 static void si_pc_emit_start(struct si_context *sctx,
 554                              struct r600_resource *buffer, uint64_t va)
 555 {
 556         struct radeon_winsys_cs *cs = sctx->b.gfx_cs;
 557
 558         radeon_add_to_buffer_list(&sctx->b, sctx->b.gfx_cs, buffer,
 559                                   RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
 560
 561         radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 562         radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
 563                         COPY_DATA_DST_SEL(COPY_DATA_MEM));
 564         radeon_emit(cs, 1); /* immediate */
 565         radeon_emit(cs, 0); /* unused */
 566         radeon_emit(cs, va);
 567         radeon_emit(cs, va >> 32);
 568
 569         radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
 570                                S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET));
 571         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 572         radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_START) | EVENT_INDEX(0));
 573         radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
 574                                S_036020_PERFMON_STATE(V_036020_START_COUNTING));
 575 }
 576
 577 /* Note: The buffer was already added in si_pc_emit_start, so we don't have to
 578  * do it again in here. */
 579 static void si_pc_emit_stop(struct si_context *sctx,
 580                             struct r600_resource *buffer, uint64_t va)
 581 {
 582         struct radeon_winsys_cs *cs = sctx->b.gfx_cs;
 583
 584         si_gfx_write_event_eop(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0,
 585                                EOP_DATA_SEL_VALUE_32BIT,
 586                                buffer, va, 0, SI_NOT_QUERY);
 587         si_gfx_wait_fence(sctx, va, 0, 0xffffffff);
 588
 589         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 590         radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0));
 591         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 592         radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0));
 593         radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
 594                                S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) |
 595                                S_036020_PERFMON_SAMPLE_ENABLE(1));
 596 }
 597
 598 static void si_pc_emit_read(struct si_context *sctx,
 599                             struct r600_perfcounter_block *group,
 600                             unsigned count, unsigned *selectors,
 601                             struct r600_resource *buffer, uint64_t va)
 602 {
 603         struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
 604         struct si_pc_block_base *regs = sigroup->b;
 605         struct radeon_winsys_cs *cs = sctx->b.gfx_cs;
 606         unsigned idx;
 607         unsigned reg = regs->counter0_lo;
 608         unsigned reg_delta = 8;
 609
 610         if (!(regs->layout & SI_PC_FAKE)) {
 611                 if (regs->layout & SI_PC_REG_REVERSE)
 612                         reg_delta = -reg_delta;
 613
 614                 for (idx = 0; idx < count; ++idx) {
 615                         if (regs->counters)
 616                                 reg = regs->counters[idx];
 617
 618                         radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 619                         radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) |
 620                                         COPY_DATA_DST_SEL(COPY_DATA_MEM) |
 621                                         COPY_DATA_COUNT_SEL); /* 64 bits */
 622                         radeon_emit(cs, reg >> 2);
 623                         radeon_emit(cs, 0); /* unused */
 624                         radeon_emit(cs, va);
 625                         radeon_emit(cs, va >> 32);
 626                         va += sizeof(uint64_t);
 627                         reg += reg_delta;
 628                 }
 629         } else {
 630                 for (idx = 0; idx < count; ++idx) {
 631                         radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 632                         radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
 633                                         COPY_DATA_DST_SEL(COPY_DATA_MEM) |
 634                                         COPY_DATA_COUNT_SEL);
 635                         radeon_emit(cs, 0); /* immediate */
 636                         radeon_emit(cs, 0);
 637                         radeon_emit(cs, va);
 638                         radeon_emit(cs, va >> 32);
 639                         va += sizeof(uint64_t);
 640                 }
 641         }
 642 }
 643
 644 static void si_pc_cleanup(struct si_screen *sscreen)
 645 {
 646         si_perfcounters_do_destroy(sscreen->perfcounters);
 647         sscreen->perfcounters = NULL;
 648 }
 649
 650 void si_init_perfcounters(struct si_screen *screen)
 651 {
 652         struct r600_perfcounters *pc;
 653         struct si_pc_block *blocks;
 654         unsigned num_blocks;
 655         unsigned i;
 656
 657         switch (screen->info.chip_class) {
 658         case CIK:
 659                 blocks = groups_CIK;
 660                 num_blocks = ARRAY_SIZE(groups_CIK);
 661                 break;
 662         case VI:
 663                 blocks = groups_VI;
 664                 num_blocks = ARRAY_SIZE(groups_VI);
 665                 break;
 666         case GFX9:
 667                 blocks = groups_gfx9;
 668                 num_blocks = ARRAY_SIZE(groups_gfx9);
 669                 break;
 670         case SI:
 671         default:
 672                 return; /* not implemented */
 673         }
 674
 675         if (screen->info.max_sh_per_se != 1) {
 676                 /* This should not happen on non-SI chips. */
 677                 fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not "
 678                         "supported (inaccurate performance counters)\n",
 679                         screen->info.max_sh_per_se);
 680         }
 681
 682         pc = CALLOC_STRUCT(r600_perfcounters);
 683         if (!pc)
 684                 return;
 685
 686         pc->num_stop_cs_dwords = 14 + si_gfx_write_fence_dwords(screen);
 687         pc->num_instance_cs_dwords = 3;
 688
 689         pc->num_shader_types = ARRAY_SIZE(si_pc_shader_type_bits);
 690         pc->shader_type_suffixes = si_pc_shader_type_suffixes;
 691         pc->shader_type_bits = si_pc_shader_type_bits;
 692
 693         pc->emit_instance = si_pc_emit_instance;
 694         pc->emit_shaders = si_pc_emit_shaders;
 695         pc->emit_select = si_pc_emit_select;
 696         pc->emit_start = si_pc_emit_start;
 697         pc->emit_stop = si_pc_emit_stop;
 698         pc->emit_read = si_pc_emit_read;
 699         pc->cleanup = si_pc_cleanup;
 700
 701         if (!si_perfcounters_init(pc, num_blocks))
 702                 goto error;
 703
 704         for (i = 0; i < num_blocks; ++i) {
 705                 struct si_pc_block *block = &blocks[i];
 706                 unsigned instances = block->instances;
 707
 708                 if (!strcmp(block->b->name, "IA")) {
 709                         if (screen->info.max_se > 2)
 710                                 instances = 2;
 711                 }
 712
 713                 si_perfcounters_add_block(screen, pc,
 714                                             block->b->name,
 715                                             block->b->flags,
 716                                             block->b->num_counters,
 717                                             block->selectors,
 718                                             instances,
 719                                             block);
 720         }
 721
 722         screen->perfcounters = pc;
 723         return;
 724
 725 error:
 726         si_perfcounters_do_destroy(pc);
 727 }