src/gallium/drivers/radeonsi/si_perfcounter.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *  Nicolai Hähnle <nicolai.haehnle@amd.com>
  25  *
  26  */
  27
  28 #include "radeon/r600_cs.h"
  29 #include "radeon/r600_query.h"
  30 #include "util/u_memory.h"
  31
  32 #include "si_pipe.h"
  33 #include "sid.h"
  34
  35 enum si_pc_reg_layout {
  36         /* All secondary selector dwords follow as one block after the primary
  37          * selector dwords for the counters that have secondary selectors.
  38          */
  39         SI_PC_MULTI_BLOCK = 0,
  40
  41         /* Each secondary selector dword follows immediately afters the
  42          * corresponding primary.
  43          */
  44         SI_PC_MULTI_ALTERNATE = 1,
  45
  46         /* All secondary selector dwords follow as one block after all primary
  47          * selector dwords.
  48          */
  49         SI_PC_MULTI_TAIL = 2,
  50
  51         /* Free-form arrangement of selector registers. */
  52         SI_PC_MULTI_CUSTOM = 3,
  53
  54         SI_PC_MULTI_MASK = 3,
  55
  56         /* Registers are laid out in decreasing rather than increasing order. */
  57         SI_PC_REG_REVERSE = 4,
  58
  59         SI_PC_FAKE = 8,
  60 };
  61
  62 struct si_pc_block_base {
  63         char name[8];
  64         unsigned num_counters;
  65         unsigned flags;
  66
  67         unsigned select_or;
  68         unsigned select0;
  69         unsigned counter0_lo;
  70         unsigned select[4];
  71         unsigned counters[4];
  72         unsigned num_multi;
  73         unsigned num_prelude;
  74         unsigned layout;
  75 };
  76
  77 struct si_pc_block {
  78         const struct si_pc_block_base *b;
  79         unsigned selectors;
  80         unsigned instances;
  81 };
  82
  83 /* The order is chosen to be compatible with GPUPerfStudio's hardcoding of
  84  * performance counter group IDs.
  85  */
  86 static const char * const si_pc_shader_type_suffixes[] = {
  87         "", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS"
  88 };
  89
  90 static const unsigned si_pc_shader_type_bits[] = {
  91         0x7f,
  92         S_036780_ES_EN(1),
  93         S_036780_GS_EN(1),
  94         S_036780_VS_EN(1),
  95         S_036780_PS_EN(1),
  96         S_036780_LS_EN(1),
  97         S_036780_HS_EN(1),
  98         S_036780_CS_EN(1),
  99 };
 100
 101 static const struct si_pc_block_base cik_CB = {
 102         .name = "CB",
 103         .num_counters = 4,
 104         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS,
 105
 106         .select0 = R_037000_CB_PERFCOUNTER_FILTER,
 107         .counter0_lo = R_035018_CB_PERFCOUNTER0_LO,
 108         .num_multi = 1,
 109         .num_prelude = 1,
 110         .layout = SI_PC_MULTI_ALTERNATE,
 111 };
 112
 113 static const struct si_pc_block_base cik_CPC = {
 114         .name = "CPC",
 115         .num_counters = 2,
 116
 117         .select = { R_036024_CPC_PERFCOUNTER0_SELECT,
 118                     R_036010_CPC_PERFCOUNTER0_SELECT1,
 119                     R_03600C_CPC_PERFCOUNTER1_SELECT },
 120         .counter0_lo = R_034018_CPC_PERFCOUNTER0_LO,
 121         .num_multi = 1,
 122         .layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE,
 123 };
 124
 125 static const struct si_pc_block_base cik_CPF = {
 126         .name = "CPF",
 127         .num_counters = 2,
 128
 129         .select0 = R_03601C_CPF_PERFCOUNTER0_SELECT,
 130         .counter0_lo = R_034028_CPF_PERFCOUNTER0_LO,
 131         .num_multi = 1,
 132         .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
 133 };
 134
 135 static const struct si_pc_block_base cik_CPG = {
 136         .name = "CPG",
 137         .num_counters = 2,
 138
 139         .select0 = R_036008_CPG_PERFCOUNTER0_SELECT,
 140         .counter0_lo = R_034008_CPG_PERFCOUNTER0_LO,
 141         .num_multi = 1,
 142         .layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
 143 };
 144
 145 static const struct si_pc_block_base cik_DB = {
 146         .name = "DB",
 147         .num_counters = 4,
 148         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS,
 149
 150         .select0 = R_037100_DB_PERFCOUNTER0_SELECT,
 151         .counter0_lo = R_035100_DB_PERFCOUNTER0_LO,
 152         .num_multi = 3, // really only 2, but there's a gap between registers
 153         .layout = SI_PC_MULTI_ALTERNATE,
 154 };
 155
 156 static const struct si_pc_block_base cik_GDS = {
 157         .name = "GDS",
 158         .num_counters = 4,
 159
 160         .select0 = R_036A00_GDS_PERFCOUNTER0_SELECT,
 161         .counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO,
 162         .num_multi = 1,
 163         .layout = SI_PC_MULTI_TAIL,
 164 };
 165
 166 static const struct si_pc_block_base cik_GRBM = {
 167         .name = "GRBM",
 168         .num_counters = 2,
 169
 170         .select0 = R_036100_GRBM_PERFCOUNTER0_SELECT,
 171         .counters = { R_034100_GRBM_PERFCOUNTER0_LO,
 172                       R_03410C_GRBM_PERFCOUNTER1_LO },
 173 };
 174
 175 static const struct si_pc_block_base cik_GRBMSE = {
 176         .name = "GRBMSE",
 177         .num_counters = 4,
 178
 179         .select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT,
 180         .counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO,
 181 };
 182
 183 static const struct si_pc_block_base cik_IA = {
 184         .name = "IA",
 185         .num_counters = 4,
 186
 187         .select0 = R_036210_IA_PERFCOUNTER0_SELECT,
 188         .counter0_lo = R_034220_IA_PERFCOUNTER0_LO,
 189         .num_multi = 1,
 190         .layout = SI_PC_MULTI_TAIL,
 191 };
 192
 193 static const struct si_pc_block_base cik_PA_SC = {
 194         .name = "PA_SC",
 195         .num_counters = 8,
 196         .flags = R600_PC_BLOCK_SE,
 197
 198         .select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT,
 199         .counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO,
 200         .num_multi = 1,
 201         .layout = SI_PC_MULTI_ALTERNATE,
 202 };
 203
 204 /* According to docs, PA_SU counters are only 48 bits wide. */
 205 static const struct si_pc_block_base cik_PA_SU = {
 206         .name = "PA_SU",
 207         .num_counters = 4,
 208         .flags = R600_PC_BLOCK_SE,
 209
 210         .select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT,
 211         .counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO,
 212         .num_multi = 2,
 213         .layout = SI_PC_MULTI_ALTERNATE,
 214 };
 215
 216 static const struct si_pc_block_base cik_SPI = {
 217         .name = "SPI",
 218         .num_counters = 6,
 219         .flags = R600_PC_BLOCK_SE,
 220
 221         .select0 = R_036600_SPI_PERFCOUNTER0_SELECT,
 222         .counter0_lo = R_034604_SPI_PERFCOUNTER0_LO,
 223         .num_multi = 4,
 224         .layout = SI_PC_MULTI_BLOCK,
 225 };
 226
 227 static const struct si_pc_block_base cik_SQ = {
 228         .name = "SQ",
 229         .num_counters = 16,
 230         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_SHADER,
 231
 232         .select0 = R_036700_SQ_PERFCOUNTER0_SELECT,
 233         .select_or = S_036700_SQC_BANK_MASK(15) |
 234                         S_036700_SQC_CLIENT_MASK(15) |
 235                         S_036700_SIMD_MASK(15),
 236         .counter0_lo = R_034700_SQ_PERFCOUNTER0_LO,
 237 };
 238
 239 static const struct si_pc_block_base cik_SX = {
 240         .name = "SX",
 241         .num_counters = 4,
 242         .flags = R600_PC_BLOCK_SE,
 243
 244         .select0 = R_036900_SX_PERFCOUNTER0_SELECT,
 245         .counter0_lo = R_034900_SX_PERFCOUNTER0_LO,
 246         .num_multi = 2,
 247         .layout = SI_PC_MULTI_TAIL,
 248 };
 249
 250 static const struct si_pc_block_base cik_TA = {
 251         .name = "TA",
 252         .num_counters = 2,
 253         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
 254
 255         .select0 = R_036B00_TA_PERFCOUNTER0_SELECT,
 256         .counter0_lo = R_034B00_TA_PERFCOUNTER0_LO,
 257         .num_multi = 1,
 258         .layout = SI_PC_MULTI_ALTERNATE,
 259 };
 260
 261 static const struct si_pc_block_base cik_TD = {
 262         .name = "TD",
 263         .num_counters = 2,
 264         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
 265
 266         .select0 = R_036C00_TD_PERFCOUNTER0_SELECT,
 267         .counter0_lo = R_034C00_TD_PERFCOUNTER0_LO,
 268         .num_multi = 1,
 269         .layout = SI_PC_MULTI_ALTERNATE,
 270 };
 271
 272 static const struct si_pc_block_base cik_TCA = {
 273         .name = "TCA",
 274         .num_counters = 4,
 275         .flags = R600_PC_BLOCK_INSTANCE_GROUPS,
 276
 277         .select0 = R_036E40_TCA_PERFCOUNTER0_SELECT,
 278         .counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO,
 279         .num_multi = 2,
 280         .layout = SI_PC_MULTI_ALTERNATE,
 281 };
 282
 283 static const struct si_pc_block_base cik_TCC = {
 284         .name = "TCC",
 285         .num_counters = 4,
 286         .flags = R600_PC_BLOCK_INSTANCE_GROUPS,
 287
 288         .select0 = R_036E00_TCC_PERFCOUNTER0_SELECT,
 289         .counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO,
 290         .num_multi = 2,
 291         .layout = SI_PC_MULTI_ALTERNATE,
 292 };
 293
 294 static const struct si_pc_block_base cik_TCP = {
 295         .name = "TCP",
 296         .num_counters = 4,
 297         .flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
 298
 299         .select0 = R_036D00_TCP_PERFCOUNTER0_SELECT,
 300         .counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO,
 301         .num_multi = 2,
 302         .layout = SI_PC_MULTI_ALTERNATE,
 303 };
 304
 305 static const struct si_pc_block_base cik_VGT = (const struct si_pc_block_base) {
 306         .name = "VGT",
 307         .num_counters = 4,
 308         .flags = R600_PC_BLOCK_SE,
 309
 310         .select0 = R_036230_VGT_PERFCOUNTER0_SELECT,
 311         .counter0_lo = R_034240_VGT_PERFCOUNTER0_LO,
 312         .num_multi = 1,
 313         .layout = SI_PC_MULTI_TAIL,
 314 };
 315
 316 static const struct si_pc_block_base cik_WD = {
 317         .name = "WD",
 318         .num_counters = 4,
 319
 320         .select0 = R_036200_WD_PERFCOUNTER0_SELECT,
 321         .counter0_lo = R_034200_WD_PERFCOUNTER0_LO,
 322 };
 323
 324 static const struct si_pc_block_base cik_MC = {
 325         .name = "MC",
 326         .num_counters = 4,
 327
 328         .layout = SI_PC_FAKE,
 329 };
 330
 331 static const struct si_pc_block_base cik_SRBM = {
 332         .name = "SRBM",
 333         .num_counters = 2,
 334
 335         .layout = SI_PC_FAKE,
 336 };
 337
 338 /* Both the number of instances and selectors varies between chips of the same
 339  * class. We only differentiate by class here and simply expose the maximum
 340  * number over all chips in a class.
 341  *
 342  * Unfortunately, GPUPerfStudio uses the order of performance counter groups
 343  * blindly once it believes it has identified the hardware, so the order of
 344  * blocks here matters.
 345  */
 346 static const struct si_pc_block groups_CIK[] = {
 347         { &cik_CB, 226, 4 },
 348         { &cik_CPF, 17 },
 349         { &cik_DB, 257, 4 },
 350         { &cik_GRBM, 34 },
 351         { &cik_GRBMSE, 15 },
 352         { &cik_PA_SU, 153 },
 353         { &cik_PA_SC, 395 },
 354         { &cik_SPI, 186 },
 355         { &cik_SQ, 252 },
 356         { &cik_SX, 32 },
 357         { &cik_TA, 111, 11 },
 358         { &cik_TCA, 39, 2 },
 359         { &cik_TCC, 160, 16 },
 360         { &cik_TD, 55, 11 },
 361         { &cik_TCP, 154, 11 },
 362         { &cik_GDS, 121 },
 363         { &cik_VGT, 140 },
 364         { &cik_IA, 22 },
 365         { &cik_MC, 22 },
 366         { &cik_SRBM, 19 },
 367         { &cik_WD, 22 },
 368         { &cik_CPG, 46 },
 369         { &cik_CPC, 22 },
 370
 371 };
 372
 373 static const struct si_pc_block groups_VI[] = {
 374         { &cik_CB, 396, 4 },
 375         { &cik_CPF, 19 },
 376         { &cik_DB, 257, 4 },
 377         { &cik_GRBM, 34 },
 378         { &cik_GRBMSE, 15 },
 379         { &cik_PA_SU, 153 },
 380         { &cik_PA_SC, 397 },
 381         { &cik_SPI, 197 },
 382         { &cik_SQ, 273 },
 383         { &cik_SX, 34 },
 384         { &cik_TA, 119, 16 },
 385         { &cik_TCA, 35, 2 },
 386         { &cik_TCC, 192, 16 },
 387         { &cik_TD, 55, 16 },
 388         { &cik_TCP, 180, 16 },
 389         { &cik_GDS, 121 },
 390         { &cik_VGT, 147 },
 391         { &cik_IA, 24 },
 392         { &cik_MC, 22 },
 393         { &cik_SRBM, 27 },
 394         { &cik_WD, 37 },
 395         { &cik_CPG, 48 },
 396         { &cik_CPC, 24 },
 397
 398 };
 399
 400 static void si_pc_get_size(struct r600_perfcounter_block *group,
 401                         unsigned count, unsigned *selectors,
 402                         unsigned *num_select_dw, unsigned *num_read_dw)
 403 {
 404         const struct si_pc_block *sigroup = (const struct si_pc_block *)group->data;
 405         const struct si_pc_block_base *regs = sigroup->b;
 406         unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
 407
 408         if (regs->layout & SI_PC_FAKE) {
 409                 *num_select_dw = 0;
 410         } else if (layout_multi == SI_PC_MULTI_BLOCK) {
 411                 if (count < regs->num_multi)
 412                         *num_select_dw = 2 * (count + 2) + regs->num_prelude;
 413                 else
 414                         *num_select_dw = 2 + count + regs->num_multi + regs->num_prelude;
 415         } else if (layout_multi == SI_PC_MULTI_TAIL) {
 416                 *num_select_dw = 4 + count + MIN2(count, regs->num_multi) + regs->num_prelude;
 417         } else if (layout_multi == SI_PC_MULTI_CUSTOM) {
 418                 assert(regs->num_prelude == 0);
 419                 *num_select_dw = 3 * (count + MIN2(count, regs->num_multi));
 420         } else {
 421                 assert(layout_multi == SI_PC_MULTI_ALTERNATE);
 422
 423                 *num_select_dw = 2 + count + MIN2(count, regs->num_multi) + regs->num_prelude;
 424         }
 425
 426         *num_read_dw = 6 * count;
 427 }
 428
 429 static void si_pc_emit_instance(struct r600_common_context *ctx,
 430                                 int se, int instance)
 431 {
 432         struct radeon_winsys_cs *cs = ctx->gfx.cs;
 433         unsigned value = S_030800_SH_BROADCAST_WRITES(1);
 434
 435         if (se >= 0) {
 436                 value |= S_030800_SE_INDEX(se);
 437         } else {
 438                 value |= S_030800_SE_BROADCAST_WRITES(1);
 439         }
 440
 441         if (instance >= 0) {
 442                 value |= S_030800_INSTANCE_INDEX(instance);
 443         } else {
 444                 value |= S_030800_INSTANCE_BROADCAST_WRITES(1);
 445         }
 446
 447         radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value);
 448 }
 449
 450 static void si_pc_emit_shaders(struct r600_common_context *ctx,
 451                                unsigned shaders)
 452 {
 453         struct radeon_winsys_cs *cs = ctx->gfx.cs;
 454
 455         radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2);
 456         radeon_emit(cs, shaders & 0x7f);
 457         radeon_emit(cs, 0xffffffff);
 458 }
 459
 460 static void si_pc_emit_select(struct r600_common_context *ctx,
 461                         struct r600_perfcounter_block *group,
 462                         unsigned count, unsigned *selectors)
 463 {
 464         const struct si_pc_block *sigroup = (const struct si_pc_block *)group->data;
 465         const struct si_pc_block_base *regs = sigroup->b;
 466         struct radeon_winsys_cs *cs = ctx->gfx.cs;
 467         unsigned idx;
 468         unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
 469         unsigned dw;
 470
 471         assert(count <= regs->num_counters);
 472
 473         if (regs->layout & SI_PC_FAKE)
 474                 return;
 475
 476         if (layout_multi == SI_PC_MULTI_BLOCK) {
 477                 assert(!(regs->layout & SI_PC_REG_REVERSE));
 478
 479                 dw = count + regs->num_prelude;
 480                 if (count >= regs->num_multi)
 481                         dw += regs->num_multi;
 482                 radeon_set_uconfig_reg_seq(cs, regs->select0, dw);
 483                 for (idx = 0; idx < regs->num_prelude; ++idx)
 484                         radeon_emit(cs, 0);
 485                 for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
 486                         radeon_emit(cs, selectors[idx] | regs->select_or);
 487
 488                 if (count < regs->num_multi) {
 489                         unsigned select1 =
 490                                 regs->select0 + 4 * regs->num_multi;
 491                         radeon_set_uconfig_reg_seq(cs, select1, count);
 492                 }
 493
 494                 for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
 495                         radeon_emit(cs, 0);
 496
 497                 if (count > regs->num_multi) {
 498                         for (idx = regs->num_multi; idx < count; ++idx)
 499                                 radeon_emit(cs, selectors[idx] | regs->select_or);
 500                 }
 501         } else if (layout_multi == SI_PC_MULTI_TAIL) {
 502                 unsigned select1, select1_count;
 503
 504                 assert(!(regs->layout & SI_PC_REG_REVERSE));
 505
 506                 radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude);
 507                 for (idx = 0; idx < regs->num_prelude; ++idx)
 508                         radeon_emit(cs, 0);
 509                 for (idx = 0; idx < count; ++idx)
 510                         radeon_emit(cs, selectors[idx] | regs->select_or);
 511
 512                 select1 = regs->select0 + 4 * regs->num_counters;
 513                 select1_count = MIN2(count, regs->num_multi);
 514                 radeon_set_uconfig_reg_seq(cs, select1, select1_count);
 515                 for (idx = 0; idx < select1_count; ++idx)
 516                         radeon_emit(cs, 0);
 517         } else if (layout_multi == SI_PC_MULTI_CUSTOM) {
 518                 const unsigned *reg = regs->select;
 519                 for (idx = 0; idx < count; ++idx) {
 520                         radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or);
 521                         if (idx < regs->num_multi)
 522                                 radeon_set_uconfig_reg(cs, *reg++, 0);
 523                 }
 524         } else {
 525                 assert(layout_multi == SI_PC_MULTI_ALTERNATE);
 526
 527                 unsigned reg_base = regs->select0;
 528                 unsigned reg_count = count + MIN2(count, regs->num_multi);
 529                 reg_count += regs->num_prelude;
 530
 531                 if (!(regs->layout & SI_PC_REG_REVERSE)) {
 532                         radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);
 533
 534                         for (idx = 0; idx < regs->num_prelude; ++idx)
 535                                 radeon_emit(cs, 0);
 536                         for (idx = 0; idx < count; ++idx) {
 537                                 radeon_emit(cs, selectors[idx] | regs->select_or);
 538                                 if (idx < regs->num_multi)
 539                                         radeon_emit(cs, 0);
 540                         }
 541                 } else {
 542                         reg_base -= (reg_count - 1) * 4;
 543                         radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);
 544
 545                         for (idx = count; idx > 0; --idx) {
 546                                 if (idx <= regs->num_multi)
 547                                         radeon_emit(cs, 0);
 548                                 radeon_emit(cs, selectors[idx - 1] | regs->select_or);
 549                         }
 550                         for (idx = 0; idx < regs->num_prelude; ++idx)
 551                                 radeon_emit(cs, 0);
 552                 }
 553         }
 554 }
 555
 556 static void si_pc_emit_start(struct r600_common_context *ctx,
 557                              struct r600_resource *buffer, uint64_t va)
 558 {
 559         struct radeon_winsys_cs *cs = ctx->gfx.cs;
 560
 561         radeon_add_to_buffer_list(ctx, &ctx->gfx, buffer,
 562                                   RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
 563
 564         radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 565         radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
 566                         COPY_DATA_DST_SEL(COPY_DATA_MEM));
 567         radeon_emit(cs, 1); /* immediate */
 568         radeon_emit(cs, 0); /* unused */
 569         radeon_emit(cs, va);
 570         radeon_emit(cs, va >> 32);
 571
 572         radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
 573                                S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET));
 574         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 575         radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PERFCOUNTER_START) | EVENT_INDEX(0));
 576         radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
 577                                S_036020_PERFMON_STATE(V_036020_START_COUNTING));
 578 }
 579
 580 /* Note: The buffer was already added in si_pc_emit_start, so we don't have to
 581  * do it again in here. */
 582 static void si_pc_emit_stop(struct r600_common_context *ctx,
 583                             struct r600_resource *buffer, uint64_t va)
 584 {
 585         struct radeon_winsys_cs *cs = ctx->gfx.cs;
 586
 587         r600_gfx_write_event_eop(ctx, EVENT_TYPE_BOTTOM_OF_PIPE_TS, 0, 1,
 588                                  buffer, va, 1, 0);
 589         r600_gfx_wait_fence(ctx, va, 0, 0xffffffff);
 590
 591         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 592         radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0));
 593         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 594         radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PERFCOUNTER_STOP) | EVENT_INDEX(0));
 595         radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
 596                                S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) |
 597                                S_036020_PERFMON_SAMPLE_ENABLE(1));
 598 }
 599
 600 static void si_pc_emit_read(struct r600_common_context *ctx,
 601                             struct r600_perfcounter_block *group,
 602                             unsigned count, unsigned *selectors,
 603                             struct r600_resource *buffer, uint64_t va)
 604 {
 605         const struct si_pc_block *sigroup = (const struct si_pc_block *)group->data;
 606         const struct si_pc_block_base *regs = sigroup->b;
 607         struct radeon_winsys_cs *cs = ctx->gfx.cs;
 608         unsigned idx;
 609         unsigned reg = regs->counter0_lo;
 610         unsigned reg_delta = 8;
 611
 612         if (!(regs->layout & SI_PC_FAKE)) {
 613                 if (regs->layout & SI_PC_REG_REVERSE)
 614                         reg_delta = -reg_delta;
 615
 616                 for (idx = 0; idx < count; ++idx) {
 617                         if (regs->counters)
 618                                 reg = regs->counters[idx];
 619
 620                         radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 621                         radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) |
 622                                         COPY_DATA_DST_SEL(COPY_DATA_MEM) |
 623                                         COPY_DATA_COUNT_SEL); /* 64 bits */
 624                         radeon_emit(cs, reg >> 2);
 625                         radeon_emit(cs, 0); /* unused */
 626                         radeon_emit(cs, va);
 627                         radeon_emit(cs, va >> 32);
 628                         va += sizeof(uint64_t);
 629                         reg += reg_delta;
 630                 }
 631         } else {
 632                 for (idx = 0; idx < count; ++idx) {
 633                         radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 634                         radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
 635                                         COPY_DATA_DST_SEL(COPY_DATA_MEM) |
 636                                         COPY_DATA_COUNT_SEL);
 637                         radeon_emit(cs, 0); /* immediate */
 638                         radeon_emit(cs, 0);
 639                         radeon_emit(cs, va);
 640                         radeon_emit(cs, va >> 32);
 641                         va += sizeof(uint64_t);
 642                 }
 643         }
 644 }
 645
 646 static void si_pc_cleanup(struct r600_common_screen *rscreen)
 647 {
 648         r600_perfcounters_do_destroy(rscreen->perfcounters);
 649         rscreen->perfcounters = NULL;
 650 }
 651
 652 void si_init_perfcounters(struct si_screen *screen)
 653 {
 654         struct r600_perfcounters *pc;
 655         const struct si_pc_block *blocks;
 656         unsigned num_blocks;
 657         unsigned i;
 658
 659         switch (screen->b.chip_class) {
 660         case CIK:
 661                 blocks = groups_CIK;
 662                 num_blocks = ARRAY_SIZE(groups_CIK);
 663                 break;
 664         case VI:
 665                 blocks = groups_VI;
 666                 num_blocks = ARRAY_SIZE(groups_VI);
 667                 break;
 668         case SI:
 669         case GFX9:
 670         default:
 671                 return; /* not implemented */
 672         }
 673
 674         if (screen->b.info.max_sh_per_se != 1) {
 675                 /* This should not happen on non-SI chips. */
 676                 fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not "
 677                         "supported (inaccurate performance counters)\n",
 678                         screen->b.info.max_sh_per_se);
 679         }
 680
 681         pc = CALLOC_STRUCT(r600_perfcounters);
 682         if (!pc)
 683                 return;
 684
 685         pc->num_start_cs_dwords = 14;
 686         pc->num_stop_cs_dwords = 14 + r600_gfx_write_fence_dwords(&screen->b);
 687         pc->num_instance_cs_dwords = 3;
 688         pc->num_shaders_cs_dwords = 4;
 689
 690         pc->num_shader_types = ARRAY_SIZE(si_pc_shader_type_bits);
 691         pc->shader_type_suffixes = si_pc_shader_type_suffixes;
 692         pc->shader_type_bits = si_pc_shader_type_bits;
 693
 694         pc->get_size = si_pc_get_size;
 695         pc->emit_instance = si_pc_emit_instance;
 696         pc->emit_shaders = si_pc_emit_shaders;
 697         pc->emit_select = si_pc_emit_select;
 698         pc->emit_start = si_pc_emit_start;
 699         pc->emit_stop = si_pc_emit_stop;
 700         pc->emit_read = si_pc_emit_read;
 701         pc->cleanup = si_pc_cleanup;
 702
 703         if (!r600_perfcounters_init(pc, num_blocks))
 704                 goto error;
 705
 706         for (i = 0; i < num_blocks; ++i) {
 707                 const struct si_pc_block *block = &blocks[i];
 708                 unsigned instances = block->instances;
 709
 710                 if (!strcmp(block->b->name, "IA")) {
 711                         if (screen->b.info.max_se > 2)
 712                                 instances = 2;
 713                 }
 714
 715                 r600_perfcounters_add_block(&screen->b, pc,
 716                                             block->b->name,
 717                                             block->b->flags,
 718                                             block->b->num_counters,
 719                                             block->selectors,
 720                                             instances,
 721                                             block);
 722         }
 723
 724         screen->b.perfcounters = pc;
 725         return;
 726
 727 error:
 728         r600_perfcounters_do_destroy(pc);
 729 }