2 * Copyright 2011 Christoph Bumiller
3 * Copyright 2015 Samuel Pitoiset
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
26 #include "nvc0/nvc0_context.h"
27 #include "nvc0/nvc0_query_hw_sm.h"
29 #include "nv_object.xml.h"
30 #include "nvc0/nve4_compute.xml.h"
31 #include "nvc0/nvc0_compute.xml.h"
33 /* NOTE: intentionally using the same names as NV */
34 #define _Q(t, n, d) { NVC0_HW_SM_QUERY_##t, n, d }
39 } nvc0_hw_sm_queries
[] = {
42 "Accumulated number of active blocks per cycle. For every cycle it "
43 "increments by the number of active blocks in the cycle which can be in "
44 "the range 0 to 32."),
48 "Number of cycles a multiprocessor has at least one active warp"),
52 "Accumulated number of active warps per cycle. For every cycle it "
53 "increments by the number of active warps in the cycle which can be in "
58 "Number of warps executing atomic compare and swap operations. Increments "
59 "by one if at least one thread in a warp executes the instruction."),
63 "Number of warps executing atomic reduction operations. Increments by one "
64 "if at least one thread in a warp executes the instruction"),
68 "Number of branch instructions executed per warp on a multiprocessor"),
72 "Number of divergent branches within a warp. This counter will be "
73 "incremented by one if at least one thread in a warp diverges (that is, "
74 "follows a different execution path) via a conditional branch"),
78 "Number of executed load instructions where the state space is not "
79 "specified and hence generic addressing is used, increments per warp on a "
80 "multiprocessor. It can include the load operations from global,local and "
81 "shared state space"),
83 _Q(GLD_MEM_DIV_REPLAY
,
84 "global_ld_mem_divergence_replays",
85 "Number of instruction replays for global memory loads. Instruction is "
86 "replayed if the instruction is accessing more than one cache line of "
87 "128 bytes. For each extra cache line access the counter is incremented "
92 "Number of ATOM.CAS instructions executed per warp."),
96 "Number of executed load instructions where state space is specified as "
97 "global, increments per warp on a multiprocessor."),
101 "Number of executed store instructions where state space is specified as "
102 "global, increments per warp on a multiprocessor."),
105 "global_store_transaction",
106 "Number of global store transactions. Increments by 1 per transaction. "
107 "Transaction can be 32/64/96/128B"),
109 _Q(GST_MEM_DIV_REPLAY
,
110 "global_st_mem_divergence_replays",
111 "Number of instruction replays for global memory stores. Instruction is "
112 "replayed if the instruction is accessing more than one cache line of "
113 "128 bytes. For each extra cache line access the counter is incremented "
118 "Number of warps executing reduction operations on global memory. "
119 "Increments by one if at least one thread in a warp executes the "
124 "Number of executed store instructions where the state space is not "
125 "specified and hence generic addressing is used, increments per warp on a "
126 "multiprocessor. It can include the store operations to global,local and "
127 "shared state space"),
131 "Number of instructions executed, do not include replays"),
135 "Number of instructions issued including replays"),
139 "Number of cycles that did not issue any instruction, increments per "
144 "Number of single instruction issued per cycle"),
148 "Number of dual instructions issued per cycle"),
152 "Number of single instruction issued per cycle in pipeline 0"),
156 "Number of single instruction issued per cycle in pipeline 1"),
160 "Number of dual instructions issued per cycle in pipeline 0"),
164 "Number of dual instructions issued per cycle in pipeline 1"),
167 "l1_global_load_hit",
168 "Number of cache lines that hit in L1 cache for global memory load "
169 "accesses. In case of perfect coalescing this increments by 1,2, and 4 for "
170 "32, 64 and 128 bit accesses by a warp respectively"),
173 "l1_global_load_miss",
174 "Number of cache lines that miss in L1 cache for global memory load "
175 "accesses. In case of perfect coalescing this increments by 1,2, and 4 for "
176 "32, 64 and 128 bit accesses by a warp respectively"),
178 _Q(L1_GLD_TRANSACTIONS
,
179 "__l1_global_load_transactions",
180 "Number of global load transactions from L1 cache. Increments by 1 per "
181 "transaction. Transaction can be 32/64/96/128B"),
183 _Q(L1_GST_TRANSACTIONS
,
184 "__l1_global_store_transactions",
185 "Number of global store transactions from L1 cache. Increments by 1 per "
186 "transaction. Transaction can be 32/64/96/128B"),
190 "Number of cache lines that hit in L1 cache for local memory load "
191 "accesses. In case of perfect coalescing this increments by 1,2, and 4 for "
192 "32, 64 and 128 bit accesses by a warp respectively"),
195 "l1_local_load_miss",
196 "Number of cache lines that miss in L1 cache for local memory load "
197 "accesses. In case of perfect coalescing this increments by 1,2, and 4 for "
198 "32, 64 and 128 bit accesses by a warp respectively"),
201 "l1_local_store_hit",
202 "Number of cache lines that hit in L1 cache for local memory store "
203 "accesses. In case of perfect coalescing this increments by 1,2, and 4 for "
204 "32, 64 and 128 bit accesses by a warp respectively"),
207 "l1_local_store_miss",
208 "Number of cache lines that miss in L1 cache for local memory store "
209 "accesses. In case of perfect coalescing this increments by 1,2, and 4 for "
210 "32,64 and 128 bit accesses by a warp respectively"),
212 _Q(L1_SHARED_LD_TRANSACTIONS
,
213 "l1_shared_load_transactions",
214 "Number of shared load transactions. Increments by 1 per transaction. "
215 "Transaction can be 32/64/96/128B"),
217 _Q(L1_SHARED_ST_TRANSACTIONS
,
218 "l1_shared_store_transactions",
219 "Number of shared store transactions. Increments by 1 per transaction. "
220 "Transaction can be 32/64/96/128B"),
224 "Number of executed load instructions where state space is specified as "
225 "local, increments per warp on a multiprocessor"),
227 _Q(LOCAL_LD_TRANSACTIONS
,
228 "local_load_transactions",
229 "Number of local load transactions from L1 cache. Increments by 1 per "
230 "transaction. Transaction can be 32/64/96/128B"),
234 "Number of executed store instructions where state space is specified as "
235 "local, increments per warp on a multiprocessor"),
237 _Q(LOCAL_ST_TRANSACTIONS
,
238 "local_store_transactions",
239 "Number of local store transactions to L1 cache. Increments by 1 per "
240 "transaction. Transaction can be 32/64/96/128B."),
242 _Q(NOT_PRED_OFF_INST_EXECUTED
,
243 "not_predicated_off_thread_inst_executed",
244 "Number of not predicated off instructions executed by all threads, does "
245 "not include replays. For each instruction it increments by the number of "
246 "threads that execute this instruction"),
250 "User profiled generic trigger that can be inserted in any place of the "
251 "code to collect the related information. Increments per warp."),
255 "User profiled generic trigger that can be inserted in any place of the "
256 "code to collect the related information. Increments per warp."),
260 "User profiled generic trigger that can be inserted in any place of the "
261 "code to collect the related information. Increments per warp."),
265 "User profiled generic trigger that can be inserted in any place of the "
266 "code to collect the related information. Increments per warp."),
270 "User profiled generic trigger that can be inserted in any place of the "
271 "code to collect the related information. Increments per warp."),
275 "User profiled generic trigger that can be inserted in any place of the "
276 "code to collect the related information. Increments per warp."),
280 "User profiled generic trigger that can be inserted in any place of the "
281 "code to collect the related information. Increments per warp."),
285 "User profiled generic trigger that can be inserted in any place of the "
286 "code to collect the related information. Increments per warp."),
290 "Number of ATOMS instructions executed per warp."),
294 "Number of ATOMS.CAS instructions executed per warp."),
298 "Number of executed load instructions where state space is specified as "
299 "shared, increments per warp on a multiprocessor"),
301 _Q(SHARED_LD_BANK_CONFLICT
,
302 "shared_load_bank_conflict",
303 "Number of shared load bank conflict generated when the addresses for "
304 "two or more shared memory load requests fall in the same memory bank."),
307 "shared_load_replay",
308 "Replays caused due to shared load bank conflict (when the addresses for "
309 "two or more shared memory load requests fall in the same memory bank) or "
310 "when there is no conflict but the total number of words accessed by all "
311 "threads in the warp executing that instruction exceed the number of words "
312 "that can be loaded in one cycle (256 bytes)"),
314 _Q(SHARED_LD_TRANSACTIONS
,
315 "shared_ld_transactions",
316 "Number of transactions for shared load accesses. Maximum transaction "
317 "size in maxwell is 128 bytes, any warp accessing more that 128 bytes "
318 "will cause multiple transactions for a shared load instruction. This "
319 "also includes extra transactions caused by shared bank conflicts."),
323 "Number of executed store instructions where state space is specified as "
324 "shared, increments per warp on a multiprocessor"),
326 _Q(SHARED_ST_BANK_CONFLICT
,
327 "shared_store_bank_conflict",
328 "Number of shared store bank conflict generated when the addresses for "
329 "two or more shared memory store requests fall in the same memory bank."),
332 "shared_store_replay",
333 "Replays caused due to shared store bank conflict (when the addresses for "
334 "two or more shared memory store requests fall in the same memory bank) or "
335 "when there is no conflict but the total number of words accessed by all "
336 "threads in the warp executing that instruction exceed the number of words "
337 "that can be stored in one cycle"),
339 _Q(SHARED_ST_TRANSACTIONS
,
340 "shared_st_transactions",
341 "Number of transactions for shared store accesses. Maximum transaction "
342 "size in maxwell is 128 bytes, any warp accessing more that 128 bytes "
343 "will cause multiple transactions for a shared store instruction. This "
344 "also includes extra transactions caused by shared bank conflicts."),
348 "Number of thread blocks launched on a multiprocessor"),
352 "Number of threads launched on a multiprocessor"),
355 "thread_inst_executed",
356 "Number of instructions executed by all threads, does not include "
357 "replays. For each instruction it increments by the number of threads in "
358 "the warp that execute the instruction"),
360 _Q(TH_INST_EXECUTED_0
,
361 "thread_inst_executed_0",
362 "Number of instructions executed by all threads, does not include "
363 "replays. For each instruction it increments by the number of threads in "
364 "the warp that execute the instruction in pipeline 0"),
366 _Q(TH_INST_EXECUTED_1
,
367 "thread_inst_executed_1",
368 "Number of instructions executed by all threads, does not include "
369 "replays. For each instruction it increments by the number of threads in "
370 "the warp that execute the instruction in pipeline 1"),
372 _Q(TH_INST_EXECUTED_2
,
373 "thread_inst_executed_2",
374 "Number of instructions executed by all threads, does not include "
375 "replays. For each instruction it increments by the number of threads in "
376 "the warp that execute the instruction in pipeline 2"),
378 _Q(TH_INST_EXECUTED_3
,
379 "thread_inst_executed_3",
380 "Number of instructions executed by all threads, does not include "
381 "replays. For each instruction it increments by the number of threads in "
382 "the warp that execute the instruction in pipeline 3"),
384 _Q(UNCACHED_GLD_TRANSACTIONS
,
385 "uncached_global_load_transaction",
386 "Number of uncached global load transactions. Increments by 1 per "
387 "transaction. Transaction can be 32/64/96/128B."),
391 "Number of warps launched on a multiprocessor"),
396 static inline const char *
397 nvc0_hw_sm_query_get_name(unsigned query_type
)
401 for (i
= 0; i
< ARRAY_SIZE(nvc0_hw_sm_queries
); i
++) {
402 if (nvc0_hw_sm_queries
[i
].type
== query_type
)
403 return nvc0_hw_sm_queries
[i
].name
;
409 /* === PERFORMANCE MONITORING COUNTERS for NVE4+ === */
411 /* Code to read out MP counters: They are accessible via mmio, too, but let's
412 * just avoid mapping registers in userspace. We'd have to know which MPs are
413 * enabled/present, too, and that information is not presently exposed.
414 * We could add a kernel interface for it, but reading the counters like this
415 * has the advantage of being async (if get_result isn't called immediately).
417 static const uint64_t nve4_read_hw_sm_counters_code
[] =
419 /* sched 0x20 0x20 0x20 0x20 0x20 0x20 0x20
421 * mov b32 $r12 $physid
427 * sched 0x20 0x20 0x23 0x04 0x20 0x04 0x2b
431 * set $p0 0x1 eq u32 $r8 0x0
432 * mov b32 $r10 c7[0x620]
433 * ext u32 $r8 $r12 0x414
434 * mov b32 $r11 c7[0x624]
435 * sched 0x04 0x2e 0x04 0x20 0x20 0x28 0x04
436 * ext u32 $r9 $r12 0x208
438 * set $p1 0x1 eq u32 $r9 0x0
439 * mul $r8 u32 $r8 u32 96
440 * mul $r12 u32 $r9 u32 16
441 * mul $r13 u32 $r9 u32 4
442 * add b32 $r9 $r8 $r13
443 * sched 0x28 0x04 0x2c 0x04 0x2c 0x04 0x2c
444 * add b32 $r8 $r8 $r12
446 * add b32 $r10 $c $r10 $r8
448 * add b32 $r11 $r11 0x0 $c
449 * add b32 $r12 $c $r12 $r9
450 * st b128 wt g[$r10d] $r0q
451 * sched 0x4 0x2c 0x20 0x04 0x2e 0x00 0x00
452 * mov b32 $r0 c7[0x628]
453 * add b32 $r13 $r13 0x0 $c
454 * $p1 st b128 wt g[$r12d+0x40] $r4q
455 * st b32 wt g[$r12d+0x50] $r0
457 0x2202020202020207ULL
,
458 0x2c00000084021c04ULL
,
459 0x2c0000000c031c04ULL
,
460 0x2c00000010001c04ULL
,
461 0x2c00000014005c04ULL
,
462 0x2c00000018009c04ULL
,
463 0x2c0000001c00dc04ULL
,
464 0x2c00000020011c04ULL
,
465 0x22b0420042320207ULL
,
466 0x2c00000024015c04ULL
,
467 0x2c00000028019c04ULL
,
468 0x2c0000002c01dc04ULL
,
469 0x190e0000fc81dc03ULL
,
470 0x28005c1880029de4ULL
,
471 0x7000c01050c21c03ULL
,
472 0x28005c189002dde4ULL
,
473 0x204282020042e047ULL
,
474 0x7000c00820c25c03ULL
,
475 0x80000000000021e7ULL
,
476 0x190e0000fc93dc03ULL
,
477 0x1000000180821c02ULL
,
478 0x1000000040931c02ULL
,
479 0x1000000010935c02ULL
,
480 0x4800000034825c03ULL
,
481 0x22c042c042c04287ULL
,
482 0x4800000030821c03ULL
,
483 0x2800000028031de4ULL
,
484 0x4801000020a29c03ULL
,
485 0x280000002c035de4ULL
,
486 0x0800000000b2dc42ULL
,
487 0x4801000024c31c03ULL
,
488 0x9400000000a01fc5ULL
,
489 0x200002e04202c047ULL
,
490 0x28005c18a0001de4ULL
,
491 0x0800000000d35c42ULL
,
492 0x9400000100c107c5ULL
,
493 0x9400000140c01f85ULL
,
494 0x8000000000001de7ULL
497 static const uint64_t nvf0_read_hw_sm_counters_code
[] =
499 /* Same kernel as GK104 */
500 0x0880808080808080ULL
,
501 0x86400000109c0022ULL
,
502 0x86400000019c0032ULL
,
503 0x86400000021c0002ULL
,
504 0x86400000029c0006ULL
,
505 0x86400000031c000aULL
,
506 0x86400000039c000eULL
,
507 0x86400000041c0012ULL
,
508 0x08ac1080108c8080ULL
,
509 0x86400000049c0016ULL
,
510 0x86400000051c001aULL
,
511 0x86400000059c001eULL
,
512 0xdb201c007f9c201eULL
,
513 0x64c03ce0c41c002aULL
,
514 0xc00000020a1c3021ULL
,
515 0x64c03ce0c49c002eULL
,
516 0x0810a0808010b810ULL
,
517 0xc0000001041c3025ULL
,
518 0x180000000020003cULL
,
519 0xdb201c007f9c243eULL
,
520 0xc1c00000301c2021ULL
,
521 0xc1c00000081c2431ULL
,
522 0xc1c00000021c2435ULL
,
523 0xe0800000069c2026ULL
,
524 0x08b010b010b010a0ULL
,
525 0xe0800000061c2022ULL
,
526 0xe4c03c00051c0032ULL
,
527 0xe0840000041c282aULL
,
528 0xe4c03c00059c0036ULL
,
529 0xe08040007f9c2c2eULL
,
530 0xe0840000049c3032ULL
,
531 0xfe800000001c2800ULL
,
532 0x080000b81080b010ULL
,
533 0x64c03ce0c51c0002ULL
,
534 0xe08040007f9c3436ULL
,
535 0xfe80000020043010ULL
,
536 0xfc800000281c3000ULL
,
537 0x18000000001c003cULL
,
540 static const uint64_t gm107_read_hw_sm_counters_code
[] =
542 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
543 0xf0c8000002170008ULL
, /* mov $r8 $tidx */
544 0xf0c800000037000cULL
, /* mov $r12 $virtid */
545 0xf0c8000000470000ULL
, /* mov $r0 $pm0 */
546 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
547 0xf0c8000000570001ULL
, /* mov $r1 $pm1 */
548 0xf0c8000000670002ULL
, /* mov $r2 $pm2 */
549 0xf0c8000000770003ULL
, /* mov $r3 $pm3 */
550 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
551 0xf0c8000000870004ULL
, /* mov $r4 $pm4 */
552 0xf0c8000000970005ULL
, /* mov $r5 $pm5 */
553 0xf0c8000000a70006ULL
, /* mov $r6 $pm6 */
554 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
555 0xf0c8000000b70007ULL
, /* mov $r7 $pm7 */
556 0x5b6403800087ff07ULL
, /* isetp eq u32 and $p0 0x1 0x0 $r8 0x1 */
557 0x4c98079c1887000aULL
, /* mov $r10 c7[0x620] 0xf */
558 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
559 0x3800000091470c08ULL
, /* bfe u32 $r8 $r12 0x914 */
560 0x4c98079c1897000bULL
, /* mov $r11 c7[0x624] 0xf */
561 0x3800000020870c09ULL
, /* bfe u32 $r9 $r12 0x208 */
562 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
563 0xe30000000008000fULL
, /* not $p0 exit */
564 0x5b6403800097ff0fULL
, /* isetp eq u32 and $p1 0x1 0x0 $r9 0x1 */
565 0x3838000006070808ULL
, /* imul u32 u32 $r8 $r8 0x60 */
566 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
567 0x383800000107090cULL
, /* imul u32 u32 $r12 $r9 0x10 */
568 0x383800000047090dULL
, /* imul u32 u32 $r13 $r9 0x4 */
569 0x5c10000000d70809ULL
, /* iadd $r9 $r8 $r13 */
570 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
571 0x5c10000000c70808ULL
, /* iadd $r8 $r8 $r12 */
572 0x5c98078000a7000cULL
, /* mov $r12 $r10 0xf */
573 0x5c10800000870a0aULL
, /* iadd cc $r10 $r10 $r8 */
574 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
575 0x5c98078000b7000dULL
, /* mov $r13 $r11 0xf */
576 0x5c1008000ff70b0bULL
, /* iadd x $r11 $r11 0x0 */
577 0x5c10800000970c0cULL
, /* iadd cc $r12 $r12 $r9 */
578 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
579 0x5c1008000ff70d0dULL
, /* iadd x $r13 $r13 0x0 */
580 0xbfd0000000070a00ULL
, /* st e wt b128 g[$r10] $r0 0x1 */
581 0x4c98079c18a70000ULL
, /* mov $r0 c7[0x628] 0xf */
582 0x001f8000fc0007e0ULL
, /* sched (st 0x0) (st 0x0) (st 0x0) */
583 0xbfd0000004010c04ULL
, /* $p1 st e wt b128 g[$r12+0x40] $r4 0x1 */
584 0xbf90000005070c00ULL
, /* st e wt b32 g[$r12+0x50] $r0 0x1 */
585 0xe30000000007000fULL
, /* exit */
588 /* For simplicity, we will allocate as many group slots as we allocate counter
589 * slots. This means that a single counter which wants to source from 2 groups
590 * will have to be declared as using 2 counter slots. This shouldn't really be
591 * a problem because such queries don't make much sense ... (unless someone is
594 struct nvc0_hw_sm_counter_cfg
596 uint32_t func
: 16; /* mask or 4-bit logic op (depending on mode) */
597 uint32_t mode
: 4; /* LOGOP,B6,LOGOP_B6(_PULSE) */
598 uint32_t sig_dom
: 1; /* if 0, MP_PM_A (per warp-sched), if 1, MP_PM_B */
599 uint32_t sig_sel
: 8; /* signal group */
600 uint32_t src_mask
; /* mask for signal selection (only for NVC0:NVE4) */
601 uint32_t src_sel
; /* signal selection for up to 4 sources */
604 struct nvc0_hw_sm_query_cfg
607 struct nvc0_hw_sm_counter_cfg ctr
[8];
608 uint8_t num_counters
;
609 uint8_t norm
[2]; /* normalization num,denom */
612 #define _CA(f, m, g, s) { f, NVE4_COMPUTE_MP_PM_FUNC_MODE_##m, 0, g, 0, s }
613 #define _CB(f, m, g, s) { f, NVE4_COMPUTE_MP_PM_FUNC_MODE_##m, 1, g, 0, s }
614 #define _Q(n, c) [NVE4_HW_SM_QUERY_##n] = c
616 /* ==== Compute capability 3.0 (GK104:GK110) ==== */
617 static const struct nvc0_hw_sm_query_cfg
620 .type
= NVC0_HW_SM_QUERY_ACTIVE_CYCLES
,
621 .ctr
[0] = _CB(0x0001, B6
, 0x02, 0x00000000),
626 static const struct nvc0_hw_sm_query_cfg
629 .type
= NVC0_HW_SM_QUERY_ACTIVE_WARPS
,
630 .ctr
[0] = _CB(0x003f, B6
, 0x02, 0x31483104),
635 static const struct nvc0_hw_sm_query_cfg
636 sm30_atom_cas_count
=
638 .type
= NVC0_HW_SM_QUERY_ATOM_CAS_COUNT
,
639 .ctr
[0] = _CA(0x0001, B6
, 0x1c, 0x000000004),
644 static const struct nvc0_hw_sm_query_cfg
647 .type
= NVC0_HW_SM_QUERY_ATOM_COUNT
,
648 .ctr
[0] = _CA(0x0001, B6
, 0x1c, 0x00000000),
653 static const struct nvc0_hw_sm_query_cfg
656 .type
= NVC0_HW_SM_QUERY_BRANCH
,
657 .ctr
[0] = _CA(0x0001, B6
, 0x1c, 0x0000000c),
662 static const struct nvc0_hw_sm_query_cfg
663 sm30_divergent_branch
=
665 .type
= NVC0_HW_SM_QUERY_DIVERGENT_BRANCH
,
666 .ctr
[0] = _CA(0x0001, B6
, 0x1c, 0x00000010),
671 static const struct nvc0_hw_sm_query_cfg
674 .type
= NVC0_HW_SM_QUERY_GLD_REQUEST
,
675 .ctr
[0] = _CA(0x0001, B6
, 0x1b, 0x00000010),
680 static const struct nvc0_hw_sm_query_cfg
681 sm30_gld_mem_div_replay
=
683 .type
= NVC0_HW_SM_QUERY_GLD_MEM_DIV_REPLAY
,
684 .ctr
[0] = _CB(0x0001, B6
, 0x08, 0x00000010),
689 static const struct nvc0_hw_sm_query_cfg
690 sm30_gst_transactions
=
692 .type
= NVC0_HW_SM_QUERY_GST_TRANSACTIONS
,
693 .ctr
[0] = _CB(0x0001, B6
, 0x11, 0x00000004),
698 static const struct nvc0_hw_sm_query_cfg
699 sm30_gst_mem_div_replay
=
701 .type
= NVC0_HW_SM_QUERY_GST_MEM_DIV_REPLAY
,
702 .ctr
[0] = _CB(0x0001, B6
, 0x08, 0x00000014),
707 static const struct nvc0_hw_sm_query_cfg
710 .type
= NVC0_HW_SM_QUERY_GRED_COUNT
,
711 .ctr
[0] = _CA(0x0001, B6
, 0x1c, 0x00000008),
716 static const struct nvc0_hw_sm_query_cfg
719 .type
= NVC0_HW_SM_QUERY_GST_REQUEST
,
720 .ctr
[0] = _CA(0x0001, B6
, 0x1b, 0x00000014),
725 static const struct nvc0_hw_sm_query_cfg
728 .type
= NVC0_HW_SM_QUERY_INST_EXECUTED
,
729 .ctr
[0] = _CA(0x0003, B6
, 0x04, 0x00000398),
734 static const struct nvc0_hw_sm_query_cfg
737 .type
= NVC0_HW_SM_QUERY_INST_ISSUED1
,
738 .ctr
[0] = _CA(0x0001, B6
, 0x05, 0x00000004),
743 static const struct nvc0_hw_sm_query_cfg
746 .type
= NVC0_HW_SM_QUERY_INST_ISSUED2
,
747 .ctr
[0] = _CA(0x0001, B6
, 0x05, 0x00000008),
752 static const struct nvc0_hw_sm_query_cfg
755 .type
= NVC0_HW_SM_QUERY_L1_GLD_HIT
,
756 .ctr
[0] = _CB(0x0001, B6
, 0x10, 0x00000010),
761 static const struct nvc0_hw_sm_query_cfg
764 .type
= NVC0_HW_SM_QUERY_L1_GLD_MISS
,
765 .ctr
[0] = _CB(0x0001, B6
, 0x10, 0x00000014),
770 static const struct nvc0_hw_sm_query_cfg
771 sm30_l1_gld_transactions
=
773 .type
= NVC0_HW_SM_QUERY_L1_GLD_TRANSACTIONS
,
774 .ctr
[0] = _CB(0x0001, B6
, 0x0f, 0x00000000),
779 static const struct nvc0_hw_sm_query_cfg
780 sm30_l1_gst_transactions
=
782 .type
= NVC0_HW_SM_QUERY_L1_GST_TRANSACTIONS
,
783 .ctr
[0] = _CB(0x0001, B6
, 0x0f, 0x00000004),
788 static const struct nvc0_hw_sm_query_cfg
789 sm30_l1_local_ld_hit
=
791 .type
= NVC0_HW_SM_QUERY_L1_LOCAL_LD_HIT
,
792 .ctr
[0] = _CB(0x0001, B6
, 0x10, 0x00000000),
797 static const struct nvc0_hw_sm_query_cfg
798 sm30_l1_local_ld_miss
=
800 .type
= NVC0_HW_SM_QUERY_L1_LOCAL_LD_MISS
,
801 .ctr
[0] = _CB(0x0001, B6
, 0x10, 0x00000004),
806 static const struct nvc0_hw_sm_query_cfg
807 sm30_l1_local_st_hit
=
809 .type
= NVC0_HW_SM_QUERY_L1_LOCAL_ST_HIT
,
810 .ctr
[0] = _CB(0x0001, B6
, 0x10, 0x00000008),
815 static const struct nvc0_hw_sm_query_cfg
816 sm30_l1_local_st_miss
=
818 .type
= NVC0_HW_SM_QUERY_L1_LOCAL_ST_MISS
,
819 .ctr
[0] = _CB(0x0001, B6
, 0x10, 0x0000000c),
824 static const struct nvc0_hw_sm_query_cfg
825 sm30_l1_shared_ld_transactions
=
827 .type
= NVC0_HW_SM_QUERY_L1_SHARED_LD_TRANSACTIONS
,
828 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x00000008),
833 static const struct nvc0_hw_sm_query_cfg
834 sm30_l1_shared_st_transactions
=
836 .type
= NVC0_HW_SM_QUERY_L1_SHARED_ST_TRANSACTIONS
,
837 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x0000000c),
842 static const struct nvc0_hw_sm_query_cfg
845 .type
= NVC0_HW_SM_QUERY_LOCAL_LD
,
846 .ctr
[0] = _CA(0x0001, B6
, 0x1b, 0x00000008),
851 static const struct nvc0_hw_sm_query_cfg
852 sm30_local_ld_transactions
=
854 .type
= NVC0_HW_SM_QUERY_LOCAL_LD_TRANSACTIONS
,
855 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x00000000),
860 static const struct nvc0_hw_sm_query_cfg
863 .type
= NVC0_HW_SM_QUERY_LOCAL_ST
,
864 .ctr
[0] = _CA(0x0001, B6
, 0x1b, 0x0000000c),
869 static const struct nvc0_hw_sm_query_cfg
870 sm30_local_st_transactions
=
872 .type
= NVC0_HW_SM_QUERY_LOCAL_ST_TRANSACTIONS
,
873 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x00000004),
878 static const struct nvc0_hw_sm_query_cfg
879 sm30_prof_trigger_0
=
881 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_0
,
882 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x00000000),
887 static const struct nvc0_hw_sm_query_cfg
888 sm30_prof_trigger_1
=
890 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_1
,
891 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x00000004),
896 static const struct nvc0_hw_sm_query_cfg
897 sm30_prof_trigger_2
=
899 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_2
,
900 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x00000008),
905 static const struct nvc0_hw_sm_query_cfg
906 sm30_prof_trigger_3
=
908 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_3
,
909 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x0000000c),
914 static const struct nvc0_hw_sm_query_cfg
915 sm30_prof_trigger_4
=
917 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_4
,
918 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x00000010),
923 static const struct nvc0_hw_sm_query_cfg
924 sm30_prof_trigger_5
=
926 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_5
,
927 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x00000014),
932 static const struct nvc0_hw_sm_query_cfg
933 sm30_prof_trigger_6
=
935 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_6
,
936 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x00000018),
941 static const struct nvc0_hw_sm_query_cfg
942 sm30_prof_trigger_7
=
944 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_7
,
945 .ctr
[0] = _CA(0x0001, B6
, 0x01, 0x0000001c),
950 static const struct nvc0_hw_sm_query_cfg
953 .type
= NVC0_HW_SM_QUERY_SHARED_LD
,
954 .ctr
[0] = _CA(0x0001, B6
, 0x1b, 0x00000000),
959 static const struct nvc0_hw_sm_query_cfg
960 sm30_shared_ld_replay
=
962 .type
= NVC0_HW_SM_QUERY_SHARED_LD_REPLAY
,
963 .ctr
[0] = _CB(0x0001, B6
, 0x08, 0x00000008),
968 static const struct nvc0_hw_sm_query_cfg
971 .type
= NVC0_HW_SM_QUERY_SHARED_ST
,
972 .ctr
[0] = _CA(0x0001, B6
, 0x1b, 0x00000004),
977 static const struct nvc0_hw_sm_query_cfg
978 sm30_shared_st_replay
=
980 .type
= NVC0_HW_SM_QUERY_SHARED_ST_REPLAY
,
981 .ctr
[0] = _CB(0x0001, B6
, 0x08, 0x0000000c),
986 static const struct nvc0_hw_sm_query_cfg
987 sm30_sm_cta_launched
=
989 .type
= NVC0_HW_SM_QUERY_SM_CTA_LAUNCHED
,
990 .ctr
[0] = _CB(0x0001, B6
, 0x02, 0x0000001c),
995 static const struct nvc0_hw_sm_query_cfg
996 sm30_threads_launched
=
998 .type
= NVC0_HW_SM_QUERY_THREADS_LAUNCHED
,
999 .ctr
[0] = _CA(0x003f, B6
, 0x03, 0x398a4188),
1004 static const struct nvc0_hw_sm_query_cfg
1005 sm30_uncached_gld_transactions
=
1007 .type
= NVC0_HW_SM_QUERY_UNCACHED_GLD_TRANSACTIONS
,
1008 .ctr
[0] = _CB(0x0001, B6
, 0x11, 0x00000000),
1013 static const struct nvc0_hw_sm_query_cfg
1014 sm30_warps_launched
=
1016 .type
= NVC0_HW_SM_QUERY_WARPS_LAUNCHED
,
1017 .ctr
[0] = _CA(0x0001, B6
, 0x03, 0x00000004),
1023 * active_warps: bit 0 alternates btw 0 and 1 for odd nr of warps
1024 * inst_executed etc.: we only count a single warp scheduler
1026 static const struct nvc0_hw_sm_query_cfg
*sm30_hw_sm_queries
[] =
1028 &sm30_active_cycles
,
1030 &sm30_atom_cas_count
,
1033 &sm30_divergent_branch
,
1035 &sm30_gld_mem_div_replay
,
1036 &sm30_gst_transactions
,
1037 &sm30_gst_mem_div_replay
,
1040 &sm30_inst_executed
,
1045 &sm30_l1_gld_transactions
,
1046 &sm30_l1_gst_transactions
,
1047 &sm30_l1_local_ld_hit
,
1048 &sm30_l1_local_ld_miss
,
1049 &sm30_l1_local_st_hit
,
1050 &sm30_l1_local_st_miss
,
1051 &sm30_l1_shared_ld_transactions
,
1052 &sm30_l1_shared_st_transactions
,
1054 &sm30_local_ld_transactions
,
1056 &sm30_local_st_transactions
,
1057 &sm30_prof_trigger_0
,
1058 &sm30_prof_trigger_1
,
1059 &sm30_prof_trigger_2
,
1060 &sm30_prof_trigger_3
,
1061 &sm30_prof_trigger_4
,
1062 &sm30_prof_trigger_5
,
1063 &sm30_prof_trigger_6
,
1064 &sm30_prof_trigger_7
,
1066 &sm30_shared_ld_replay
,
1068 &sm30_shared_st_replay
,
1069 &sm30_sm_cta_launched
,
1070 &sm30_threads_launched
,
1071 &sm30_uncached_gld_transactions
,
1072 &sm30_warps_launched
,
1075 /* ==== Compute capability 3.5 (GK110/GK208) ==== */
1076 static const struct nvc0_hw_sm_query_cfg
1077 sm35_atom_cas_count
=
1079 .type
= NVC0_HW_SM_QUERY_ATOM_CAS_COUNT
,
1080 .ctr
[0] = _CA(0x0001, B6
, 0x1a, 0x00000014),
1085 static const struct nvc0_hw_sm_query_cfg
1088 .type
= NVC0_HW_SM_QUERY_ATOM_COUNT
,
1089 .ctr
[0] = _CA(0x0001, B6
, 0x1a, 0x00000010),
1094 static const struct nvc0_hw_sm_query_cfg
1097 .type
= NVC0_HW_SM_QUERY_GRED_COUNT
,
1098 .ctr
[0] = _CA(0x0001, B6
, 0x1a, 0x00000018),
1103 static const struct nvc0_hw_sm_query_cfg
1104 sm35_not_pred_off_inst_executed
=
1106 .type
= NVC0_HW_SM_QUERY_NOT_PRED_OFF_INST_EXECUTED
,
1107 .ctr
[0] = _CA(0x003f, B6
, 0x14, 0x29062080),
1112 static const struct nvc0_hw_sm_query_cfg
1113 sm35_shared_ld_replay
=
1115 .type
= NVC0_HW_SM_QUERY_SHARED_LD_REPLAY
,
1116 .ctr
[0] = _CB(0xaaaa, LOGOP
, 0x13, 0x00000018),
1117 .ctr
[1] = _CB(0x8888, LOGOP
, 0x08, 0x00000151),
1122 static const struct nvc0_hw_sm_query_cfg
1123 sm35_shared_st_replay
=
1125 .type
= NVC0_HW_SM_QUERY_SHARED_ST_REPLAY
,
1126 .ctr
[0] = _CB(0xaaaa, LOGOP
, 0x13, 0x00000018),
1127 .ctr
[1] = _CB(0x8888, LOGOP
, 0x08, 0x000001d1),
1132 static const struct nvc0_hw_sm_query_cfg
1133 sm35_th_inst_executed
=
1135 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED
,
1136 .ctr
[0] = _CA(0x003f, B6
, 0x11, 0x29062080),
1141 static const struct nvc0_hw_sm_query_cfg
*sm35_hw_sm_queries
[] =
1143 &sm30_active_cycles
,
1145 &sm35_atom_cas_count
,
1148 &sm30_gld_mem_div_replay
,
1149 &sm30_gst_transactions
,
1150 &sm30_gst_mem_div_replay
,
1153 &sm30_inst_executed
,
1158 &sm30_l1_gld_transactions
,
1159 &sm30_l1_gst_transactions
,
1160 &sm30_l1_local_ld_hit
,
1161 &sm30_l1_local_ld_miss
,
1162 &sm30_l1_local_st_hit
,
1163 &sm30_l1_local_st_miss
,
1164 &sm30_l1_shared_ld_transactions
,
1165 &sm30_l1_shared_st_transactions
,
1167 &sm30_local_ld_transactions
,
1169 &sm30_local_st_transactions
,
1170 &sm35_not_pred_off_inst_executed
,
1171 &sm30_prof_trigger_0
,
1172 &sm30_prof_trigger_1
,
1173 &sm30_prof_trigger_2
,
1174 &sm30_prof_trigger_3
,
1175 &sm30_prof_trigger_4
,
1176 &sm30_prof_trigger_5
,
1177 &sm30_prof_trigger_6
,
1178 &sm30_prof_trigger_7
,
1180 &sm35_shared_ld_replay
,
1182 &sm35_shared_st_replay
,
1183 &sm30_sm_cta_launched
,
1184 &sm35_th_inst_executed
,
1185 &sm30_threads_launched
,
1186 &sm30_uncached_gld_transactions
,
1187 &sm30_warps_launched
,
1190 /* ==== Compute capability 5.0 (GM107/GM108) ==== */
1191 static const struct nvc0_hw_sm_query_cfg
1194 .type
= NVC0_HW_SM_QUERY_ACTIVE_CTAS
,
1195 .ctr
[0] = _CB(0x003f, B6
, 0x01, 0x29062080),
1200 static const struct nvc0_hw_sm_query_cfg
1201 sm50_active_cycles
=
1203 .type
= NVC0_HW_SM_QUERY_ACTIVE_CYCLES
,
1204 .ctr
[0] = _CB(0x0001, B6
, 0x00, 0x00000004),
1209 static const struct nvc0_hw_sm_query_cfg
1212 .type
= NVC0_HW_SM_QUERY_ACTIVE_WARPS
,
1213 .ctr
[0] = _CB(0x003f, B6
, 0x00, 0x398a4188),
1218 static const struct nvc0_hw_sm_query_cfg
1221 .type
= NVC0_HW_SM_QUERY_ATOM_COUNT
,
1222 .ctr
[0] = _CA(0x0001, B6
, 0x14, 0x00000004),
1227 static const struct nvc0_hw_sm_query_cfg
1230 .type
= NVC0_HW_SM_QUERY_BRANCH
,
1231 .ctr
[0] = _CA(0x0001, B6
, 0x1a, 0x00000010),
1236 static const struct nvc0_hw_sm_query_cfg
1237 sm50_divergent_branch
=
1239 .type
= NVC0_HW_SM_QUERY_DIVERGENT_BRANCH
,
1240 .ctr
[0] = _CA(0x0001, B6
, 0x1a, 0x00000004),
1245 static const struct nvc0_hw_sm_query_cfg
1246 sm50_global_atom_cas
=
1248 .type
= NVC0_HW_SM_QUERY_GLOBAL_ATOM_CAS
,
1249 .ctr
[0] = _CA(0x0001, B6
, 0x14, 0x00000000),
1254 static const struct nvc0_hw_sm_query_cfg
1257 .type
= NVC0_HW_SM_QUERY_GLOBAL_LD
,
1258 .ctr
[0] = _CA(0x0001, B6
, 0x14, 0x0000000c),
1263 static const struct nvc0_hw_sm_query_cfg
1266 .type
= NVC0_HW_SM_QUERY_GLOBAL_ST
,
1267 .ctr
[0] = _CA(0x0001, B6
, 0x14, 0x00000010),
1272 static const struct nvc0_hw_sm_query_cfg
1275 .type
= NVC0_HW_SM_QUERY_GRED_COUNT
,
1276 .ctr
[0] = _CA(0x0001, B6
, 0x14, 0x00000008),
1281 static const struct nvc0_hw_sm_query_cfg
1282 sm50_inst_executed
=
1284 .type
= NVC0_HW_SM_QUERY_INST_EXECUTED
,
1285 .ctr
[0] = _CA(0x0003, B6
, 0x02, 0x00000398),
1290 static const struct nvc0_hw_sm_query_cfg
1293 .type
= NVC0_HW_SM_QUERY_INST_ISSUED0
,
1294 .ctr
[0] = _CA(0x0001, B6
, 0x02, 0x0000000c),
1299 static const struct nvc0_hw_sm_query_cfg
1302 .type
= NVC0_HW_SM_QUERY_INST_ISSUED1
,
1303 .ctr
[0] = _CA(0x0001, B6
, 0x02, 0x00000010),
1308 static const struct nvc0_hw_sm_query_cfg
1311 .type
= NVC0_HW_SM_QUERY_INST_ISSUED2
,
1312 .ctr
[0] = _CA(0x0001, B6
, 0x02, 0x00000014),
1317 static const struct nvc0_hw_sm_query_cfg
1320 .type
= NVC0_HW_SM_QUERY_LOCAL_LD
,
1321 .ctr
[0] = _CA(0x0001, B6
, 0x13, 0x00000004),
1326 static const struct nvc0_hw_sm_query_cfg
1329 .type
= NVC0_HW_SM_QUERY_LOCAL_ST
,
1330 .ctr
[0] = _CA(0x0001, B6
, 0x13, 0x00000000),
1335 static const struct nvc0_hw_sm_query_cfg
1336 sm50_not_pred_off_inst_executed
=
1338 .type
= NVC0_HW_SM_QUERY_NOT_PRED_OFF_INST_EXECUTED
,
1339 .ctr
[0] = _CA(0x003f, B6
, 0x05, 0x29062080),
1344 static const struct nvc0_hw_sm_query_cfg
1345 sm50_prof_trigger_0
=
1347 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_0
,
1348 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x00000000),
1353 static const struct nvc0_hw_sm_query_cfg
1354 sm50_prof_trigger_1
=
1356 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_1
,
1357 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x00000004),
1362 static const struct nvc0_hw_sm_query_cfg
1363 sm50_prof_trigger_2
=
1365 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_2
,
1366 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x00000008),
1371 static const struct nvc0_hw_sm_query_cfg
1372 sm50_prof_trigger_3
=
1374 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_3
,
1375 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x0000000c),
1380 static const struct nvc0_hw_sm_query_cfg
1381 sm50_prof_trigger_4
=
1383 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_4
,
1384 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x00000010),
1389 static const struct nvc0_hw_sm_query_cfg
1390 sm50_prof_trigger_5
=
1392 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_5
,
1393 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x00000014),
1398 static const struct nvc0_hw_sm_query_cfg
1399 sm50_prof_trigger_6
=
1401 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_6
,
1402 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x00000018),
1407 static const struct nvc0_hw_sm_query_cfg
1408 sm50_prof_trigger_7
=
1410 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_7
,
1411 .ctr
[0] = _CA(0x0001, B6
, 0x00, 0x0000001c),
1416 static const struct nvc0_hw_sm_query_cfg
1419 .type
= NVC0_HW_SM_QUERY_SHARED_ATOM
,
1420 .ctr
[0] = _CA(0x0001, B6
, 0x13, 0x00000014),
1425 static const struct nvc0_hw_sm_query_cfg
1426 sm50_shared_atom_cas
=
1428 .type
= NVC0_HW_SM_QUERY_SHARED_ATOM_CAS
,
1429 .ctr
[0] = _CA(0x0001, B6
, 0x13, 0x00000010),
1434 static const struct nvc0_hw_sm_query_cfg
1437 .type
= NVC0_HW_SM_QUERY_SHARED_LD
,
1438 .ctr
[0] = _CA(0x0001, B6
, 0x13, 0x00000008),
1443 static const struct nvc0_hw_sm_query_cfg
1444 sm50_shared_ld_bank_conflict
=
1446 .type
= NVC0_HW_SM_QUERY_SHARED_LD_BANK_CONFLICT
,
1447 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x00000000),
1452 static const struct nvc0_hw_sm_query_cfg
1453 sm50_shared_ld_transactions
=
1455 .type
= NVC0_HW_SM_QUERY_SHARED_LD_TRANSACTIONS
,
1456 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x00000008),
1461 static const struct nvc0_hw_sm_query_cfg
1464 .type
= NVC0_HW_SM_QUERY_SHARED_ST
,
1465 .ctr
[0] = _CA(0x0001, B6
, 0x13, 0x0000000c),
1470 static const struct nvc0_hw_sm_query_cfg
1471 sm50_shared_st_bank_conflict
=
1473 .type
= NVC0_HW_SM_QUERY_SHARED_ST_BANK_CONFLICT
,
1474 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x00000004),
1479 static const struct nvc0_hw_sm_query_cfg
1480 sm50_shared_st_transactions
=
1482 .type
= NVC0_HW_SM_QUERY_SHARED_ST_TRANSACTIONS
,
1483 .ctr
[0] = _CB(0x0001, B6
, 0x0e, 0x0000000c),
1488 static const struct nvc0_hw_sm_query_cfg
1489 sm50_sm_cta_launched
=
1491 .type
= NVC0_HW_SM_QUERY_SM_CTA_LAUNCHED
,
1492 .ctr
[0] = _CB(0x0001, B6
, 0x01, 0x00000018),
1497 static const struct nvc0_hw_sm_query_cfg
1498 sm50_th_inst_executed
=
1500 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED
,
1501 .ctr
[0] = _CA(0x003f, B6
, 0x04, 0x29062080),
1506 static const struct nvc0_hw_sm_query_cfg
1507 sm50_warps_launched
=
1509 .type
= NVC0_HW_SM_QUERY_WARPS_LAUNCHED
,
1510 .ctr
[0] = _CA(0x0001, B6
, 0x02, 0x00000008),
1515 static const struct nvc0_hw_sm_query_cfg
*sm50_hw_sm_queries
[] =
1518 &sm50_active_cycles
,
1522 &sm50_divergent_branch
,
1523 &sm50_global_atom_cas
,
1527 &sm50_inst_executed
,
1533 &sm50_not_pred_off_inst_executed
,
1534 &sm50_prof_trigger_0
,
1535 &sm50_prof_trigger_1
,
1536 &sm50_prof_trigger_2
,
1537 &sm50_prof_trigger_3
,
1538 &sm50_prof_trigger_4
,
1539 &sm50_prof_trigger_5
,
1540 &sm50_prof_trigger_6
,
1541 &sm50_prof_trigger_7
,
1543 &sm50_shared_atom_cas
,
1545 &sm50_shared_ld_bank_conflict
,
1546 &sm50_shared_ld_transactions
,
1548 &sm50_shared_st_bank_conflict
,
1549 &sm50_shared_st_transactions
,
1550 &sm50_sm_cta_launched
,
1551 &sm50_th_inst_executed
,
1552 &sm50_warps_launched
,
1555 /* ==== Compute capability 5.2 (GM200/GM204/GM206) ==== */
1556 static const struct nvc0_hw_sm_query_cfg
1559 .type
= NVC0_HW_SM_QUERY_ATOM_COUNT
,
1560 .ctr
[0] = _CA(0x0001, B6
, 0x0a, 0x0000001c),
1565 static const struct nvc0_hw_sm_query_cfg
1566 sm52_global_atom_cas
=
1568 .type
= NVC0_HW_SM_QUERY_GLOBAL_ATOM_CAS
,
1569 .ctr
[0] = _CA(0x0001, B6
, 0x0a, 0x00000018),
1574 static const struct nvc0_hw_sm_query_cfg
1577 .type
= NVC0_HW_SM_QUERY_GLOBAL_LD
,
1578 .ctr
[0] = _CA(0x0001, B6
, 0x0b, 0x00000018),
1583 static const struct nvc0_hw_sm_query_cfg
1586 .type
= NVC0_HW_SM_QUERY_GLOBAL_ST
,
1587 .ctr
[0] = _CA(0x0001, B6
, 0x0b, 0x0000001c),
1592 static const struct nvc0_hw_sm_query_cfg
1595 .type
= NVC0_HW_SM_QUERY_GRED_COUNT
,
1596 .ctr
[0] = _CA(0x0001, B6
, 0x0f, 0x00000018),
1601 static const struct nvc0_hw_sm_query_cfg
1602 sm52_inst_executed
=
1604 .type
= NVC0_HW_SM_QUERY_INST_EXECUTED
,
1605 .ctr
[0] = _CA(0x0003, B6
, 0x03, 0x0000020c),
1610 static const struct nvc0_hw_sm_query_cfg
1613 .type
= NVC0_HW_SM_QUERY_INST_ISSUED0
,
1614 .ctr
[0] = _CA(0x0001, B6
, 0x03, 0x00000000),
1619 static const struct nvc0_hw_sm_query_cfg
1622 .type
= NVC0_HW_SM_QUERY_INST_ISSUED1
,
1623 .ctr
[0] = _CA(0x0001, B6
, 0x03, 0x00000004),
1628 static const struct nvc0_hw_sm_query_cfg
1631 .type
= NVC0_HW_SM_QUERY_INST_ISSUED2
,
1632 .ctr
[0] = _CA(0x0001, B6
, 0x03, 0x00000008),
1637 static const struct nvc0_hw_sm_query_cfg
1640 .type
= NVC0_HW_SM_QUERY_LOCAL_LD
,
1641 .ctr
[0] = _CA(0x0001, B6
, 0x06, 0x0000001c),
1646 static const struct nvc0_hw_sm_query_cfg
1649 .type
= NVC0_HW_SM_QUERY_LOCAL_ST
,
1650 .ctr
[0] = _CA(0x0001, B6
, 0x06, 0x00000018),
1655 static const struct nvc0_hw_sm_query_cfg
1658 .type
= NVC0_HW_SM_QUERY_SHARED_ATOM
,
1659 .ctr
[0] = _CA(0x0001, B6
, 0x08, 0x0000001c),
1664 static const struct nvc0_hw_sm_query_cfg
1665 sm52_shared_atom_cas
=
1667 .type
= NVC0_HW_SM_QUERY_SHARED_ATOM_CAS
,
1668 .ctr
[0] = _CA(0x0001, B6
, 0x08, 0x00000018),
1673 static const struct nvc0_hw_sm_query_cfg
1676 .type
= NVC0_HW_SM_QUERY_SHARED_LD
,
1677 .ctr
[0] = _CA(0x0001, B6
, 0x07, 0x00000018),
1682 static const struct nvc0_hw_sm_query_cfg
1685 .type
= NVC0_HW_SM_QUERY_SHARED_ST
,
1686 .ctr
[0] = _CA(0x0001, B6
, 0x07, 0x0000001c),
1691 static const struct nvc0_hw_sm_query_cfg
1692 sm52_warps_launched
=
1694 .type
= NVC0_HW_SM_QUERY_WARPS_LAUNCHED
,
1695 .ctr
[0] = _CA(0x0001, B6
, 0x02, 0x0000001c),
1700 static const struct nvc0_hw_sm_query_cfg
*sm52_hw_sm_queries
[] =
1703 &sm50_active_cycles
,
1707 &sm50_divergent_branch
,
1708 &sm52_global_atom_cas
,
1712 &sm52_inst_executed
,
1718 &sm50_not_pred_off_inst_executed
,
1719 &sm50_prof_trigger_0
,
1720 &sm50_prof_trigger_1
,
1721 &sm50_prof_trigger_2
,
1722 &sm50_prof_trigger_3
,
1723 &sm50_prof_trigger_4
,
1724 &sm50_prof_trigger_5
,
1725 &sm50_prof_trigger_6
,
1726 &sm50_prof_trigger_7
,
1728 &sm52_shared_atom_cas
,
1730 &sm50_shared_ld_bank_conflict
,
1731 &sm50_shared_ld_transactions
,
1733 &sm50_shared_st_bank_conflict
,
1734 &sm50_shared_st_transactions
,
1735 &sm50_sm_cta_launched
,
1736 &sm50_th_inst_executed
,
1737 &sm52_warps_launched
,
1744 /* === PERFORMANCE MONITORING COUNTERS for NVC0:NVE4 === */
1746 * - MP counters on GF100/GF110 (compute capability 2.0) are buggy
1747 * because there is a context-switch problem that we need to fix.
1748 * Results might be wrong sometimes, be careful!
1750 static const uint64_t nvc0_read_hw_sm_counters_code
[] =
1752 /* mov b32 $r8 $tidx
1753 * mov b32 $r9 $physid
1762 * set $p0 0x1 eq u32 $r8 0x0
1763 * mov b32 $r10 c15[0x620]
1764 * mov b32 $r11 c15[0x624]
1765 * ext u32 $r8 $r9 0x414
1767 * mul $r8 u32 $r8 u32 48
1768 * add b32 $r10 $c $r10 $r8
1769 * add b32 $r11 $r11 0x0 $c
1770 * mov b32 $r8 c15[0x628]
1771 * st b128 wt g[$r10d+0x00] $r0q
1772 * st b128 wt g[$r10d+0x10] $r4q
1773 * st b32 wt g[$r10d+0x20] $r8
1775 0x2c00000084021c04ULL
,
1776 0x2c0000000c025c04ULL
,
1777 0x2c00000010001c04ULL
,
1778 0x2c00000014005c04ULL
,
1779 0x2c00000018009c04ULL
,
1780 0x2c0000001c00dc04ULL
,
1781 0x2c00000020011c04ULL
,
1782 0x2c00000024015c04ULL
,
1783 0x2c00000028019c04ULL
,
1784 0x2c0000002c01dc04ULL
,
1785 0x190e0000fc81dc03ULL
,
1786 0x28007c1880029de4ULL
,
1787 0x28007c189002dde4ULL
,
1788 0x7000c01050921c03ULL
,
1789 0x80000000000021e7ULL
,
1790 0x10000000c0821c02ULL
,
1791 0x4801000020a29c03ULL
,
1792 0x0800000000b2dc42ULL
,
1793 0x28007c18a0021de4ULL
,
1794 0x9400000000a01fc5ULL
,
1795 0x9400000040a11fc5ULL
,
1796 0x9400000080a21f85ULL
,
1797 0x8000000000001de7ULL
1800 #define _C(f, o, g, m, s) { f, NVC0_COMPUTE_MP_PM_OP_MODE_##o, 0, g, m, s }
1802 /* ==== Compute capability 2.0 (GF100/GF110) ==== */
1803 static const struct nvc0_hw_sm_query_cfg
1804 sm20_active_cycles
=
1806 .type
= NVC0_HW_SM_QUERY_ACTIVE_CYCLES
,
1807 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x11, 0x000000ff, 0x00000000),
1812 static const struct nvc0_hw_sm_query_cfg
1815 .type
= NVC0_HW_SM_QUERY_ACTIVE_WARPS
,
1816 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x24, 0x000000ff, 0x00000010),
1817 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x24, 0x000000ff, 0x00000020),
1818 .ctr
[2] = _C(0xaaaa, LOGOP
, 0x24, 0x000000ff, 0x00000030),
1819 .ctr
[3] = _C(0xaaaa, LOGOP
, 0x24, 0x000000ff, 0x00000040),
1820 .ctr
[4] = _C(0xaaaa, LOGOP
, 0x24, 0x000000ff, 0x00000050),
1821 .ctr
[5] = _C(0xaaaa, LOGOP
, 0x24, 0x000000ff, 0x00000060),
1826 static const struct nvc0_hw_sm_query_cfg
1829 .type
= NVC0_HW_SM_QUERY_ATOM_COUNT
,
1830 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x63, 0x000000ff, 0x00000030),
1835 static const struct nvc0_hw_sm_query_cfg
1838 .type
= NVC0_HW_SM_QUERY_BRANCH
,
1839 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x1a, 0x000000ff, 0x00000000),
1840 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x1a, 0x000000ff, 0x00000010),
1845 static const struct nvc0_hw_sm_query_cfg
1846 sm20_divergent_branch
=
1848 .type
= NVC0_HW_SM_QUERY_DIVERGENT_BRANCH
,
1849 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x19, 0x000000ff, 0x00000020),
1850 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x19, 0x000000ff, 0x00000030),
1855 static const struct nvc0_hw_sm_query_cfg
1858 .type
= NVC0_HW_SM_QUERY_GLD_REQUEST
,
1859 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x64, 0x000000ff, 0x00000030),
1864 static const struct nvc0_hw_sm_query_cfg
1867 .type
= NVC0_HW_SM_QUERY_GRED_COUNT
,
1868 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x63, 0x000000ff, 0x00000040),
1873 static const struct nvc0_hw_sm_query_cfg
1876 .type
= NVC0_HW_SM_QUERY_GST_REQUEST
,
1877 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x64, 0x000000ff, 0x00000060),
1882 static const struct nvc0_hw_sm_query_cfg
1883 sm20_inst_executed
=
1885 .type
= NVC0_HW_SM_QUERY_INST_EXECUTED
,
1886 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x2d, 0x0000ffff, 0x00001000),
1887 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x2d, 0x0000ffff, 0x00001010),
1892 static const struct nvc0_hw_sm_query_cfg
1895 .type
= NVC0_HW_SM_QUERY_INST_ISSUED
,
1896 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x27, 0x0000ffff, 0x00007060),
1897 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x27, 0x0000ffff, 0x00007070),
1902 static const struct nvc0_hw_sm_query_cfg
1905 .type
= NVC0_HW_SM_QUERY_LOCAL_LD
,
1906 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x64, 0x000000ff, 0x00000020),
1911 static const struct nvc0_hw_sm_query_cfg
1914 .type
= NVC0_HW_SM_QUERY_LOCAL_ST
,
1915 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x64, 0x000000ff, 0x00000050),
1920 static const struct nvc0_hw_sm_query_cfg
1921 sm20_prof_trigger_0
=
1923 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_0
,
1924 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000000),
1929 static const struct nvc0_hw_sm_query_cfg
1930 sm20_prof_trigger_1
=
1932 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_1
,
1933 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000010),
1938 static const struct nvc0_hw_sm_query_cfg
1939 sm20_prof_trigger_2
=
1941 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_2
,
1942 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000020),
1947 static const struct nvc0_hw_sm_query_cfg
1948 sm20_prof_trigger_3
=
1950 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_3
,
1951 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000030),
1956 static const struct nvc0_hw_sm_query_cfg
1957 sm20_prof_trigger_4
=
1959 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_4
,
1960 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000040),
1965 static const struct nvc0_hw_sm_query_cfg
1966 sm20_prof_trigger_5
=
1968 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_5
,
1969 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000050),
1974 static const struct nvc0_hw_sm_query_cfg
1975 sm20_prof_trigger_6
=
1977 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_6
,
1978 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000060),
1983 static const struct nvc0_hw_sm_query_cfg
1984 sm20_prof_trigger_7
=
1986 .type
= NVC0_HW_SM_QUERY_PROF_TRIGGER_7
,
1987 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x01, 0x000000ff, 0x00000070),
1992 static const struct nvc0_hw_sm_query_cfg
1995 .type
= NVC0_HW_SM_QUERY_SHARED_LD
,
1996 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x64, 0x000000ff, 0x00000010),
2001 static const struct nvc0_hw_sm_query_cfg
2004 .type
= NVC0_HW_SM_QUERY_SHARED_ST
,
2005 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x64, 0x000000ff, 0x00000040),
2010 static const struct nvc0_hw_sm_query_cfg
2011 sm20_threads_launched
=
2013 .type
= NVC0_HW_SM_QUERY_THREADS_LAUNCHED
,
2014 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000010),
2015 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000020),
2016 .ctr
[2] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000030),
2017 .ctr
[3] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000040),
2018 .ctr
[4] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000050),
2019 .ctr
[5] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000060),
2024 static const struct nvc0_hw_sm_query_cfg
2025 sm20_th_inst_executed_0
=
2027 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED_0
,
2028 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x2f, 0x000000ff, 0x00000000),
2029 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x2f, 0x000000ff, 0x00000010),
2030 .ctr
[2] = _C(0xaaaa, LOGOP
, 0x2f, 0x000000ff, 0x00000020),
2031 .ctr
[3] = _C(0xaaaa, LOGOP
, 0x2f, 0x000000ff, 0x00000030),
2032 .ctr
[4] = _C(0xaaaa, LOGOP
, 0x2f, 0x000000ff, 0x00000040),
2033 .ctr
[5] = _C(0xaaaa, LOGOP
, 0x2f, 0x000000ff, 0x00000050),
2038 static const struct nvc0_hw_sm_query_cfg
2039 sm20_th_inst_executed_1
=
2041 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED_1
,
2042 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x30, 0x000000ff, 0x00000000),
2043 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x30, 0x000000ff, 0x00000010),
2044 .ctr
[2] = _C(0xaaaa, LOGOP
, 0x30, 0x000000ff, 0x00000020),
2045 .ctr
[3] = _C(0xaaaa, LOGOP
, 0x30, 0x000000ff, 0x00000030),
2046 .ctr
[4] = _C(0xaaaa, LOGOP
, 0x30, 0x000000ff, 0x00000040),
2047 .ctr
[5] = _C(0xaaaa, LOGOP
, 0x30, 0x000000ff, 0x00000050),
2052 static const struct nvc0_hw_sm_query_cfg
2053 sm20_warps_launched
=
2055 .type
= NVC0_HW_SM_QUERY_WARPS_LAUNCHED
,
2056 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x26, 0x000000ff, 0x00000000),
2061 static const struct nvc0_hw_sm_query_cfg
*sm20_hw_sm_queries
[] =
2063 &sm20_active_cycles
,
2067 &sm20_divergent_branch
,
2071 &sm20_inst_executed
,
2075 &sm20_prof_trigger_0
,
2076 &sm20_prof_trigger_1
,
2077 &sm20_prof_trigger_2
,
2078 &sm20_prof_trigger_3
,
2079 &sm20_prof_trigger_4
,
2080 &sm20_prof_trigger_5
,
2081 &sm20_prof_trigger_6
,
2082 &sm20_prof_trigger_7
,
2085 &sm20_threads_launched
,
2086 &sm20_th_inst_executed_0
,
2087 &sm20_th_inst_executed_1
,
2088 &sm20_warps_launched
,
2091 /* ==== Compute capability 2.1 (GF108+ except GF110) ==== */
2092 static const struct nvc0_hw_sm_query_cfg
2093 sm21_inst_executed
=
2095 .type
= NVC0_HW_SM_QUERY_INST_EXECUTED
,
2096 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x2d, 0x000000ff, 0x00000000),
2097 .ctr
[1] = _C(0xaaaa, LOGOP
, 0x2d, 0x000000ff, 0x00000010),
2098 .ctr
[2] = _C(0xaaaa, LOGOP
, 0x2d, 0x000000ff, 0x00000020),
2103 static const struct nvc0_hw_sm_query_cfg
2104 sm21_inst_issued1_0
=
2106 .type
= NVC0_HW_SM_QUERY_INST_ISSUED1_0
,
2107 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x7e, 0x000000ff, 0x00000010),
2112 static const struct nvc0_hw_sm_query_cfg
2113 sm21_inst_issued1_1
=
2115 .type
= NVC0_HW_SM_QUERY_INST_ISSUED1_1
,
2116 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x7e, 0x000000ff, 0x00000040),
2121 static const struct nvc0_hw_sm_query_cfg
2122 sm21_inst_issued2_0
=
2124 .type
= NVC0_HW_SM_QUERY_INST_ISSUED2_0
,
2125 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x7e, 0x000000ff, 0x00000020),
2130 static const struct nvc0_hw_sm_query_cfg
2131 sm21_inst_issued2_1
=
2133 .type
= NVC0_HW_SM_QUERY_INST_ISSUED2_1
,
2134 .ctr
[0] = _C(0xaaaa, LOGOP
, 0x7e, 0x000000ff, 0x00000050),
2139 static const struct nvc0_hw_sm_query_cfg
2140 sm21_th_inst_executed_0
=
2142 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED_0
,
2143 .ctr
[0] = _C(0xaaaa, LOGOP
, 0xa3, 0x000000ff, 0x00000000),
2144 .ctr
[1] = _C(0xaaaa, LOGOP
, 0xa3, 0x000000ff, 0x00000010),
2145 .ctr
[2] = _C(0xaaaa, LOGOP
, 0xa3, 0x000000ff, 0x00000020),
2146 .ctr
[3] = _C(0xaaaa, LOGOP
, 0xa3, 0x000000ff, 0x00000030),
2147 .ctr
[4] = _C(0xaaaa, LOGOP
, 0xa3, 0x000000ff, 0x00000040),
2148 .ctr
[5] = _C(0xaaaa, LOGOP
, 0xa3, 0x000000ff, 0x00000050),
2153 static const struct nvc0_hw_sm_query_cfg
2154 sm21_th_inst_executed_1
=
2156 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED_1
,
2157 .ctr
[0] = _C(0xaaaa, LOGOP
, 0xa5, 0x000000ff, 0x00000000),
2158 .ctr
[1] = _C(0xaaaa, LOGOP
, 0xa5, 0x000000ff, 0x00000010),
2159 .ctr
[2] = _C(0xaaaa, LOGOP
, 0xa5, 0x000000ff, 0x00000020),
2160 .ctr
[3] = _C(0xaaaa, LOGOP
, 0xa5, 0x000000ff, 0x00000030),
2161 .ctr
[4] = _C(0xaaaa, LOGOP
, 0xa5, 0x000000ff, 0x00000040),
2162 .ctr
[5] = _C(0xaaaa, LOGOP
, 0xa5, 0x000000ff, 0x00000050),
2167 static const struct nvc0_hw_sm_query_cfg
2168 sm21_th_inst_executed_2
=
2170 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED_2
,
2171 .ctr
[0] = _C(0xaaaa, LOGOP
, 0xa4, 0x000000ff, 0x00000000),
2172 .ctr
[1] = _C(0xaaaa, LOGOP
, 0xa4, 0x000000ff, 0x00000010),
2173 .ctr
[2] = _C(0xaaaa, LOGOP
, 0xa4, 0x000000ff, 0x00000020),
2174 .ctr
[3] = _C(0xaaaa, LOGOP
, 0xa4, 0x000000ff, 0x00000030),
2175 .ctr
[4] = _C(0xaaaa, LOGOP
, 0xa4, 0x000000ff, 0x00000040),
2176 .ctr
[5] = _C(0xaaaa, LOGOP
, 0xa4, 0x000000ff, 0x00000050),
2181 static const struct nvc0_hw_sm_query_cfg
2182 sm21_th_inst_executed_3
=
2184 .type
= NVC0_HW_SM_QUERY_TH_INST_EXECUTED_3
,
2185 .ctr
[0] = _C(0xaaaa, LOGOP
, 0xa6, 0x000000ff, 0x00000000),
2186 .ctr
[1] = _C(0xaaaa, LOGOP
, 0xa6, 0x000000ff, 0x00000010),
2187 .ctr
[2] = _C(0xaaaa, LOGOP
, 0xa6, 0x000000ff, 0x00000020),
2188 .ctr
[3] = _C(0xaaaa, LOGOP
, 0xa6, 0x000000ff, 0x00000030),
2189 .ctr
[4] = _C(0xaaaa, LOGOP
, 0xa6, 0x000000ff, 0x00000040),
2190 .ctr
[5] = _C(0xaaaa, LOGOP
, 0xa6, 0x000000ff, 0x00000050),
2195 static const struct nvc0_hw_sm_query_cfg
*sm21_hw_sm_queries
[] =
2197 &sm20_active_cycles
,
2201 &sm20_divergent_branch
,
2205 &sm21_inst_executed
,
2206 &sm21_inst_issued1_0
,
2207 &sm21_inst_issued1_1
,
2208 &sm21_inst_issued2_0
,
2209 &sm21_inst_issued2_1
,
2212 &sm20_prof_trigger_0
,
2213 &sm20_prof_trigger_1
,
2214 &sm20_prof_trigger_2
,
2215 &sm20_prof_trigger_3
,
2216 &sm20_prof_trigger_4
,
2217 &sm20_prof_trigger_5
,
2218 &sm20_prof_trigger_6
,
2219 &sm20_prof_trigger_7
,
2222 &sm20_threads_launched
,
2223 &sm21_th_inst_executed_0
,
2224 &sm21_th_inst_executed_1
,
2225 &sm21_th_inst_executed_2
,
2226 &sm21_th_inst_executed_3
,
2227 &sm20_warps_launched
,
2232 static inline const struct nvc0_hw_sm_query_cfg
**
2233 nvc0_hw_sm_get_queries(struct nvc0_screen
*screen
)
2235 struct nouveau_device
*dev
= screen
->base
.device
;
2237 switch (screen
->base
.class_3d
) {
2238 case GM200_3D_CLASS
:
2239 return sm52_hw_sm_queries
;
2240 case GM107_3D_CLASS
:
2241 return sm50_hw_sm_queries
;
2243 return sm35_hw_sm_queries
;
2245 return sm30_hw_sm_queries
;
2247 if (dev
->chipset
== 0xc0 || dev
->chipset
== 0xc8)
2248 return sm20_hw_sm_queries
;
2249 return sm21_hw_sm_queries
;
2256 nvc0_hw_sm_get_num_queries(struct nvc0_screen
*screen
)
2258 struct nouveau_device
*dev
= screen
->base
.device
;
2260 switch (screen
->base
.class_3d
) {
2261 case GM200_3D_CLASS
:
2262 return ARRAY_SIZE(sm52_hw_sm_queries
);
2263 case GM107_3D_CLASS
:
2264 return ARRAY_SIZE(sm50_hw_sm_queries
);
2266 return ARRAY_SIZE(sm35_hw_sm_queries
);
2268 return ARRAY_SIZE(sm30_hw_sm_queries
);
2270 if (dev
->chipset
== 0xc0 || dev
->chipset
== 0xc8)
2271 return ARRAY_SIZE(sm20_hw_sm_queries
);
2272 return ARRAY_SIZE(sm21_hw_sm_queries
);
2277 static const struct nvc0_hw_sm_query_cfg
*
2278 nvc0_hw_sm_query_get_cfg(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
)
2280 const struct nvc0_hw_sm_query_cfg
**queries
;
2281 struct nvc0_screen
*screen
= nvc0
->screen
;
2282 struct nvc0_query
*q
= &hq
->base
;
2283 unsigned num_queries
;
2286 num_queries
= nvc0_hw_sm_get_num_queries(screen
);
2287 queries
= nvc0_hw_sm_get_queries(screen
);
2289 for (i
= 0; i
< num_queries
; i
++) {
2290 if (NVC0_HW_SM_QUERY(queries
[i
]->type
) == q
->type
)
2298 nvc0_hw_sm_destroy_query(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
)
2300 struct nvc0_query
*q
= &hq
->base
;
2301 nvc0_hw_query_allocate(nvc0
, q
, 0);
2302 nouveau_fence_ref(NULL
, &hq
->fence
);
2307 nve4_hw_sm_begin_query(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
)
2309 struct nvc0_screen
*screen
= nvc0
->screen
;
2310 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
2311 struct nvc0_hw_sm_query
*hsq
= nvc0_hw_sm_query(hq
);
2312 const struct nvc0_hw_sm_query_cfg
*cfg
;
2314 unsigned num_ab
[2] = { 0, 0 };
2316 cfg
= nvc0_hw_sm_query_get_cfg(nvc0
, hq
);
2318 /* check if we have enough free counter slots */
2319 for (i
= 0; i
< cfg
->num_counters
; ++i
)
2320 num_ab
[cfg
->ctr
[i
].sig_dom
]++;
2322 if (screen
->pm
.num_hw_sm_active
[0] + num_ab
[0] > 4 ||
2323 screen
->pm
.num_hw_sm_active
[1] + num_ab
[1] > 4) {
2324 NOUVEAU_ERR("Not enough free MP counter slots !\n");
2328 assert(cfg
->num_counters
<= 4);
2329 PUSH_SPACE(push
, 4 * 8 * + 6);
2331 if (!screen
->pm
.mp_counters_enabled
) {
2332 screen
->pm
.mp_counters_enabled
= true;
2333 BEGIN_NVC0(push
, SUBC_SW(0x06ac), 1);
2334 PUSH_DATA (push
, 0x1fcb);
2337 /* set sequence field to 0 (used to check if result is available) */
2338 for (i
= 0; i
< screen
->mp_count
; ++i
)
2339 hq
->data
[i
* 10 + 10] = 0;
2342 for (i
= 0; i
< cfg
->num_counters
; ++i
) {
2343 const unsigned d
= cfg
->ctr
[i
].sig_dom
;
2345 if (!screen
->pm
.num_hw_sm_active
[d
]) {
2346 uint32_t m
= (1 << 22) | (1 << (7 + (8 * !d
)));
2347 if (screen
->pm
.num_hw_sm_active
[!d
])
2348 m
|= 1 << (7 + (8 * d
));
2349 BEGIN_NVC0(push
, SUBC_SW(0x0600), 1);
2350 PUSH_DATA (push
, m
);
2352 screen
->pm
.num_hw_sm_active
[d
]++;
2354 for (c
= d
* 4; c
< (d
* 4 + 4); ++c
) {
2355 if (!screen
->pm
.mp_counter
[c
]) {
2357 screen
->pm
.mp_counter
[c
] = hsq
;
2361 assert(c
<= (d
* 4 + 3)); /* must succeed, already checked for space */
2363 /* configure and reset the counter(s) */
2365 BEGIN_NVC0(push
, NVE4_CP(MP_PM_A_SIGSEL(c
& 3)), 1);
2367 BEGIN_NVC0(push
, NVE4_CP(MP_PM_B_SIGSEL(c
& 3)), 1);
2368 PUSH_DATA (push
, cfg
->ctr
[i
].sig_sel
);
2369 BEGIN_NVC0(push
, NVE4_CP(MP_PM_SRCSEL(c
)), 1);
2370 PUSH_DATA (push
, cfg
->ctr
[i
].src_sel
+ 0x2108421 * (c
& 3));
2371 BEGIN_NVC0(push
, NVE4_CP(MP_PM_FUNC(c
)), 1);
2372 PUSH_DATA (push
, (cfg
->ctr
[i
].func
<< 4) | cfg
->ctr
[i
].mode
);
2373 BEGIN_NVC0(push
, NVE4_CP(MP_PM_SET(c
)), 1);
2374 PUSH_DATA (push
, 0);
2377 if (screen
->base
.class_3d
>= GM107_3D_CLASS
) {
2378 /* Enable mask for counters, it's 8-bits value where 0:3 is for domain A
2379 * and 4:7 for domain B. For example, the mask for active_warps should be
2380 * 0x70 because it uses 3 counters in domain B. However, let's always
2381 * enable all counters because we don't want to track which ones is
2382 * enabled or not, and this allows to monitor multiple queries at the
2384 BEGIN_NVC0(push
, SUBC_CP(0x33e0), 1);
2385 PUSH_DATA (push
, 0xff);
2392 nvc0_hw_sm_begin_query(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
)
2394 struct nvc0_screen
*screen
= nvc0
->screen
;
2395 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
2396 struct nvc0_hw_sm_query
*hsq
= nvc0_hw_sm_query(hq
);
2397 const struct nvc0_hw_sm_query_cfg
*cfg
;
2400 if (screen
->base
.class_3d
>= NVE4_3D_CLASS
)
2401 return nve4_hw_sm_begin_query(nvc0
, hq
);
2403 cfg
= nvc0_hw_sm_query_get_cfg(nvc0
, hq
);
2405 /* check if we have enough free counter slots */
2406 if (screen
->pm
.num_hw_sm_active
[0] + cfg
->num_counters
> 8) {
2407 NOUVEAU_ERR("Not enough free MP counter slots !\n");
2411 assert(cfg
->num_counters
<= 8);
2412 PUSH_SPACE(push
, 8 * 8 + 2);
2414 /* set sequence field to 0 (used to check if result is available) */
2415 for (i
= 0; i
< screen
->mp_count
; ++i
) {
2416 const unsigned b
= (0x30 / 4) * i
;
2417 hq
->data
[b
+ 8] = 0;
2421 for (i
= 0; i
< cfg
->num_counters
; ++i
) {
2422 uint32_t mask_sel
= 0x00000000;
2424 if (!screen
->pm
.num_hw_sm_active
[0]) {
2425 BEGIN_NVC0(push
, SUBC_SW(0x0600), 1);
2426 PUSH_DATA (push
, 0x80000000);
2428 screen
->pm
.num_hw_sm_active
[0]++;
2430 for (c
= 0; c
< 8; ++c
) {
2431 if (!screen
->pm
.mp_counter
[c
]) {
2433 screen
->pm
.mp_counter
[c
] = hsq
;
2438 /* Oddly-enough, the signal id depends on the slot selected on Fermi but
2439 * not on Kepler. Fortunately, the signal ids are just offseted by the
2442 mask_sel
|= (c
<< 8);
2443 mask_sel
|= (c
<< 16);
2444 mask_sel
|= (c
<< 24);
2445 mask_sel
&= cfg
->ctr
[i
].src_mask
;
2447 /* configure and reset the counter(s) */
2448 BEGIN_NVC0(push
, NVC0_CP(MP_PM_SIGSEL(c
)), 1);
2449 PUSH_DATA (push
, cfg
->ctr
[i
].sig_sel
);
2450 BEGIN_NVC0(push
, NVC0_CP(MP_PM_SRCSEL(c
)), 1);
2451 PUSH_DATA (push
, cfg
->ctr
[i
].src_sel
| mask_sel
);
2452 BEGIN_NVC0(push
, NVC0_CP(MP_PM_OP(c
)), 1);
2453 PUSH_DATA (push
, (cfg
->ctr
[i
].func
<< 4) | cfg
->ctr
[i
].mode
);
2454 BEGIN_NVC0(push
, NVC0_CP(MP_PM_SET(c
)), 1);
2455 PUSH_DATA (push
, 0);
2460 static inline struct nvc0_program
*
2461 nvc0_hw_sm_get_program(struct nvc0_screen
*screen
)
2463 struct nvc0_program
*prog
;
2465 prog
= CALLOC_STRUCT(nvc0_program
);
2469 prog
->type
= PIPE_SHADER_COMPUTE
;
2470 prog
->translated
= true;
2471 prog
->parm_size
= 12;
2473 if (screen
->base
.class_3d
>= GM107_3D_CLASS
) {
2474 prog
->code
= (uint32_t *)gm107_read_hw_sm_counters_code
;
2475 prog
->code_size
= sizeof(gm107_read_hw_sm_counters_code
);
2476 prog
->num_gprs
= 14;
2478 if (screen
->base
.class_3d
== NVE4_3D_CLASS
||
2479 screen
->base
.class_3d
== NVF0_3D_CLASS
) {
2480 if (screen
->base
.class_3d
== NVE4_3D_CLASS
) {
2481 prog
->code
= (uint32_t *)nve4_read_hw_sm_counters_code
;
2482 prog
->code_size
= sizeof(nve4_read_hw_sm_counters_code
);
2484 prog
->code
= (uint32_t *)nvf0_read_hw_sm_counters_code
;
2485 prog
->code_size
= sizeof(nvf0_read_hw_sm_counters_code
);
2487 prog
->num_gprs
= 14;
2489 prog
->code
= (uint32_t *)nvc0_read_hw_sm_counters_code
;
2490 prog
->code_size
= sizeof(nvc0_read_hw_sm_counters_code
);
2491 prog
->num_gprs
= 12;
2497 nvc0_hw_sm_upload_input(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
)
2499 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
2500 struct nvc0_screen
*screen
= nvc0
->screen
;
2504 address
= screen
->uniform_bo
->offset
+ NVC0_CB_AUX_INFO(s
);
2506 PUSH_SPACE(push
, 11);
2508 if (screen
->base
.class_3d
>= NVE4_3D_CLASS
) {
2509 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH
), 2);
2510 PUSH_DATAh(push
, address
+ NVC0_CB_AUX_MP_INFO
);
2511 PUSH_DATA (push
, address
+ NVC0_CB_AUX_MP_INFO
);
2512 BEGIN_NVC0(push
, NVE4_CP(UPLOAD_LINE_LENGTH_IN
), 2);
2513 PUSH_DATA (push
, 3 * 4);
2514 PUSH_DATA (push
, 0x1);
2515 BEGIN_1IC0(push
, NVE4_CP(UPLOAD_EXEC
), 1 + 3);
2516 PUSH_DATA (push
, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR
| (0x20 << 1));
2518 BEGIN_NVC0(push
, NVC0_CP(CB_SIZE
), 3);
2519 PUSH_DATA (push
, NVC0_CB_AUX_SIZE
);
2520 PUSH_DATAh(push
, address
);
2521 PUSH_DATA (push
, address
);
2522 BEGIN_1IC0(push
, NVC0_CP(CB_POS
), 1 + 3);
2523 PUSH_DATA (push
, NVC0_CB_AUX_MP_INFO
);
2525 PUSH_DATA (push
, (hq
->bo
->offset
+ hq
->base_offset
));
2526 PUSH_DATAh(push
, (hq
->bo
->offset
+ hq
->base_offset
));
2527 PUSH_DATA (push
, hq
->sequence
);
2531 nvc0_hw_sm_end_query(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
)
2533 struct nvc0_screen
*screen
= nvc0
->screen
;
2534 struct pipe_context
*pipe
= &nvc0
->base
.pipe
;
2535 struct nouveau_pushbuf
*push
= nvc0
->base
.pushbuf
;
2536 const bool is_nve4
= screen
->base
.class_3d
>= NVE4_3D_CLASS
;
2537 struct nvc0_hw_sm_query
*hsq
= nvc0_hw_sm_query(hq
);
2538 struct nvc0_program
*old
= nvc0
->compprog
;
2539 struct pipe_grid_info info
= {};
2542 const uint block
[3] = { 32, is_nve4
? 4 : 1, 1 };
2543 const uint grid
[3] = { screen
->mp_count
, screen
->gpc_count
, 1 };
2546 if (unlikely(!screen
->pm
.prog
))
2547 screen
->pm
.prog
= nvc0_hw_sm_get_program(screen
);
2549 /* disable all counting */
2550 PUSH_SPACE(push
, 8);
2551 for (c
= 0; c
< 8; ++c
)
2552 if (screen
->pm
.mp_counter
[c
]) {
2554 IMMED_NVC0(push
, NVE4_CP(MP_PM_FUNC(c
)), 0);
2556 IMMED_NVC0(push
, NVC0_CP(MP_PM_OP(c
)), 0);
2559 /* release counters for this query */
2560 for (c
= 0; c
< 8; ++c
) {
2561 if (screen
->pm
.mp_counter
[c
] == hsq
) {
2562 uint8_t d
= is_nve4
? c
/ 4 : 0; /* only one domain for NVC0:NVE4 */
2563 screen
->pm
.num_hw_sm_active
[d
]--;
2564 screen
->pm
.mp_counter
[c
] = NULL
;
2568 if (screen
->base
.class_3d
>= GM107_3D_CLASS
)
2569 IMMED_NVC0(push
, SUBC_CP(0x33e0), 0);
2571 BCTX_REFN_bo(nvc0
->bufctx_cp
, CP_QUERY
, NOUVEAU_BO_GART
| NOUVEAU_BO_WR
,
2574 PUSH_SPACE(push
, 1);
2575 IMMED_NVC0(push
, SUBC_CP(NV50_GRAPH_SERIALIZE
), 0);
2577 /* upload input data for the compute shader which reads MP counters */
2578 nvc0_hw_sm_upload_input(nvc0
, hq
);
2580 pipe
->bind_compute_state(pipe
, screen
->pm
.prog
);
2581 for (i
= 0; i
< 3; i
++) {
2582 info
.block
[i
] = block
[i
];
2583 info
.grid
[i
] = grid
[i
];
2587 pipe
->launch_grid(pipe
, &info
);
2588 pipe
->bind_compute_state(pipe
, old
);
2590 nouveau_bufctx_reset(nvc0
->bufctx_cp
, NVC0_BIND_CP_QUERY
);
2592 /* re-activate other counters */
2593 PUSH_SPACE(push
, 16);
2595 for (c
= 0; c
< 8; ++c
) {
2596 const struct nvc0_hw_sm_query_cfg
*cfg
;
2599 hsq
= screen
->pm
.mp_counter
[c
];
2603 cfg
= nvc0_hw_sm_query_get_cfg(nvc0
, &hsq
->base
);
2604 for (i
= 0; i
< cfg
->num_counters
; ++i
) {
2605 if (mask
& (1 << hsq
->ctr
[i
]))
2607 mask
|= 1 << hsq
->ctr
[i
];
2609 BEGIN_NVC0(push
, NVE4_CP(MP_PM_FUNC(hsq
->ctr
[i
])), 1);
2611 BEGIN_NVC0(push
, NVC0_CP(MP_PM_OP(hsq
->ctr
[i
])), 1);
2613 PUSH_DATA (push
, (cfg
->ctr
[i
].func
<< 4) | cfg
->ctr
[i
].mode
);
2619 nvc0_hw_sm_query_read_data(uint32_t count
[32][8],
2620 struct nvc0_context
*nvc0
, bool wait
,
2621 struct nvc0_hw_query
*hq
,
2622 const struct nvc0_hw_sm_query_cfg
*cfg
,
2625 struct nvc0_hw_sm_query
*hsq
= nvc0_hw_sm_query(hq
);
2628 for (p
= 0; p
< mp_count
; ++p
) {
2629 const unsigned b
= (0x30 / 4) * p
;
2631 for (c
= 0; c
< cfg
->num_counters
; ++c
) {
2632 if (hq
->data
[b
+ 8] != hq
->sequence
) {
2635 if (nouveau_bo_wait(hq
->bo
, NOUVEAU_BO_RD
, nvc0
->base
.client
))
2638 count
[p
][c
] = hq
->data
[b
+ hsq
->ctr
[c
]] * (1 << c
);
2645 nve4_hw_sm_query_read_data(uint32_t count
[32][8],
2646 struct nvc0_context
*nvc0
, bool wait
,
2647 struct nvc0_hw_query
*hq
,
2648 const struct nvc0_hw_sm_query_cfg
*cfg
,
2651 struct nvc0_hw_sm_query
*hsq
= nvc0_hw_sm_query(hq
);
2654 for (p
= 0; p
< mp_count
; ++p
) {
2655 const unsigned b
= (0x60 / 4) * p
;
2657 for (c
= 0; c
< cfg
->num_counters
; ++c
) {
2659 for (d
= 0; d
< ((hsq
->ctr
[c
] & ~3) ? 1 : 4); ++d
) {
2660 if (hq
->data
[b
+ 20 + d
] != hq
->sequence
) {
2663 if (nouveau_bo_wait(hq
->bo
, NOUVEAU_BO_RD
, nvc0
->base
.client
))
2666 if (hsq
->ctr
[c
] & ~0x3)
2667 count
[p
][c
] = hq
->data
[b
+ 16 + (hsq
->ctr
[c
] & 3)];
2669 count
[p
][c
] += hq
->data
[b
+ d
* 4 + hsq
->ctr
[c
]];
2677 nvc0_hw_sm_get_query_result(struct nvc0_context
*nvc0
, struct nvc0_hw_query
*hq
,
2678 boolean wait
, union pipe_query_result
*result
)
2680 uint32_t count
[32][8];
2682 unsigned mp_count
= MIN2(nvc0
->screen
->mp_count_compute
, 32);
2684 const struct nvc0_hw_sm_query_cfg
*cfg
;
2687 cfg
= nvc0_hw_sm_query_get_cfg(nvc0
, hq
);
2689 if (nvc0
->screen
->base
.class_3d
>= NVE4_3D_CLASS
)
2690 ret
= nve4_hw_sm_query_read_data(count
, nvc0
, wait
, hq
, cfg
, mp_count
);
2692 ret
= nvc0_hw_sm_query_read_data(count
, nvc0
, wait
, hq
, cfg
, mp_count
);
2696 for (c
= 0; c
< cfg
->num_counters
; ++c
)
2697 for (p
= 0; p
< mp_count
; ++p
)
2698 value
+= count
[p
][c
];
2699 value
= (value
* cfg
->norm
[0]) / cfg
->norm
[1];
2701 *(uint64_t *)result
= value
;
2705 static const struct nvc0_hw_query_funcs hw_sm_query_funcs
= {
2706 .destroy_query
= nvc0_hw_sm_destroy_query
,
2707 .begin_query
= nvc0_hw_sm_begin_query
,
2708 .end_query
= nvc0_hw_sm_end_query
,
2709 .get_query_result
= nvc0_hw_sm_get_query_result
,
2712 struct nvc0_hw_query
*
2713 nvc0_hw_sm_create_query(struct nvc0_context
*nvc0
, unsigned type
)
2715 struct nvc0_screen
*screen
= nvc0
->screen
;
2716 struct nvc0_hw_sm_query
*hsq
;
2717 struct nvc0_hw_query
*hq
;
2720 if (nvc0
->screen
->base
.drm
->version
< 0x01000101)
2723 if (type
< NVC0_HW_SM_QUERY(0) || type
> NVC0_HW_SM_QUERY_LAST
)
2726 hsq
= CALLOC_STRUCT(nvc0_hw_sm_query
);
2731 hq
->funcs
= &hw_sm_query_funcs
;
2732 hq
->base
.type
= type
;
2734 if (screen
->base
.class_3d
>= NVE4_3D_CLASS
) {
2756 * [50] = WS0.sequence
2757 * [54] = WS1.sequence
2758 * [58] = WS2.sequence
2759 * [5c] = WS3.sequence
2761 space
= (4 * 4 + 4 + 4) * nvc0
->screen
->mp_count
* sizeof(uint32_t);
2764 * Note that padding is used to align memory access to 128 bits.
2775 * [20] = MP.sequence
2780 space
= (8 + 1 + 3) * nvc0
->screen
->mp_count
* sizeof(uint32_t);
2783 if (!nvc0_hw_query_allocate(nvc0
, &hq
->base
, space
)) {
2792 nvc0_hw_sm_get_driver_query_info(struct nvc0_screen
*screen
, unsigned id
,
2793 struct pipe_driver_query_info
*info
)
2797 if (screen
->base
.drm
->version
>= 0x01000101) {
2798 if (screen
->compute
)
2799 count
= nvc0_hw_sm_get_num_queries(screen
);
2806 if (screen
->compute
) {
2807 if (screen
->base
.class_3d
<= GM200_3D_CLASS
) {
2808 const struct nvc0_hw_sm_query_cfg
**queries
=
2809 nvc0_hw_sm_get_queries(screen
);
2811 info
->name
= nvc0_hw_sm_query_get_name(queries
[id
]->type
);
2812 info
->query_type
= NVC0_HW_SM_QUERY(queries
[id
]->type
);
2813 info
->group_id
= NVC0_HW_SM_QUERY_GROUP
;