2 * Copyright © 2013 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 * \file brw_performance_query.c
27 * Implementation of the GL_INTEL_performance_query extension.
29 * Currently there are two possible counter sources exposed here:
31 * On Gen6+ hardware we have numerous 64bit Pipeline Statistics Registers
32 * that we can snapshot at the beginning and end of a query.
34 * On Gen7.5+ we have Observability Architecture counters which are
35 * covered in separate document from the rest of the PRMs. It is available at:
36 * https://01.org/linuxgraphics/documentation/driver-documentation-prms
37 * => 2013 Intel Core Processor Family => Observability Performance Counters
38 * (This one volume covers Sandybridge, Ivybridge, Baytrail, and Haswell,
39 * though notably we currently only support OA counters for Haswell+)
44 /* put before sys/types.h to silence glibc warnings */
46 #include <sys/mkdev.h>
48 #ifdef MAJOR_IN_SYSMACROS
49 #include <sys/sysmacros.h>
51 #include <sys/types.h>
55 #include <sys/ioctl.h>
58 #include "drm-uapi/i915_drm.h"
60 #include "main/hash.h"
61 #include "main/macros.h"
62 #include "main/mtypes.h"
63 #include "main/performance_query.h"
65 #include "util/bitset.h"
66 #include "util/ralloc.h"
67 #include "util/hash_table.h"
68 #include "util/list.h"
69 #include "util/u_math.h"
71 #include "brw_context.h"
72 #include "brw_defines.h"
73 #include "intel_batchbuffer.h"
75 #include "perf/gen_perf.h"
76 #include "perf/gen_perf_mdapi.h"
78 #define FILE_DEBUG_FLAG DEBUG_PERFMON
80 #define OAREPORT_REASON_MASK 0x3f
81 #define OAREPORT_REASON_SHIFT 19
82 #define OAREPORT_REASON_TIMER (1<<0)
83 #define OAREPORT_REASON_TRIGGER1 (1<<1)
84 #define OAREPORT_REASON_TRIGGER2 (1<<2)
85 #define OAREPORT_REASON_CTX_SWITCH (1<<3)
86 #define OAREPORT_REASON_GO_TRANSITION (1<<4)
88 struct brw_perf_query_object
{
89 struct gl_perf_query_object base
;
90 struct gen_perf_query_object
*query
;
93 /** Downcasting convenience macro. */
94 static inline struct brw_perf_query_object
*
95 brw_perf_query(struct gl_perf_query_object
*o
)
97 return (struct brw_perf_query_object
*) o
;
100 #define MI_RPC_BO_SIZE 4096
101 #define MI_RPC_BO_END_OFFSET_BYTES (MI_RPC_BO_SIZE / 2)
102 #define MI_FREQ_START_OFFSET_BYTES (3072)
103 #define MI_FREQ_END_OFFSET_BYTES (3076)
105 /******************************************************************************/
108 brw_is_perf_query_ready(struct gl_context
*ctx
,
109 struct gl_perf_query_object
*o
);
112 dump_perf_query_callback(GLuint id
, void *query_void
, void *brw_void
)
114 struct gl_context
*ctx
= brw_void
;
115 struct gl_perf_query_object
*o
= query_void
;
116 struct brw_perf_query_object
* brw_query
= brw_perf_query(o
);
117 struct gen_perf_query_object
*obj
= brw_query
->query
;
119 switch (obj
->queryinfo
->kind
) {
120 case GEN_PERF_QUERY_TYPE_OA
:
121 case GEN_PERF_QUERY_TYPE_RAW
:
122 DBG("%4d: %-6s %-8s BO: %-4s OA data: %-10s %-15s\n",
124 o
->Used
? "Dirty," : "New,",
125 o
->Active
? "Active," : (o
->Ready
? "Ready," : "Pending,"),
126 obj
->oa
.bo
? "yes," : "no,",
127 brw_is_perf_query_ready(ctx
, o
) ? "ready," : "not ready,",
128 obj
->oa
.results_accumulated
? "accumulated" : "not accumulated");
130 case GEN_PERF_QUERY_TYPE_PIPELINE
:
131 DBG("%4d: %-6s %-8s BO: %-4s\n",
133 o
->Used
? "Dirty," : "New,",
134 o
->Active
? "Active," : (o
->Ready
? "Ready," : "Pending,"),
135 obj
->pipeline_stats
.bo
? "yes" : "no");
138 unreachable("Unknown query type");
144 dump_perf_queries(struct brw_context
*brw
)
146 struct gl_context
*ctx
= &brw
->ctx
;
147 DBG("Queries: (Open queries = %d, OA users = %d)\n",
148 brw
->perf_ctx
.n_active_oa_queries
, brw
->perf_ctx
.n_oa_users
);
149 _mesa_HashWalk(ctx
->PerfQuery
.Objects
, dump_perf_query_callback
, brw
);
153 * Driver hook for glGetPerfQueryInfoINTEL().
156 brw_get_perf_query_info(struct gl_context
*ctx
,
157 unsigned query_index
,
163 struct brw_context
*brw
= brw_context(ctx
);
164 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
165 const struct gen_perf_query_info
*query
=
166 &perf_ctx
->perf
->queries
[query_index
];
169 *data_size
= query
->data_size
;
170 *n_counters
= query
->n_counters
;
172 switch (query
->kind
) {
173 case GEN_PERF_QUERY_TYPE_OA
:
174 case GEN_PERF_QUERY_TYPE_RAW
:
175 *n_active
= perf_ctx
->n_active_oa_queries
;
178 case GEN_PERF_QUERY_TYPE_PIPELINE
:
179 *n_active
= perf_ctx
->n_active_pipeline_stats_queries
;
183 unreachable("Unknown query type");
189 gen_counter_type_enum_to_gl_type(enum gen_perf_counter_type type
)
192 case GEN_PERF_COUNTER_TYPE_EVENT
: return GL_PERFQUERY_COUNTER_EVENT_INTEL
;
193 case GEN_PERF_COUNTER_TYPE_DURATION_NORM
: return GL_PERFQUERY_COUNTER_DURATION_NORM_INTEL
;
194 case GEN_PERF_COUNTER_TYPE_DURATION_RAW
: return GL_PERFQUERY_COUNTER_DURATION_RAW_INTEL
;
195 case GEN_PERF_COUNTER_TYPE_THROUGHPUT
: return GL_PERFQUERY_COUNTER_THROUGHPUT_INTEL
;
196 case GEN_PERF_COUNTER_TYPE_RAW
: return GL_PERFQUERY_COUNTER_RAW_INTEL
;
197 case GEN_PERF_COUNTER_TYPE_TIMESTAMP
: return GL_PERFQUERY_COUNTER_TIMESTAMP_INTEL
;
199 unreachable("Unknown counter type");
204 gen_counter_data_type_to_gl_type(enum gen_perf_counter_data_type type
)
207 case GEN_PERF_COUNTER_DATA_TYPE_BOOL32
: return GL_PERFQUERY_COUNTER_DATA_BOOL32_INTEL
;
208 case GEN_PERF_COUNTER_DATA_TYPE_UINT32
: return GL_PERFQUERY_COUNTER_DATA_UINT32_INTEL
;
209 case GEN_PERF_COUNTER_DATA_TYPE_UINT64
: return GL_PERFQUERY_COUNTER_DATA_UINT64_INTEL
;
210 case GEN_PERF_COUNTER_DATA_TYPE_FLOAT
: return GL_PERFQUERY_COUNTER_DATA_FLOAT_INTEL
;
211 case GEN_PERF_COUNTER_DATA_TYPE_DOUBLE
: return GL_PERFQUERY_COUNTER_DATA_DOUBLE_INTEL
;
213 unreachable("Unknown counter data type");
218 * Driver hook for glGetPerfCounterInfoINTEL().
221 brw_get_perf_counter_info(struct gl_context
*ctx
,
222 unsigned query_index
,
223 unsigned counter_index
,
229 GLuint
*data_type_enum
,
232 struct brw_context
*brw
= brw_context(ctx
);
233 const struct gen_perf_query_info
*query
=
234 &brw
->perf_ctx
.perf
->queries
[query_index
];
235 const struct gen_perf_query_counter
*counter
=
236 &query
->counters
[counter_index
];
238 *name
= counter
->name
;
239 *desc
= counter
->desc
;
240 *offset
= counter
->offset
;
241 *data_size
= gen_perf_query_counter_get_size(counter
);
242 *type_enum
= gen_counter_type_enum_to_gl_type(counter
->type
);
243 *data_type_enum
= gen_counter_data_type_to_gl_type(counter
->data_type
);
244 *raw_max
= counter
->raw_max
;
248 * Remove a query from the global list of unaccumulated queries once
249 * after successfully accumulating the OA reports associated with the
250 * query in accumulate_oa_reports() or when discarding unwanted query
254 drop_from_unaccumulated_query_list(struct brw_context
*brw
,
255 struct gen_perf_query_object
*obj
)
257 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
258 for (int i
= 0; i
< perf_ctx
->unaccumulated_elements
; i
++) {
259 if (perf_ctx
->unaccumulated
[i
] == obj
) {
260 int last_elt
= --perf_ctx
->unaccumulated_elements
;
263 perf_ctx
->unaccumulated
[i
] = NULL
;
265 perf_ctx
->unaccumulated
[i
] =
266 perf_ctx
->unaccumulated
[last_elt
];
273 /* Drop our samples_head reference so that associated periodic
274 * sample data buffers can potentially be reaped if they aren't
275 * referenced by any other queries...
278 struct oa_sample_buf
*buf
=
279 exec_node_data(struct oa_sample_buf
, obj
->oa
.samples_head
, link
);
281 assert(buf
->refcount
> 0);
284 obj
->oa
.samples_head
= NULL
;
286 gen_perf_reap_old_sample_buffers(&brw
->perf_ctx
);
289 /* In general if we see anything spurious while accumulating results,
290 * we don't try and continue accumulating the current query, hoping
291 * for the best, we scrap anything outstanding, and then hope for the
292 * best with new queries.
295 discard_all_queries(struct brw_context
*brw
)
297 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
298 while (perf_ctx
->unaccumulated_elements
) {
299 struct gen_perf_query_object
*obj
= perf_ctx
->unaccumulated
[0];
301 obj
->oa
.results_accumulated
= true;
302 drop_from_unaccumulated_query_list(brw
, perf_ctx
->unaccumulated
[0]);
304 gen_perf_dec_n_users(perf_ctx
);
309 OA_READ_STATUS_ERROR
,
310 OA_READ_STATUS_UNFINISHED
,
311 OA_READ_STATUS_FINISHED
,
315 * Accumulate raw OA counter values based on deltas between pairs of
318 * Accumulation starts from the first report captured via
319 * MI_REPORT_PERF_COUNT (MI_RPC) by brw_begin_perf_query() until the
320 * last MI_RPC report requested by brw_end_perf_query(). Between these
321 * two reports there may also some number of periodically sampled OA
322 * reports collected via the i915 perf interface - depending on the
323 * duration of the query.
325 * These periodic snapshots help to ensure we handle counter overflow
326 * correctly by being frequent enough to ensure we don't miss multiple
327 * overflows of a counter between snapshots. For Gen8+ the i915 perf
328 * snapshots provide the extra context-switch reports that let us
329 * subtract out the progress of counters associated with other
330 * contexts running on the system.
333 accumulate_oa_reports(struct brw_context
*brw
,
334 struct brw_perf_query_object
*brw_query
)
336 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
337 struct gen_perf_query_object
*obj
= brw_query
->query
;
338 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
342 struct exec_node
*first_samples_node
;
344 int out_duration
= 0;
346 assert(brw_query
->base
.Ready
);
347 assert(obj
->oa
.map
!= NULL
);
349 start
= last
= obj
->oa
.map
;
350 end
= obj
->oa
.map
+ MI_RPC_BO_END_OFFSET_BYTES
;
352 if (start
[0] != obj
->oa
.begin_report_id
) {
353 DBG("Spurious start report id=%"PRIu32
"\n", start
[0]);
356 if (end
[0] != (obj
->oa
.begin_report_id
+ 1)) {
357 DBG("Spurious end report id=%"PRIu32
"\n", end
[0]);
361 /* See if we have any periodic reports to accumulate too... */
363 /* N.B. The oa.samples_head was set when the query began and
364 * pointed to the tail of the perf_ctx->sample_buffers list at
365 * the time the query started. Since the buffer existed before the
366 * first MI_REPORT_PERF_COUNT command was emitted we therefore know
367 * that no data in this particular node's buffer can possibly be
368 * associated with the query - so skip ahead one...
370 first_samples_node
= obj
->oa
.samples_head
->next
;
372 foreach_list_typed_from(struct oa_sample_buf
, buf
, link
,
373 &brw
->perf_ctx
.sample_buffers
,
378 while (offset
< buf
->len
) {
379 const struct drm_i915_perf_record_header
*header
=
380 (const struct drm_i915_perf_record_header
*)(buf
->buf
+ offset
);
382 assert(header
->size
!= 0);
383 assert(header
->size
<= buf
->len
);
385 offset
+= header
->size
;
387 switch (header
->type
) {
388 case DRM_I915_PERF_RECORD_SAMPLE
: {
389 uint32_t *report
= (uint32_t *)(header
+ 1);
392 /* Ignore reports that come before the start marker.
393 * (Note: takes care to allow overflow of 32bit timestamps)
395 if (gen_device_info_timebase_scale(devinfo
,
396 report
[1] - start
[1]) > 5000000000) {
400 /* Ignore reports that come after the end marker.
401 * (Note: takes care to allow overflow of 32bit timestamps)
403 if (gen_device_info_timebase_scale(devinfo
,
404 report
[1] - end
[1]) <= 5000000000) {
408 /* For Gen8+ since the counters continue while other
409 * contexts are running we need to discount any unrelated
410 * deltas. The hardware automatically generates a report
411 * on context switch which gives us a new reference point
412 * to continuing adding deltas from.
414 * For Haswell we can rely on the HW to stop the progress
415 * of OA counters while any other context is acctive.
417 if (devinfo
->gen
>= 8) {
418 if (in_ctx
&& report
[2] != obj
->oa
.result
.hw_id
) {
419 DBG("i915 perf: Switch AWAY (observed by ID change)\n");
422 } else if (in_ctx
== false && report
[2] == obj
->oa
.result
.hw_id
) {
423 DBG("i915 perf: Switch TO\n");
426 /* From experimentation in IGT, we found that the OA unit
427 * might label some report as "idle" (using an invalid
428 * context ID), right after a report for a given context.
429 * Deltas generated by those reports actually belong to the
430 * previous context, even though they're not labelled as
433 * We didn't *really* Switch AWAY in the case that we e.g.
434 * saw a single periodic report while idle...
436 if (out_duration
>= 1)
439 assert(report
[2] == obj
->oa
.result
.hw_id
);
440 DBG("i915 perf: Continuation IN\n");
442 assert(report
[2] != obj
->oa
.result
.hw_id
);
443 DBG("i915 perf: Continuation OUT\n");
450 gen_perf_query_result_accumulate(&obj
->oa
.result
, obj
->queryinfo
,
459 case DRM_I915_PERF_RECORD_OA_BUFFER_LOST
:
460 DBG("i915 perf: OA error: all reports lost\n");
462 case DRM_I915_PERF_RECORD_OA_REPORT_LOST
:
463 DBG("i915 perf: OA report lost\n");
471 gen_perf_query_result_accumulate(&obj
->oa
.result
, obj
->queryinfo
,
474 DBG("Marking %d accumulated - results gathered\n", brw_query
->base
.Id
);
476 obj
->oa
.results_accumulated
= true;
477 drop_from_unaccumulated_query_list(brw
, obj
);
478 gen_perf_dec_n_users(perf_ctx
);
484 discard_all_queries(brw
);
487 /******************************************************************************/
490 capture_frequency_stat_register(struct brw_context
*brw
,
494 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
496 if (devinfo
->gen
>= 7 && devinfo
->gen
<= 8 &&
497 !devinfo
->is_baytrail
&& !devinfo
->is_cherryview
) {
498 brw_store_register_mem32(brw
, bo
, GEN7_RPSTAT1
, bo_offset
);
499 } else if (devinfo
->gen
>= 9) {
500 brw_store_register_mem32(brw
, bo
, GEN9_RPSTAT0
, bo_offset
);
505 * Driver hook for glBeginPerfQueryINTEL().
508 brw_begin_perf_query(struct gl_context
*ctx
,
509 struct gl_perf_query_object
*o
)
511 struct brw_context
*brw
= brw_context(ctx
);
512 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
513 struct gen_perf_query_object
*obj
= brw_query
->query
;
514 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
516 /* We can assume the frontend hides mistaken attempts to Begin a
517 * query object multiple times before its End. Similarly if an
518 * application reuses a query object before results have arrived
519 * the frontend will wait for prior results so we don't need
520 * to support abandoning in-flight results.
523 assert(!o
->Used
|| o
->Ready
); /* no in-flight query to worry about */
525 DBG("Begin(%d)\n", o
->Id
);
527 gen_perf_begin_query(perf_ctx
, obj
);
529 if (INTEL_DEBUG
& DEBUG_PERFMON
)
530 dump_perf_queries(brw
);
536 * Driver hook for glEndPerfQueryINTEL().
539 brw_end_perf_query(struct gl_context
*ctx
,
540 struct gl_perf_query_object
*o
)
542 struct brw_context
*brw
= brw_context(ctx
);
543 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
544 struct gen_perf_query_object
*obj
= brw_query
->query
;
545 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
547 DBG("End(%d)\n", o
->Id
);
548 gen_perf_end_query(perf_ctx
, obj
);
552 brw_wait_perf_query(struct gl_context
*ctx
, struct gl_perf_query_object
*o
)
554 struct brw_context
*brw
= brw_context(ctx
);
555 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
556 struct gen_perf_query_object
*obj
= brw_query
->query
;
560 gen_perf_wait_query(&brw
->perf_ctx
, obj
, &brw
->batch
);
564 brw_is_perf_query_ready(struct gl_context
*ctx
,
565 struct gl_perf_query_object
*o
)
567 struct brw_context
*brw
= brw_context(ctx
);
568 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
569 struct gen_perf_query_object
*obj
= brw_query
->query
;
574 return gen_perf_is_query_ready(&brw
->perf_ctx
, obj
, &brw
->batch
);
578 read_slice_unslice_frequencies(struct brw_context
*brw
,
579 struct gen_perf_query_object
*obj
)
581 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
582 uint32_t *begin_report
= obj
->oa
.map
, *end_report
= obj
->oa
.map
+ MI_RPC_BO_END_OFFSET_BYTES
;
584 gen_perf_query_result_read_frequencies(&obj
->oa
.result
,
585 devinfo
, begin_report
, end_report
);
589 read_gt_frequency(struct brw_context
*brw
,
590 struct gen_perf_query_object
*obj
)
592 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
593 uint32_t start
= *((uint32_t *)(obj
->oa
.map
+ MI_FREQ_START_OFFSET_BYTES
)),
594 end
= *((uint32_t *)(obj
->oa
.map
+ MI_FREQ_END_OFFSET_BYTES
));
596 switch (devinfo
->gen
) {
599 obj
->oa
.gt_frequency
[0] = GET_FIELD(start
, GEN7_RPSTAT1_CURR_GT_FREQ
) * 50ULL;
600 obj
->oa
.gt_frequency
[1] = GET_FIELD(end
, GEN7_RPSTAT1_CURR_GT_FREQ
) * 50ULL;
605 obj
->oa
.gt_frequency
[0] = GET_FIELD(start
, GEN9_RPSTAT0_CURR_GT_FREQ
) * 50ULL / 3ULL;
606 obj
->oa
.gt_frequency
[1] = GET_FIELD(end
, GEN9_RPSTAT0_CURR_GT_FREQ
) * 50ULL / 3ULL;
609 unreachable("unexpected gen");
612 /* Put the numbers into Hz. */
613 obj
->oa
.gt_frequency
[0] *= 1000000ULL;
614 obj
->oa
.gt_frequency
[1] *= 1000000ULL;
618 get_oa_counter_data(struct brw_context
*brw
,
619 struct gen_perf_query_object
*obj
,
623 struct gen_perf_config
*perf
= brw
->perf_ctx
.perf
;
624 const struct gen_perf_query_info
*query
= obj
->queryinfo
;
625 int n_counters
= query
->n_counters
;
628 for (int i
= 0; i
< n_counters
; i
++) {
629 const struct gen_perf_query_counter
*counter
= &query
->counters
[i
];
630 uint64_t *out_uint64
;
632 size_t counter_size
= gen_perf_query_counter_get_size(counter
);
635 switch (counter
->data_type
) {
636 case GEN_PERF_COUNTER_DATA_TYPE_UINT64
:
637 out_uint64
= (uint64_t *)(data
+ counter
->offset
);
639 counter
->oa_counter_read_uint64(perf
, query
,
640 obj
->oa
.result
.accumulator
);
642 case GEN_PERF_COUNTER_DATA_TYPE_FLOAT
:
643 out_float
= (float *)(data
+ counter
->offset
);
645 counter
->oa_counter_read_float(perf
, query
,
646 obj
->oa
.result
.accumulator
);
649 /* So far we aren't using uint32, double or bool32... */
650 unreachable("unexpected counter data type");
652 written
= counter
->offset
+ counter_size
;
660 get_pipeline_stats_data(struct brw_context
*brw
,
661 struct gen_perf_query_object
*obj
,
666 const struct gen_perf_query_info
*query
= obj
->queryinfo
;
667 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
668 struct gen_perf_config
*perf_cfg
= perf_ctx
->perf
;
669 int n_counters
= obj
->queryinfo
->n_counters
;
672 uint64_t *start
= perf_cfg
->vtbl
.bo_map(perf_ctx
->ctx
, obj
->pipeline_stats
.bo
, MAP_READ
);
673 uint64_t *end
= start
+ (STATS_BO_END_OFFSET_BYTES
/ sizeof(uint64_t));
675 for (int i
= 0; i
< n_counters
; i
++) {
676 const struct gen_perf_query_counter
*counter
= &query
->counters
[i
];
677 uint64_t value
= end
[i
] - start
[i
];
679 if (counter
->pipeline_stat
.numerator
!=
680 counter
->pipeline_stat
.denominator
) {
681 value
*= counter
->pipeline_stat
.numerator
;
682 value
/= counter
->pipeline_stat
.denominator
;
685 *((uint64_t *)p
) = value
;
689 perf_cfg
->vtbl
.bo_unmap(obj
->pipeline_stats
.bo
);
695 * Driver hook for glGetPerfQueryDataINTEL().
698 brw_get_perf_query_data(struct gl_context
*ctx
,
699 struct gl_perf_query_object
*o
,
702 GLuint
*bytes_written
)
704 struct brw_context
*brw
= brw_context(ctx
);
705 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
706 struct gen_perf_query_object
*obj
= brw_query
->query
;
709 assert(brw_is_perf_query_ready(ctx
, o
));
711 DBG("GetData(%d)\n", o
->Id
);
713 if (INTEL_DEBUG
& DEBUG_PERFMON
)
714 dump_perf_queries(brw
);
716 /* We expect that the frontend only calls this hook when it knows
717 * that results are available.
721 switch (obj
->queryinfo
->kind
) {
722 case GEN_PERF_QUERY_TYPE_OA
:
723 case GEN_PERF_QUERY_TYPE_RAW
:
724 if (!obj
->oa
.results_accumulated
) {
725 read_gt_frequency(brw
, obj
);
726 read_slice_unslice_frequencies(brw
, obj
);
727 accumulate_oa_reports(brw
, brw_query
);
728 assert(obj
->oa
.results_accumulated
);
730 brw
->perf_ctx
.perf
->vtbl
.bo_unmap(obj
->oa
.bo
);
733 if (obj
->queryinfo
->kind
== GEN_PERF_QUERY_TYPE_OA
) {
734 written
= get_oa_counter_data(brw
, obj
, data_size
, (uint8_t *)data
);
736 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
738 written
= gen_perf_query_result_write_mdapi((uint8_t *)data
, data_size
,
739 devinfo
, &obj
->oa
.result
,
740 obj
->oa
.gt_frequency
[0],
741 obj
->oa
.gt_frequency
[1]);
745 case GEN_PERF_QUERY_TYPE_PIPELINE
:
746 written
= get_pipeline_stats_data(brw
, obj
, data_size
, (uint8_t *)data
);
750 unreachable("Unknown query type");
755 *bytes_written
= written
;
758 static struct gl_perf_query_object
*
759 brw_new_perf_query_object(struct gl_context
*ctx
, unsigned query_index
)
761 struct brw_context
*brw
= brw_context(ctx
);
762 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
763 const struct gen_perf_query_info
*queryinfo
=
764 &perf_ctx
->perf
->queries
[query_index
];
765 struct gen_perf_query_object
*obj
=
766 calloc(1, sizeof(struct gen_perf_query_object
));
771 obj
->queryinfo
= queryinfo
;
773 perf_ctx
->n_query_instances
++;
775 struct brw_perf_query_object
*brw_query
= calloc(1, sizeof(struct brw_perf_query_object
));
776 if (unlikely(!brw_query
))
778 brw_query
->query
= obj
;
779 return &brw_query
->base
;
783 * Driver hook for glDeletePerfQueryINTEL().
786 brw_delete_perf_query(struct gl_context
*ctx
,
787 struct gl_perf_query_object
*o
)
789 struct brw_context
*brw
= brw_context(ctx
);
790 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
791 struct gen_perf_query_object
*obj
= brw_query
->query
;
792 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
794 /* We can assume that the frontend waits for a query to complete
795 * before ever calling into here, so we don't have to worry about
796 * deleting an in-flight query object.
799 assert(!o
->Used
|| o
->Ready
);
801 DBG("Delete(%d)\n", o
->Id
);
803 gen_perf_delete_query(perf_ctx
, obj
);
807 /******************************************************************************/
810 init_pipeline_statistic_query_registers(struct brw_context
*brw
)
812 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
813 struct gen_perf_config
*perf
= brw
->perf_ctx
.perf
;
814 struct gen_perf_query_info
*query
=
815 gen_perf_query_append_query_info(perf
, MAX_STAT_COUNTERS
);
817 query
->kind
= GEN_PERF_QUERY_TYPE_PIPELINE
;
818 query
->name
= "Pipeline Statistics Registers";
820 gen_perf_query_info_add_basic_stat_reg(query
, IA_VERTICES_COUNT
,
821 "N vertices submitted");
822 gen_perf_query_info_add_basic_stat_reg(query
, IA_PRIMITIVES_COUNT
,
823 "N primitives submitted");
824 gen_perf_query_info_add_basic_stat_reg(query
, VS_INVOCATION_COUNT
,
825 "N vertex shader invocations");
827 if (devinfo
->gen
== 6) {
828 gen_perf_query_info_add_stat_reg(query
, GEN6_SO_PRIM_STORAGE_NEEDED
, 1, 1,
829 "SO_PRIM_STORAGE_NEEDED",
830 "N geometry shader stream-out primitives (total)");
831 gen_perf_query_info_add_stat_reg(query
, GEN6_SO_NUM_PRIMS_WRITTEN
, 1, 1,
832 "SO_NUM_PRIMS_WRITTEN",
833 "N geometry shader stream-out primitives (written)");
835 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_PRIM_STORAGE_NEEDED(0), 1, 1,
836 "SO_PRIM_STORAGE_NEEDED (Stream 0)",
837 "N stream-out (stream 0) primitives (total)");
838 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_PRIM_STORAGE_NEEDED(1), 1, 1,
839 "SO_PRIM_STORAGE_NEEDED (Stream 1)",
840 "N stream-out (stream 1) primitives (total)");
841 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_PRIM_STORAGE_NEEDED(2), 1, 1,
842 "SO_PRIM_STORAGE_NEEDED (Stream 2)",
843 "N stream-out (stream 2) primitives (total)");
844 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_PRIM_STORAGE_NEEDED(3), 1, 1,
845 "SO_PRIM_STORAGE_NEEDED (Stream 3)",
846 "N stream-out (stream 3) primitives (total)");
847 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_NUM_PRIMS_WRITTEN(0), 1, 1,
848 "SO_NUM_PRIMS_WRITTEN (Stream 0)",
849 "N stream-out (stream 0) primitives (written)");
850 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_NUM_PRIMS_WRITTEN(1), 1, 1,
851 "SO_NUM_PRIMS_WRITTEN (Stream 1)",
852 "N stream-out (stream 1) primitives (written)");
853 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_NUM_PRIMS_WRITTEN(2), 1, 1,
854 "SO_NUM_PRIMS_WRITTEN (Stream 2)",
855 "N stream-out (stream 2) primitives (written)");
856 gen_perf_query_info_add_stat_reg(query
, GEN7_SO_NUM_PRIMS_WRITTEN(3), 1, 1,
857 "SO_NUM_PRIMS_WRITTEN (Stream 3)",
858 "N stream-out (stream 3) primitives (written)");
861 gen_perf_query_info_add_basic_stat_reg(query
, HS_INVOCATION_COUNT
,
862 "N TCS shader invocations");
863 gen_perf_query_info_add_basic_stat_reg(query
, DS_INVOCATION_COUNT
,
864 "N TES shader invocations");
866 gen_perf_query_info_add_basic_stat_reg(query
, GS_INVOCATION_COUNT
,
867 "N geometry shader invocations");
868 gen_perf_query_info_add_basic_stat_reg(query
, GS_PRIMITIVES_COUNT
,
869 "N geometry shader primitives emitted");
871 gen_perf_query_info_add_basic_stat_reg(query
, CL_INVOCATION_COUNT
,
872 "N primitives entering clipping");
873 gen_perf_query_info_add_basic_stat_reg(query
, CL_PRIMITIVES_COUNT
,
874 "N primitives leaving clipping");
876 if (devinfo
->is_haswell
|| devinfo
->gen
== 8) {
877 gen_perf_query_info_add_stat_reg(query
, PS_INVOCATION_COUNT
, 1, 4,
878 "N fragment shader invocations",
879 "N fragment shader invocations");
881 gen_perf_query_info_add_basic_stat_reg(query
, PS_INVOCATION_COUNT
,
882 "N fragment shader invocations");
885 gen_perf_query_info_add_basic_stat_reg(query
, PS_DEPTH_COUNT
,
886 "N z-pass fragments");
888 if (devinfo
->gen
>= 7) {
889 gen_perf_query_info_add_basic_stat_reg(query
, CS_INVOCATION_COUNT
,
890 "N compute shader invocations");
893 query
->data_size
= sizeof(uint64_t) * query
->n_counters
;
896 /* gen_device_info will have incorrect default topology values for unsupported kernels.
897 * verify kernel support to ensure OA metrics are accurate.
900 oa_metrics_kernel_support(int fd
, const struct gen_device_info
*devinfo
)
902 if (devinfo
->gen
>= 10) {
903 /* topology uAPI required for CNL+ (kernel 4.17+) make a call to the api
906 struct drm_i915_query_item item
= {
907 .query_id
= DRM_I915_QUERY_TOPOLOGY_INFO
,
909 struct drm_i915_query query
= {
911 .items_ptr
= (uintptr_t) &item
,
914 /* kernel 4.17+ supports the query */
915 return drmIoctl(fd
, DRM_IOCTL_I915_QUERY
, &query
) == 0;
918 if (devinfo
->gen
>= 8) {
919 /* 4.13+ api required for gen8 - gen9 */
921 struct drm_i915_getparam gp
= {
922 .param
= I915_PARAM_SLICE_MASK
,
925 /* kernel 4.13+ supports this parameter */
926 return drmIoctl(fd
, DRM_IOCTL_I915_GETPARAM
, &gp
) == 0;
929 if (devinfo
->gen
== 7)
930 /* default topology values are correct for HSW */
933 /* oa not supported before gen 7*/
938 brw_oa_bo_alloc(void *bufmgr
, const char *name
, uint64_t size
)
940 return brw_bo_alloc(bufmgr
, name
, size
, BRW_MEMZONE_OTHER
);
944 brw_oa_emit_mi_report_perf_count(void *c
,
946 uint32_t offset_in_bytes
,
949 struct brw_context
*ctx
= c
;
950 ctx
->vtbl
.emit_mi_report_perf_count(ctx
,
956 typedef void (*bo_unreference_t
)(void *);
957 typedef void *(*bo_map_t
)(void *, void *, unsigned flags
);
958 typedef void (*bo_unmap_t
)(void *);
959 typedef void (* emit_mi_report_t
)(void *, void *, uint32_t, uint32_t);
960 typedef void (*emit_mi_flush_t
)(void *);
963 brw_oa_batchbuffer_flush(void *c
, const char *file
, int line
)
965 struct brw_context
*ctx
= c
;
966 _intel_batchbuffer_flush_fence(ctx
, -1, NULL
, file
, line
);
969 typedef void (*capture_frequency_stat_register_t
)(void *, void *, uint32_t );
970 typedef void (*store_register_mem64_t
)(void *ctx
, void *bo
,
971 uint32_t reg
, uint32_t offset
);
972 typedef bool (*batch_references_t
)(void *batch
, void *bo
);
973 typedef void (*bo_wait_rendering_t
)(void *bo
);
974 typedef int (*bo_busy_t
)(void *bo
);
977 brw_init_perf_query_info(struct gl_context
*ctx
)
979 struct brw_context
*brw
= brw_context(ctx
);
980 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
982 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
984 return perf_ctx
->perf
->n_queries
;
986 perf_ctx
->perf
= gen_perf_new(brw
);
987 struct gen_perf_config
*perf_cfg
= perf_ctx
->perf
;
989 perf_cfg
->vtbl
.bo_alloc
= brw_oa_bo_alloc
;
990 perf_cfg
->vtbl
.bo_unreference
= (bo_unreference_t
)brw_bo_unreference
;
991 perf_cfg
->vtbl
.bo_map
= (bo_map_t
)brw_bo_map
;
992 perf_cfg
->vtbl
.bo_unmap
= (bo_unmap_t
)brw_bo_unmap
;
993 perf_cfg
->vtbl
.emit_mi_flush
= (emit_mi_flush_t
)brw_emit_mi_flush
;
994 perf_cfg
->vtbl
.emit_mi_report_perf_count
=
995 (emit_mi_report_t
)brw_oa_emit_mi_report_perf_count
;
996 perf_cfg
->vtbl
.batchbuffer_flush
= brw_oa_batchbuffer_flush
;
997 perf_cfg
->vtbl
.capture_frequency_stat_register
=
998 (capture_frequency_stat_register_t
) capture_frequency_stat_register
;
999 perf_cfg
->vtbl
.store_register_mem64
=
1000 (store_register_mem64_t
) brw_store_register_mem64
;
1001 perf_cfg
->vtbl
.batch_references
= (batch_references_t
)brw_batch_references
;
1002 perf_cfg
->vtbl
.bo_wait_rendering
= (bo_wait_rendering_t
)brw_bo_wait_rendering
;
1003 perf_cfg
->vtbl
.bo_busy
= (bo_busy_t
)brw_bo_busy
;
1005 gen_perf_init_context(perf_ctx
, perf_cfg
, brw
, brw
->bufmgr
, devinfo
,
1006 brw
->hw_ctx
, brw
->screen
->driScrnPriv
->fd
);
1008 init_pipeline_statistic_query_registers(brw
);
1009 gen_perf_query_register_mdapi_statistic_query(devinfo
, perf_cfg
);
1011 if ((oa_metrics_kernel_support(perf_ctx
->drm_fd
, devinfo
)) &&
1012 (gen_perf_load_oa_metrics(perf_cfg
, perf_ctx
->drm_fd
, devinfo
)))
1013 gen_perf_query_register_mdapi_oa_query(devinfo
, perf_cfg
);
1015 return perf_cfg
->n_queries
;
1019 brw_init_performance_queries(struct brw_context
*brw
)
1021 struct gl_context
*ctx
= &brw
->ctx
;
1023 ctx
->Driver
.InitPerfQueryInfo
= brw_init_perf_query_info
;
1024 ctx
->Driver
.GetPerfQueryInfo
= brw_get_perf_query_info
;
1025 ctx
->Driver
.GetPerfCounterInfo
= brw_get_perf_counter_info
;
1026 ctx
->Driver
.NewPerfQueryObject
= brw_new_perf_query_object
;
1027 ctx
->Driver
.DeletePerfQuery
= brw_delete_perf_query
;
1028 ctx
->Driver
.BeginPerfQuery
= brw_begin_perf_query
;
1029 ctx
->Driver
.EndPerfQuery
= brw_end_perf_query
;
1030 ctx
->Driver
.WaitPerfQuery
= brw_wait_perf_query
;
1031 ctx
->Driver
.IsPerfQueryReady
= brw_is_perf_query_ready
;
1032 ctx
->Driver
.GetPerfQueryData
= brw_get_perf_query_data
;