2 * Copyright © 2013 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 * \file brw_performance_query.c
27 * Implementation of the GL_INTEL_performance_query extension.
29 * Currently there are two possible counter sources exposed here:
31 * On Gen6+ hardware we have numerous 64bit Pipeline Statistics Registers
32 * that we can snapshot at the beginning and end of a query.
34 * On Gen7.5+ we have Observability Architecture counters which are
35 * covered in separate document from the rest of the PRMs. It is available at:
36 * https://01.org/linuxgraphics/documentation/driver-documentation-prms
37 * => 2013 Intel Core Processor Family => Observability Performance Counters
38 * (This one volume covers Sandybridge, Ivybridge, Baytrail, and Haswell,
39 * though notably we currently only support OA counters for Haswell+)
44 /* put before sys/types.h to silence glibc warnings */
46 #include <sys/mkdev.h>
48 #ifdef MAJOR_IN_SYSMACROS
49 #include <sys/sysmacros.h>
51 #include <sys/types.h>
55 #include <sys/ioctl.h>
58 #include "drm-uapi/i915_drm.h"
60 #include "main/hash.h"
61 #include "main/macros.h"
62 #include "main/mtypes.h"
63 #include "main/performance_query.h"
65 #include "util/bitset.h"
66 #include "util/ralloc.h"
67 #include "util/hash_table.h"
68 #include "util/list.h"
69 #include "util/u_math.h"
71 #include "brw_context.h"
72 #include "brw_defines.h"
73 #include "intel_batchbuffer.h"
75 #include "perf/gen_perf.h"
76 #include "perf/gen_perf_mdapi.h"
78 #define FILE_DEBUG_FLAG DEBUG_PERFMON
80 #define OAREPORT_REASON_MASK 0x3f
81 #define OAREPORT_REASON_SHIFT 19
82 #define OAREPORT_REASON_TIMER (1<<0)
83 #define OAREPORT_REASON_TRIGGER1 (1<<1)
84 #define OAREPORT_REASON_TRIGGER2 (1<<2)
85 #define OAREPORT_REASON_CTX_SWITCH (1<<3)
86 #define OAREPORT_REASON_GO_TRANSITION (1<<4)
88 struct brw_perf_query_object
{
89 struct gl_perf_query_object base
;
90 struct gen_perf_query_object
*query
;
93 /** Downcasting convenience macro. */
94 static inline struct brw_perf_query_object
*
95 brw_perf_query(struct gl_perf_query_object
*o
)
97 return (struct brw_perf_query_object
*) o
;
100 #define MI_RPC_BO_SIZE 4096
101 #define MI_RPC_BO_END_OFFSET_BYTES (MI_RPC_BO_SIZE / 2)
102 #define MI_FREQ_START_OFFSET_BYTES (3072)
103 #define MI_FREQ_END_OFFSET_BYTES (3076)
105 /******************************************************************************/
108 brw_is_perf_query_ready(struct gl_context
*ctx
,
109 struct gl_perf_query_object
*o
);
112 dump_perf_query_callback(GLuint id
, void *query_void
, void *brw_void
)
114 struct gl_context
*ctx
= brw_void
;
115 struct gl_perf_query_object
*o
= query_void
;
116 struct brw_perf_query_object
* brw_query
= brw_perf_query(o
);
117 struct gen_perf_query_object
*obj
= brw_query
->query
;
119 switch (obj
->queryinfo
->kind
) {
120 case GEN_PERF_QUERY_TYPE_OA
:
121 case GEN_PERF_QUERY_TYPE_RAW
:
122 DBG("%4d: %-6s %-8s BO: %-4s OA data: %-10s %-15s\n",
124 o
->Used
? "Dirty," : "New,",
125 o
->Active
? "Active," : (o
->Ready
? "Ready," : "Pending,"),
126 obj
->oa
.bo
? "yes," : "no,",
127 brw_is_perf_query_ready(ctx
, o
) ? "ready," : "not ready,",
128 obj
->oa
.results_accumulated
? "accumulated" : "not accumulated");
130 case GEN_PERF_QUERY_TYPE_PIPELINE
:
131 DBG("%4d: %-6s %-8s BO: %-4s\n",
133 o
->Used
? "Dirty," : "New,",
134 o
->Active
? "Active," : (o
->Ready
? "Ready," : "Pending,"),
135 obj
->pipeline_stats
.bo
? "yes" : "no");
138 unreachable("Unknown query type");
144 dump_perf_queries(struct brw_context
*brw
)
146 struct gl_context
*ctx
= &brw
->ctx
;
147 DBG("Queries: (Open queries = %d, OA users = %d)\n",
148 brw
->perf_ctx
.n_active_oa_queries
, brw
->perf_ctx
.n_oa_users
);
149 _mesa_HashWalk(ctx
->PerfQuery
.Objects
, dump_perf_query_callback
, brw
);
153 * Driver hook for glGetPerfQueryInfoINTEL().
156 brw_get_perf_query_info(struct gl_context
*ctx
,
157 unsigned query_index
,
163 struct brw_context
*brw
= brw_context(ctx
);
164 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
165 const struct gen_perf_query_info
*query
=
166 &perf_ctx
->perf
->queries
[query_index
];
169 *data_size
= query
->data_size
;
170 *n_counters
= query
->n_counters
;
172 switch (query
->kind
) {
173 case GEN_PERF_QUERY_TYPE_OA
:
174 case GEN_PERF_QUERY_TYPE_RAW
:
175 *n_active
= perf_ctx
->n_active_oa_queries
;
178 case GEN_PERF_QUERY_TYPE_PIPELINE
:
179 *n_active
= perf_ctx
->n_active_pipeline_stats_queries
;
183 unreachable("Unknown query type");
189 gen_counter_type_enum_to_gl_type(enum gen_perf_counter_type type
)
192 case GEN_PERF_COUNTER_TYPE_EVENT
: return GL_PERFQUERY_COUNTER_EVENT_INTEL
;
193 case GEN_PERF_COUNTER_TYPE_DURATION_NORM
: return GL_PERFQUERY_COUNTER_DURATION_NORM_INTEL
;
194 case GEN_PERF_COUNTER_TYPE_DURATION_RAW
: return GL_PERFQUERY_COUNTER_DURATION_RAW_INTEL
;
195 case GEN_PERF_COUNTER_TYPE_THROUGHPUT
: return GL_PERFQUERY_COUNTER_THROUGHPUT_INTEL
;
196 case GEN_PERF_COUNTER_TYPE_RAW
: return GL_PERFQUERY_COUNTER_RAW_INTEL
;
197 case GEN_PERF_COUNTER_TYPE_TIMESTAMP
: return GL_PERFQUERY_COUNTER_TIMESTAMP_INTEL
;
199 unreachable("Unknown counter type");
204 gen_counter_data_type_to_gl_type(enum gen_perf_counter_data_type type
)
207 case GEN_PERF_COUNTER_DATA_TYPE_BOOL32
: return GL_PERFQUERY_COUNTER_DATA_BOOL32_INTEL
;
208 case GEN_PERF_COUNTER_DATA_TYPE_UINT32
: return GL_PERFQUERY_COUNTER_DATA_UINT32_INTEL
;
209 case GEN_PERF_COUNTER_DATA_TYPE_UINT64
: return GL_PERFQUERY_COUNTER_DATA_UINT64_INTEL
;
210 case GEN_PERF_COUNTER_DATA_TYPE_FLOAT
: return GL_PERFQUERY_COUNTER_DATA_FLOAT_INTEL
;
211 case GEN_PERF_COUNTER_DATA_TYPE_DOUBLE
: return GL_PERFQUERY_COUNTER_DATA_DOUBLE_INTEL
;
213 unreachable("Unknown counter data type");
218 * Driver hook for glGetPerfCounterInfoINTEL().
221 brw_get_perf_counter_info(struct gl_context
*ctx
,
222 unsigned query_index
,
223 unsigned counter_index
,
229 GLuint
*data_type_enum
,
232 struct brw_context
*brw
= brw_context(ctx
);
233 const struct gen_perf_query_info
*query
=
234 &brw
->perf_ctx
.perf
->queries
[query_index
];
235 const struct gen_perf_query_counter
*counter
=
236 &query
->counters
[counter_index
];
238 *name
= counter
->name
;
239 *desc
= counter
->desc
;
240 *offset
= counter
->offset
;
241 *data_size
= gen_perf_query_counter_get_size(counter
);
242 *type_enum
= gen_counter_type_enum_to_gl_type(counter
->type
);
243 *data_type_enum
= gen_counter_data_type_to_gl_type(counter
->data_type
);
244 *raw_max
= counter
->raw_max
;
248 OA_READ_STATUS_ERROR
,
249 OA_READ_STATUS_UNFINISHED
,
250 OA_READ_STATUS_FINISHED
,
253 /******************************************************************************/
256 capture_frequency_stat_register(struct brw_context
*brw
,
260 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
262 if (devinfo
->gen
>= 7 && devinfo
->gen
<= 8 &&
263 !devinfo
->is_baytrail
&& !devinfo
->is_cherryview
) {
264 brw_store_register_mem32(brw
, bo
, GEN7_RPSTAT1
, bo_offset
);
265 } else if (devinfo
->gen
>= 9) {
266 brw_store_register_mem32(brw
, bo
, GEN9_RPSTAT0
, bo_offset
);
271 * Driver hook for glBeginPerfQueryINTEL().
274 brw_begin_perf_query(struct gl_context
*ctx
,
275 struct gl_perf_query_object
*o
)
277 struct brw_context
*brw
= brw_context(ctx
);
278 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
279 struct gen_perf_query_object
*obj
= brw_query
->query
;
280 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
282 /* We can assume the frontend hides mistaken attempts to Begin a
283 * query object multiple times before its End. Similarly if an
284 * application reuses a query object before results have arrived
285 * the frontend will wait for prior results so we don't need
286 * to support abandoning in-flight results.
289 assert(!o
->Used
|| o
->Ready
); /* no in-flight query to worry about */
291 DBG("Begin(%d)\n", o
->Id
);
293 gen_perf_begin_query(perf_ctx
, obj
);
295 if (INTEL_DEBUG
& DEBUG_PERFMON
)
296 dump_perf_queries(brw
);
302 * Driver hook for glEndPerfQueryINTEL().
305 brw_end_perf_query(struct gl_context
*ctx
,
306 struct gl_perf_query_object
*o
)
308 struct brw_context
*brw
= brw_context(ctx
);
309 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
310 struct gen_perf_query_object
*obj
= brw_query
->query
;
311 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
313 DBG("End(%d)\n", o
->Id
);
314 gen_perf_end_query(perf_ctx
, obj
);
318 brw_wait_perf_query(struct gl_context
*ctx
, struct gl_perf_query_object
*o
)
320 struct brw_context
*brw
= brw_context(ctx
);
321 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
322 struct gen_perf_query_object
*obj
= brw_query
->query
;
326 gen_perf_wait_query(&brw
->perf_ctx
, obj
, &brw
->batch
);
330 brw_is_perf_query_ready(struct gl_context
*ctx
,
331 struct gl_perf_query_object
*o
)
333 struct brw_context
*brw
= brw_context(ctx
);
334 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
335 struct gen_perf_query_object
*obj
= brw_query
->query
;
340 return gen_perf_is_query_ready(&brw
->perf_ctx
, obj
, &brw
->batch
);
344 * Driver hook for glGetPerfQueryDataINTEL().
347 brw_get_perf_query_data(struct gl_context
*ctx
,
348 struct gl_perf_query_object
*o
,
351 GLuint
*bytes_written
)
353 struct brw_context
*brw
= brw_context(ctx
);
354 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
355 struct gen_perf_query_object
*obj
= brw_query
->query
;
357 assert(brw_is_perf_query_ready(ctx
, o
));
359 DBG("GetData(%d)\n", o
->Id
);
361 if (INTEL_DEBUG
& DEBUG_PERFMON
)
362 dump_perf_queries(brw
);
364 /* We expect that the frontend only calls this hook when it knows
365 * that results are available.
369 gen_perf_get_query_data(&brw
->perf_ctx
, obj
,
370 data_size
, data
, bytes_written
);
373 static struct gl_perf_query_object
*
374 brw_new_perf_query_object(struct gl_context
*ctx
, unsigned query_index
)
376 struct brw_context
*brw
= brw_context(ctx
);
377 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
378 struct gen_perf_query_object
* obj
= gen_perf_new_query(perf_ctx
, query_index
);
382 struct brw_perf_query_object
*brw_query
= calloc(1, sizeof(struct brw_perf_query_object
));
383 if (unlikely(!brw_query
))
386 brw_query
->query
= obj
;
387 return &brw_query
->base
;
391 * Driver hook for glDeletePerfQueryINTEL().
394 brw_delete_perf_query(struct gl_context
*ctx
,
395 struct gl_perf_query_object
*o
)
397 struct brw_context
*brw
= brw_context(ctx
);
398 struct brw_perf_query_object
*brw_query
= brw_perf_query(o
);
399 struct gen_perf_query_object
*obj
= brw_query
->query
;
400 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
402 /* We can assume that the frontend waits for a query to complete
403 * before ever calling into here, so we don't have to worry about
404 * deleting an in-flight query object.
407 assert(!o
->Used
|| o
->Ready
);
409 DBG("Delete(%d)\n", o
->Id
);
411 gen_perf_delete_query(perf_ctx
, obj
);
415 /******************************************************************************/
416 /* gen_device_info will have incorrect default topology values for unsupported kernels.
417 * verify kernel support to ensure OA metrics are accurate.
420 oa_metrics_kernel_support(int fd
, const struct gen_device_info
*devinfo
)
422 if (devinfo
->gen
>= 10) {
423 /* topology uAPI required for CNL+ (kernel 4.17+) make a call to the api
426 struct drm_i915_query_item item
= {
427 .query_id
= DRM_I915_QUERY_TOPOLOGY_INFO
,
429 struct drm_i915_query query
= {
431 .items_ptr
= (uintptr_t) &item
,
434 /* kernel 4.17+ supports the query */
435 return drmIoctl(fd
, DRM_IOCTL_I915_QUERY
, &query
) == 0;
438 if (devinfo
->gen
>= 8) {
439 /* 4.13+ api required for gen8 - gen9 */
441 struct drm_i915_getparam gp
= {
442 .param
= I915_PARAM_SLICE_MASK
,
445 /* kernel 4.13+ supports this parameter */
446 return drmIoctl(fd
, DRM_IOCTL_I915_GETPARAM
, &gp
) == 0;
449 if (devinfo
->gen
== 7)
450 /* default topology values are correct for HSW */
453 /* oa not supported before gen 7*/
458 brw_oa_bo_alloc(void *bufmgr
, const char *name
, uint64_t size
)
460 return brw_bo_alloc(bufmgr
, name
, size
, BRW_MEMZONE_OTHER
);
464 brw_oa_emit_mi_report_perf_count(void *c
,
466 uint32_t offset_in_bytes
,
469 struct brw_context
*ctx
= c
;
470 ctx
->vtbl
.emit_mi_report_perf_count(ctx
,
476 typedef void (*bo_unreference_t
)(void *);
477 typedef void *(*bo_map_t
)(void *, void *, unsigned flags
);
478 typedef void (*bo_unmap_t
)(void *);
479 typedef void (* emit_mi_report_t
)(void *, void *, uint32_t, uint32_t);
480 typedef void (*emit_mi_flush_t
)(void *);
483 brw_oa_batchbuffer_flush(void *c
, const char *file
, int line
)
485 struct brw_context
*ctx
= c
;
486 _intel_batchbuffer_flush_fence(ctx
, -1, NULL
, file
, line
);
489 typedef void (*capture_frequency_stat_register_t
)(void *, void *, uint32_t );
490 typedef void (*store_register_mem64_t
)(void *ctx
, void *bo
,
491 uint32_t reg
, uint32_t offset
);
492 typedef bool (*batch_references_t
)(void *batch
, void *bo
);
493 typedef void (*bo_wait_rendering_t
)(void *bo
);
494 typedef int (*bo_busy_t
)(void *bo
);
497 brw_init_perf_query_info(struct gl_context
*ctx
)
499 struct brw_context
*brw
= brw_context(ctx
);
500 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
502 struct gen_perf_context
*perf_ctx
= &brw
->perf_ctx
;
504 return perf_ctx
->perf
->n_queries
;
506 perf_ctx
->perf
= gen_perf_new(brw
);
507 struct gen_perf_config
*perf_cfg
= perf_ctx
->perf
;
509 perf_cfg
->vtbl
.bo_alloc
= brw_oa_bo_alloc
;
510 perf_cfg
->vtbl
.bo_unreference
= (bo_unreference_t
)brw_bo_unreference
;
511 perf_cfg
->vtbl
.bo_map
= (bo_map_t
)brw_bo_map
;
512 perf_cfg
->vtbl
.bo_unmap
= (bo_unmap_t
)brw_bo_unmap
;
513 perf_cfg
->vtbl
.emit_mi_flush
= (emit_mi_flush_t
)brw_emit_mi_flush
;
514 perf_cfg
->vtbl
.emit_mi_report_perf_count
=
515 (emit_mi_report_t
)brw_oa_emit_mi_report_perf_count
;
516 perf_cfg
->vtbl
.batchbuffer_flush
= brw_oa_batchbuffer_flush
;
517 perf_cfg
->vtbl
.capture_frequency_stat_register
=
518 (capture_frequency_stat_register_t
) capture_frequency_stat_register
;
519 perf_cfg
->vtbl
.store_register_mem64
=
520 (store_register_mem64_t
) brw_store_register_mem64
;
521 perf_cfg
->vtbl
.batch_references
= (batch_references_t
)brw_batch_references
;
522 perf_cfg
->vtbl
.bo_wait_rendering
= (bo_wait_rendering_t
)brw_bo_wait_rendering
;
523 perf_cfg
->vtbl
.bo_busy
= (bo_busy_t
)brw_bo_busy
;
525 gen_perf_init_context(perf_ctx
, perf_cfg
, brw
, brw
->bufmgr
, devinfo
,
526 brw
->hw_ctx
, brw
->screen
->driScrnPriv
->fd
);
528 if (!oa_metrics_kernel_support(perf_ctx
->drm_fd
, devinfo
))
531 gen_perf_init_metrics(perf_cfg
, devinfo
, perf_ctx
->drm_fd
);
533 return perf_cfg
->n_queries
;
537 brw_init_performance_queries(struct brw_context
*brw
)
539 struct gl_context
*ctx
= &brw
->ctx
;
541 ctx
->Driver
.InitPerfQueryInfo
= brw_init_perf_query_info
;
542 ctx
->Driver
.GetPerfQueryInfo
= brw_get_perf_query_info
;
543 ctx
->Driver
.GetPerfCounterInfo
= brw_get_perf_counter_info
;
544 ctx
->Driver
.NewPerfQueryObject
= brw_new_perf_query_object
;
545 ctx
->Driver
.DeletePerfQuery
= brw_delete_perf_query
;
546 ctx
->Driver
.BeginPerfQuery
= brw_begin_perf_query
;
547 ctx
->Driver
.EndPerfQuery
= brw_end_perf_query
;
548 ctx
->Driver
.WaitPerfQuery
= brw_wait_perf_query
;
549 ctx
->Driver
.IsPerfQueryReady
= brw_is_perf_query_ready
;
550 ctx
->Driver
.GetPerfQueryData
= brw_get_perf_query_data
;