2 * Copyright © 2018 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 #include <sys/sysmacros.h>
33 #include "util/hash_table.h"
34 #include "compiler/glsl/list.h"
35 #include "util/ralloc.h"
37 struct gen_device_info
;
39 struct gen_perf_config
;
40 struct gen_perf_query_info
;
42 enum gen_perf_counter_type
{
43 GEN_PERF_COUNTER_TYPE_EVENT
,
44 GEN_PERF_COUNTER_TYPE_DURATION_NORM
,
45 GEN_PERF_COUNTER_TYPE_DURATION_RAW
,
46 GEN_PERF_COUNTER_TYPE_THROUGHPUT
,
47 GEN_PERF_COUNTER_TYPE_RAW
,
48 GEN_PERF_COUNTER_TYPE_TIMESTAMP
,
51 enum gen_perf_counter_data_type
{
52 GEN_PERF_COUNTER_DATA_TYPE_BOOL32
,
53 GEN_PERF_COUNTER_DATA_TYPE_UINT32
,
54 GEN_PERF_COUNTER_DATA_TYPE_UINT64
,
55 GEN_PERF_COUNTER_DATA_TYPE_FLOAT
,
56 GEN_PERF_COUNTER_DATA_TYPE_DOUBLE
,
59 struct gen_pipeline_stat
{
66 * The largest OA formats we can use include:
68 * 1 timestamp, 45 A counters, 8 B counters and 8 C counters.
70 * 1 timestamp, 1 clock, 36 A counters, 8 B counters and 8 C counters
72 #define MAX_OA_REPORT_COUNTERS 62
74 #define IA_VERTICES_COUNT 0x2310
75 #define IA_PRIMITIVES_COUNT 0x2318
76 #define VS_INVOCATION_COUNT 0x2320
77 #define HS_INVOCATION_COUNT 0x2300
78 #define DS_INVOCATION_COUNT 0x2308
79 #define GS_INVOCATION_COUNT 0x2328
80 #define GS_PRIMITIVES_COUNT 0x2330
81 #define CL_INVOCATION_COUNT 0x2338
82 #define CL_PRIMITIVES_COUNT 0x2340
83 #define PS_INVOCATION_COUNT 0x2348
84 #define CS_INVOCATION_COUNT 0x2290
85 #define PS_DEPTH_COUNT 0x2350
88 * When currently allocate only one page for pipeline statistics queries. Here
89 * we derived the maximum number of counters for that amount.
91 #define STATS_BO_SIZE 4096
92 #define STATS_BO_END_OFFSET_BYTES (STATS_BO_SIZE / 2)
93 #define MAX_STAT_COUNTERS (STATS_BO_END_OFFSET_BYTES / 8)
95 #define I915_PERF_OA_SAMPLE_SIZE (8 + /* drm_i915_perf_record_header */ \
96 256) /* OA counter report */
98 struct gen_perf_query_result
{
100 * Storage for the final accumulated OA counters.
102 uint64_t accumulator
[MAX_OA_REPORT_COUNTERS
];
105 * Hw ID used by the context on which the query was running.
110 * Number of reports accumulated to produce the results.
112 uint32_t reports_accumulated
;
115 * Frequency in the slices of the GT at the begin and end of the
118 uint64_t slice_frequency
[2];
121 * Frequency in the unslice of the GT at the begin and end of the
124 uint64_t unslice_frequency
[2];
127 struct gen_perf_query_counter
{
130 enum gen_perf_counter_type type
;
131 enum gen_perf_counter_data_type data_type
;
136 uint64_t (*oa_counter_read_uint64
)(struct gen_perf_config
*perf
,
137 const struct gen_perf_query_info
*query
,
138 const uint64_t *accumulator
);
139 float (*oa_counter_read_float
)(struct gen_perf_config
*perf
,
140 const struct gen_perf_query_info
*query
,
141 const uint64_t *accumulator
);
142 struct gen_pipeline_stat pipeline_stat
;
146 struct gen_perf_query_register_prog
{
151 struct gen_perf_query_info
{
152 enum gen_perf_query_type
{
153 GEN_PERF_QUERY_TYPE_OA
,
154 GEN_PERF_QUERY_TYPE_RAW
,
155 GEN_PERF_QUERY_TYPE_PIPELINE
,
159 struct gen_perf_query_counter
*counters
;
165 uint64_t oa_metrics_set_id
;
168 /* For indexing into the accumulator[] ... */
170 int gpu_clock_offset
;
175 /* Register programming for a given query */
176 struct gen_perf_query_register_prog
*flex_regs
;
177 uint32_t n_flex_regs
;
179 struct gen_perf_query_register_prog
*mux_regs
;
182 struct gen_perf_query_register_prog
*b_counter_regs
;
183 uint32_t n_b_counter_regs
;
186 struct gen_perf_config
{
187 struct gen_perf_query_info
*queries
;
190 /* Variables referenced in the XML meta data for OA performance
191 * counters, e.g in the normalization equations.
193 * All uint64_t for consistent operand types in generated code
196 uint64_t timestamp_frequency
; /** $GpuTimestampFrequency */
197 uint64_t n_eus
; /** $EuCoresTotalCount */
198 uint64_t n_eu_slices
; /** $EuSlicesTotalCount */
199 uint64_t n_eu_sub_slices
; /** $EuSubslicesTotalCount */
200 uint64_t eu_threads_count
; /** $EuThreadsCount */
201 uint64_t slice_mask
; /** $SliceMask */
202 uint64_t subslice_mask
; /** $SubsliceMask */
203 uint64_t gt_min_freq
; /** $GpuMinFrequency */
204 uint64_t gt_max_freq
; /** $GpuMaxFrequency */
205 uint64_t revision
; /** $SkuRevisionId */
208 /* OA metric sets, indexed by GUID, as know by Mesa at build time, to
209 * cross-reference with the GUIDs of configs advertised by the kernel at
212 struct hash_table
*oa_metrics_table
;
214 /* Location of the device's sysfs entry. */
215 char sysfs_dev_dir
[256];
218 void *(*bo_alloc
)(void *bufmgr
, const char *name
, uint64_t size
);
219 void (*bo_unreference
)(void *bo
);
220 void (*emit_mi_report_perf_count
)(void *ctx
,
222 uint32_t offset_in_bytes
,
224 void (*batchbuffer_flush
)(void *ctx
,
225 const char *file
, int line
);
226 void (*capture_frequency_stat_register
)(void *ctx
, void *bo
,
232 * Periodic OA samples are read() into these buffer structures via the
233 * i915 perf kernel interface and appended to the
234 * brw->perfquery.sample_buffers linked list. When we process the
235 * results of an OA metrics query we need to consider all the periodic
236 * samples between the Begin and End MI_REPORT_PERF_COUNT command
239 * 'Periodic' is a simplification as there are other automatic reports
240 * written by the hardware also buffered here.
242 * Considering three queries, A, B and C:
245 * ________________A_________________
247 * | ________B_________ _____C___________
250 * And an illustration of sample buffers read over this time frame:
251 * [HEAD ][ ][ ][ ][ ][ ][ ][ ][TAIL ]
253 * These nodes may hold samples for query A:
254 * [ ][ ][ A ][ A ][ A ][ A ][ A ][ ][ ]
256 * These nodes may hold samples for query B:
257 * [ ][ ][ B ][ B ][ B ][ ][ ][ ][ ]
259 * These nodes may hold samples for query C:
260 * [ ][ ][ ][ ][ ][ C ][ C ][ C ][ ]
262 * The illustration assumes we have an even distribution of periodic
263 * samples so all nodes have the same size plotted against time:
265 * Note, to simplify code, the list is never empty.
267 * With overlapping queries we can see that periodic OA reports may
268 * relate to multiple queries and care needs to be take to keep
269 * track of sample buffers until there are no queries that might
270 * depend on their contents.
272 * We use a node ref counting system where a reference ensures that a
273 * node and all following nodes can't be freed/recycled until the
274 * reference drops to zero.
276 * E.g. with a ref of one here:
277 * [ 0 ][ 0 ][ 1 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ]
279 * These nodes could be freed or recycled ("reaped"):
282 * These must be preserved until the leading ref drops to zero:
283 * [ 1 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ]
285 * When a query starts we take a reference on the current tail of
286 * the list, knowing that no already-buffered samples can possibly
287 * relate to the newly-started query. A pointer to this node is
288 * also saved in the query object's ->oa.samples_head.
290 * E.g. starting query A while there are two nodes in .sample_buffers:
291 * ________________A________
295 * ^_______ Add a reference and store pointer to node in
298 * Moving forward to when the B query starts with no new buffer nodes:
299 * (for reference, i915 perf reads() are only done when queries finish)
300 * ________________A_______
305 * ^_______ Add a reference and store pointer to
306 * node in B->oa.samples_head
308 * Once a query is finished, after an OA query has become 'Ready',
309 * once the End OA report has landed and after we we have processed
310 * all the intermediate periodic samples then we drop the
311 * ->oa.samples_head reference we took at the start.
313 * So when the B query has finished we have:
314 * ________________A________
315 * | ______B___________
317 * [ 0 ][ 1 ][ 0 ][ 0 ][ 0 ]
318 * ^_______ Drop B->oa.samples_head reference
320 * We still can't free these due to the A->oa.samples_head ref:
321 * [ 1 ][ 0 ][ 0 ][ 0 ]
323 * When the A query finishes: (note there's a new ref for C's samples_head)
324 * ________________A_________________
328 * [ 0 ][ 0 ][ 0 ][ 0 ][ 1 ][ 0 ][ 0 ]
329 * ^_______ Drop A->oa.samples_head reference
331 * And we can now reap these nodes up to the C->oa.samples_head:
332 * [ X ][ X ][ X ][ X ]
333 * keeping -> [ 1 ][ 0 ][ 0 ]
335 * We reap old sample buffers each time we finish processing an OA
336 * query by iterating the sample_buffers list from the head until we
337 * find a referenced node and stop.
339 * Reaped buffers move to a perfquery.free_sample_buffers list and
340 * when we come to read() we first look to recycle a buffer from the
341 * free_sample_buffers list before allocating a new buffer.
343 struct oa_sample_buf
{
344 struct exec_node link
;
347 uint8_t buf
[I915_PERF_OA_SAMPLE_SIZE
* 10];
348 uint32_t last_timestamp
;
353 gen_perf_query_counter_get_size(const struct gen_perf_query_counter
*counter
)
355 switch (counter
->data_type
) {
356 case GEN_PERF_COUNTER_DATA_TYPE_BOOL32
:
357 return sizeof(uint32_t);
358 case GEN_PERF_COUNTER_DATA_TYPE_UINT32
:
359 return sizeof(uint32_t);
360 case GEN_PERF_COUNTER_DATA_TYPE_UINT64
:
361 return sizeof(uint64_t);
362 case GEN_PERF_COUNTER_DATA_TYPE_FLOAT
:
363 return sizeof(float);
364 case GEN_PERF_COUNTER_DATA_TYPE_DOUBLE
:
365 return sizeof(double);
367 unreachable("invalid counter data type");
371 static inline struct gen_perf_query_info
*
372 gen_perf_query_append_query_info(struct gen_perf_config
*perf
, int max_counters
)
374 struct gen_perf_query_info
*query
;
376 perf
->queries
= reralloc(perf
, perf
->queries
,
377 struct gen_perf_query_info
,
379 query
= &perf
->queries
[perf
->n_queries
- 1];
380 memset(query
, 0, sizeof(*query
));
382 if (max_counters
> 0) {
383 query
->max_counters
= max_counters
;
385 rzalloc_array(perf
, struct gen_perf_query_counter
, max_counters
);
392 gen_perf_query_info_add_stat_reg(struct gen_perf_query_info
*query
,
395 uint32_t denominator
,
397 const char *description
)
399 struct gen_perf_query_counter
*counter
;
401 assert(query
->n_counters
< query
->max_counters
);
403 counter
= &query
->counters
[query
->n_counters
];
404 counter
->name
= name
;
405 counter
->desc
= description
;
406 counter
->type
= GEN_PERF_COUNTER_TYPE_RAW
;
407 counter
->data_type
= GEN_PERF_COUNTER_DATA_TYPE_UINT64
;
408 counter
->offset
= sizeof(uint64_t) * query
->n_counters
;
409 counter
->pipeline_stat
.reg
= reg
;
410 counter
->pipeline_stat
.numerator
= numerator
;
411 counter
->pipeline_stat
.denominator
= denominator
;
417 gen_perf_query_info_add_basic_stat_reg(struct gen_perf_query_info
*query
,
418 uint32_t reg
, const char *name
)
420 gen_perf_query_info_add_stat_reg(query
, reg
, 1, 1, name
, name
);
423 static inline struct gen_perf_config
*
424 gen_perf_new(void *ctx
)
426 struct gen_perf_config
*perf
= rzalloc(ctx
, struct gen_perf_config
);
430 bool gen_perf_load_oa_metrics(struct gen_perf_config
*perf
, int fd
,
431 const struct gen_device_info
*devinfo
);
432 bool gen_perf_load_metric_id(struct gen_perf_config
*perf
, const char *guid
,
433 uint64_t *metric_id
);
435 void gen_perf_query_result_read_frequencies(struct gen_perf_query_result
*result
,
436 const struct gen_device_info
*devinfo
,
437 const uint32_t *start
,
438 const uint32_t *end
);
439 void gen_perf_query_result_accumulate(struct gen_perf_query_result
*result
,
440 const struct gen_perf_query_info
*query
,
441 const uint32_t *start
,
442 const uint32_t *end
);
443 void gen_perf_query_result_clear(struct gen_perf_query_result
*result
);
444 void gen_perf_query_register_mdapi_statistic_query(const struct gen_device_info
*devinfo
,
445 struct gen_perf_config
*perf
);
446 void gen_perf_query_register_mdapi_oa_query(const struct gen_device_info
*devinfo
,
447 struct gen_perf_config
*perf
);
448 uint64_t gen_perf_query_get_metric_id(struct gen_perf_config
*perf
,
449 const struct gen_perf_query_info
*query
);
452 #endif /* GEN_PERF_H */