1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
4 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * Rob Clark <robclark@freedesktop.org>
29 #include "pipe/p_state.h"
30 #include "util/u_memory.h"
31 #include "util/u_inlines.h"
33 #include "freedreno_query_hw.h"
34 #include "freedreno_context.h"
35 #include "freedreno_util.h"
37 struct fd_hw_sample_period
{
38 struct fd_hw_sample
*start
, *end
;
39 struct list_head list
;
42 /* maps query_type to sample provider idx: */
43 static int pidx(unsigned query_type
)
46 case PIPE_QUERY_OCCLUSION_COUNTER
:
48 case PIPE_QUERY_OCCLUSION_PREDICATE
:
50 /* TODO currently queries only emitted in main pass (not in binning pass)..
51 * which is fine for occlusion query, but pretty much not anything else.
53 case PIPE_QUERY_TIME_ELAPSED
:
55 case PIPE_QUERY_TIMESTAMP
:
62 static struct fd_hw_sample
*
63 get_sample(struct fd_context
*ctx
, struct fd_ringbuffer
*ring
,
66 struct fd_hw_sample
*samp
= NULL
;
67 int idx
= pidx(query_type
);
69 assume(idx
>= 0); /* query never would have been created otherwise */
71 if (!ctx
->sample_cache
[idx
]) {
72 ctx
->sample_cache
[idx
] =
73 ctx
->sample_providers
[idx
]->get_sample(ctx
, ring
);
74 ctx
->needs_flush
= true;
77 fd_hw_sample_reference(ctx
, &samp
, ctx
->sample_cache
[idx
]);
83 clear_sample_cache(struct fd_context
*ctx
)
87 for (i
= 0; i
< ARRAY_SIZE(ctx
->sample_cache
); i
++)
88 fd_hw_sample_reference(ctx
, &ctx
->sample_cache
[i
], NULL
);
92 is_active(struct fd_hw_query
*hq
, enum fd_render_stage stage
)
94 return !!(hq
->provider
->active
& stage
);
99 resume_query(struct fd_context
*ctx
, struct fd_hw_query
*hq
,
100 struct fd_ringbuffer
*ring
)
102 int idx
= pidx(hq
->provider
->query_type
);
103 assert(idx
>= 0); /* query never would have been created otherwise */
105 ctx
->active_providers
|= (1 << idx
);
106 hq
->period
= util_slab_alloc(&ctx
->sample_period_pool
);
107 list_inithead(&hq
->period
->list
);
108 hq
->period
->start
= get_sample(ctx
, ring
, hq
->base
.type
);
109 /* NOTE: util_slab_alloc() does not zero out the buffer: */
110 hq
->period
->end
= NULL
;
114 pause_query(struct fd_context
*ctx
, struct fd_hw_query
*hq
,
115 struct fd_ringbuffer
*ring
)
117 int idx
= pidx(hq
->provider
->query_type
);
118 assert(idx
>= 0); /* query never would have been created otherwise */
119 assert(hq
->period
&& !hq
->period
->end
);
120 assert(ctx
->active_providers
& (1 << idx
));
121 hq
->period
->end
= get_sample(ctx
, ring
, hq
->base
.type
);
122 list_addtail(&hq
->period
->list
, &hq
->current_periods
);
127 destroy_periods(struct fd_context
*ctx
, struct list_head
*list
)
129 struct fd_hw_sample_period
*period
, *s
;
130 LIST_FOR_EACH_ENTRY_SAFE(period
, s
, list
, list
) {
131 fd_hw_sample_reference(ctx
, &period
->start
, NULL
);
132 fd_hw_sample_reference(ctx
, &period
->end
, NULL
);
133 list_del(&period
->list
);
134 util_slab_free(&ctx
->sample_period_pool
, period
);
139 fd_hw_destroy_query(struct fd_context
*ctx
, struct fd_query
*q
)
141 struct fd_hw_query
*hq
= fd_hw_query(q
);
143 destroy_periods(ctx
, &hq
->periods
);
144 destroy_periods(ctx
, &hq
->current_periods
);
151 fd_hw_begin_query(struct fd_context
*ctx
, struct fd_query
*q
)
153 struct fd_hw_query
*hq
= fd_hw_query(q
);
157 /* begin_query() should clear previous results: */
158 destroy_periods(ctx
, &hq
->periods
);
160 if (is_active(hq
, ctx
->stage
))
161 resume_query(ctx
, hq
, ctx
->ring
);
165 /* add to active list: */
167 list_addtail(&hq
->list
, &ctx
->active_queries
);
172 fd_hw_end_query(struct fd_context
*ctx
, struct fd_query
*q
)
174 struct fd_hw_query
*hq
= fd_hw_query(q
);
175 /* there are a couple special cases, which don't have
176 * a matching ->begin_query():
178 if (skip_begin_query(q
->type
) && !q
->active
) {
179 fd_hw_begin_query(ctx
, q
);
183 if (is_active(hq
, ctx
->stage
))
184 pause_query(ctx
, hq
, ctx
->ring
);
186 /* move to current list: */
188 list_addtail(&hq
->list
, &ctx
->current_queries
);
191 /* helper to get ptr to specified sample: */
192 static void * sampptr(struct fd_hw_sample
*samp
, uint32_t n
, void *ptr
)
194 return ((char *)ptr
) + (samp
->tile_stride
* n
) + samp
->offset
;
198 fd_hw_get_query_result(struct fd_context
*ctx
, struct fd_query
*q
,
199 boolean wait
, union pipe_query_result
*result
)
201 struct fd_hw_query
*hq
= fd_hw_query(q
);
202 const struct fd_hw_sample_provider
*p
= hq
->provider
;
203 struct fd_hw_sample_period
*period
;
208 /* if the app tries to read back the query result before the
209 * batch is submitted, that forces us to flush so that there
210 * are actually results to wait for:
212 if (!LIST_IS_EMPTY(&hq
->list
)) {
213 /* if app didn't actually trigger any cmdstream, then
214 * we have nothing to do:
216 if (!ctx
->needs_flush
)
218 DBG("reading query result forces flush!");
219 fd_context_render(&ctx
->base
);
222 util_query_clear_result(result
, q
->type
);
224 if (LIST_IS_EMPTY(&hq
->periods
))
227 assert(LIST_IS_EMPTY(&hq
->list
));
228 assert(LIST_IS_EMPTY(&hq
->current_periods
));
231 /* if !wait, then check the last sample (the one most likely to
232 * not be ready yet) and bail if it is not ready:
237 period
= LIST_ENTRY(struct fd_hw_sample_period
,
238 hq
->periods
.prev
, list
);
240 ret
= fd_bo_cpu_prep(period
->end
->bo
, ctx
->screen
->pipe
,
241 DRM_FREEDRENO_PREP_READ
| DRM_FREEDRENO_PREP_NOSYNC
);
245 fd_bo_cpu_fini(period
->end
->bo
);
248 /* sum the result across all sample periods: */
249 LIST_FOR_EACH_ENTRY(period
, &hq
->periods
, list
) {
250 struct fd_hw_sample
*start
= period
->start
;
251 struct fd_hw_sample
*end
= period
->end
;
254 /* start and end samples should be from same batch: */
255 assert(start
->bo
== end
->bo
);
256 assert(start
->num_tiles
== end
->num_tiles
);
258 for (i
= 0; i
< start
->num_tiles
; i
++) {
261 fd_bo_cpu_prep(start
->bo
, ctx
->screen
->pipe
,
262 DRM_FREEDRENO_PREP_READ
);
264 ptr
= fd_bo_map(start
->bo
);
266 p
->accumulate_result(ctx
, sampptr(period
->start
, i
, ptr
),
267 sampptr(period
->end
, i
, ptr
), result
);
269 fd_bo_cpu_fini(start
->bo
);
276 static const struct fd_query_funcs hw_query_funcs
= {
277 .destroy_query
= fd_hw_destroy_query
,
278 .begin_query
= fd_hw_begin_query
,
279 .end_query
= fd_hw_end_query
,
280 .get_query_result
= fd_hw_get_query_result
,
284 fd_hw_create_query(struct fd_context
*ctx
, unsigned query_type
)
286 struct fd_hw_query
*hq
;
288 int idx
= pidx(query_type
);
290 if ((idx
< 0) || !ctx
->sample_providers
[idx
])
293 hq
= CALLOC_STRUCT(fd_hw_query
);
297 hq
->provider
= ctx
->sample_providers
[idx
];
299 list_inithead(&hq
->periods
);
300 list_inithead(&hq
->current_periods
);
301 list_inithead(&hq
->list
);
304 q
->funcs
= &hw_query_funcs
;
305 q
->type
= query_type
;
310 struct fd_hw_sample
*
311 fd_hw_sample_init(struct fd_context
*ctx
, uint32_t size
)
313 struct fd_hw_sample
*samp
= util_slab_alloc(&ctx
->sample_pool
);
314 pipe_reference_init(&samp
->reference
, 1);
316 debug_assert(util_is_power_of_two(size
));
317 ctx
->next_sample_offset
= align(ctx
->next_sample_offset
, size
);
318 samp
->offset
= ctx
->next_sample_offset
;
319 /* NOTE: util_slab_alloc() does not zero out the buffer: */
322 samp
->tile_stride
= 0;
323 ctx
->next_sample_offset
+= size
;
328 __fd_hw_sample_destroy(struct fd_context
*ctx
, struct fd_hw_sample
*samp
)
332 util_slab_free(&ctx
->sample_pool
, samp
);
336 prepare_sample(struct fd_hw_sample
*samp
, struct fd_bo
*bo
,
337 uint32_t num_tiles
, uint32_t tile_stride
)
340 assert(samp
->bo
== bo
);
341 assert(samp
->num_tiles
== num_tiles
);
342 assert(samp
->tile_stride
== tile_stride
);
345 samp
->bo
= fd_bo_ref(bo
);
346 samp
->num_tiles
= num_tiles
;
347 samp
->tile_stride
= tile_stride
;
351 prepare_query(struct fd_hw_query
*hq
, struct fd_bo
*bo
,
352 uint32_t num_tiles
, uint32_t tile_stride
)
354 struct fd_hw_sample_period
*period
, *s
;
356 /* prepare all the samples in the query: */
357 LIST_FOR_EACH_ENTRY_SAFE(period
, s
, &hq
->current_periods
, list
) {
358 prepare_sample(period
->start
, bo
, num_tiles
, tile_stride
);
359 prepare_sample(period
->end
, bo
, num_tiles
, tile_stride
);
361 /* move from current_periods list to periods list: */
362 list_del(&period
->list
);
363 list_addtail(&period
->list
, &hq
->periods
);
368 prepare_queries(struct fd_context
*ctx
, struct fd_bo
*bo
,
369 uint32_t num_tiles
, uint32_t tile_stride
,
370 struct list_head
*list
, bool remove
)
372 struct fd_hw_query
*hq
, *s
;
373 LIST_FOR_EACH_ENTRY_SAFE(hq
, s
, list
, list
) {
374 prepare_query(hq
, bo
, num_tiles
, tile_stride
);
376 list_delinit(&hq
->list
);
380 /* called from gmem code once total storage requirements are known (ie.
381 * number of samples times number of tiles)
384 fd_hw_query_prepare(struct fd_context
*ctx
, uint32_t num_tiles
)
386 uint32_t tile_stride
= ctx
->next_sample_offset
;
390 fd_bo_del(ctx
->query_bo
);
392 if (tile_stride
> 0) {
393 bo
= fd_bo_new(ctx
->dev
, tile_stride
* num_tiles
,
394 DRM_FREEDRENO_GEM_CACHE_WCOMBINE
|
395 DRM_FREEDRENO_GEM_TYPE_KMEM
);
401 ctx
->query_tile_stride
= tile_stride
;
403 prepare_queries(ctx
, bo
, num_tiles
, tile_stride
,
404 &ctx
->active_queries
, false);
405 prepare_queries(ctx
, bo
, num_tiles
, tile_stride
,
406 &ctx
->current_queries
, true);
408 /* reset things for next batch: */
409 ctx
->next_sample_offset
= 0;
413 fd_hw_query_prepare_tile(struct fd_context
*ctx
, uint32_t n
,
414 struct fd_ringbuffer
*ring
)
416 uint32_t tile_stride
= ctx
->query_tile_stride
;
417 uint32_t offset
= tile_stride
* n
;
419 /* bail if no queries: */
420 if (tile_stride
== 0)
424 OUT_PKT0 (ring
, HW_QUERY_BASE_REG
, 1);
425 OUT_RELOCW(ring
, ctx
->query_bo
, offset
, 0, 0);
429 fd_hw_query_set_stage(struct fd_context
*ctx
, struct fd_ringbuffer
*ring
,
430 enum fd_render_stage stage
)
432 /* special case: internal blits (like mipmap level generation)
433 * go through normal draw path (via util_blitter_blit()).. but
434 * we need to ignore the FD_STAGE_DRAW which will be set, so we
435 * don't enable queries which should be paused during internal
438 if ((ctx
->stage
== FD_STAGE_BLIT
) &&
439 (stage
!= FD_STAGE_NULL
))
442 if (stage
!= ctx
->stage
) {
443 struct fd_hw_query
*hq
;
444 LIST_FOR_EACH_ENTRY(hq
, &ctx
->active_queries
, list
) {
445 bool was_active
= is_active(hq
, ctx
->stage
);
446 bool now_active
= is_active(hq
, stage
);
448 if (now_active
&& !was_active
)
449 resume_query(ctx
, hq
, ring
);
450 else if (was_active
&& !now_active
)
451 pause_query(ctx
, hq
, ring
);
454 clear_sample_cache(ctx
);
458 /* call the provider->enable() for all the hw queries that were active
459 * in the current batch. This sets up perfctr selector regs statically
460 * for the duration of the batch.
463 fd_hw_query_enable(struct fd_context
*ctx
, struct fd_ringbuffer
*ring
)
465 for (int idx
= 0; idx
< MAX_HW_SAMPLE_PROVIDERS
; idx
++) {
466 if (ctx
->active_providers
& (1 << idx
)) {
467 assert(ctx
->sample_providers
[idx
]);
468 if (ctx
->sample_providers
[idx
]->enable
)
469 ctx
->sample_providers
[idx
]->enable(ctx
, ring
);
472 ctx
->active_providers
= 0; /* clear it for next frame */
476 fd_hw_query_register_provider(struct pipe_context
*pctx
,
477 const struct fd_hw_sample_provider
*provider
)
479 struct fd_context
*ctx
= fd_context(pctx
);
480 int idx
= pidx(provider
->query_type
);
482 assert((0 <= idx
) && (idx
< MAX_HW_SAMPLE_PROVIDERS
));
483 assert(!ctx
->sample_providers
[idx
]);
485 ctx
->sample_providers
[idx
] = provider
;
489 fd_hw_query_init(struct pipe_context
*pctx
)
491 struct fd_context
*ctx
= fd_context(pctx
);
493 util_slab_create(&ctx
->sample_pool
, sizeof(struct fd_hw_sample
),
494 16, UTIL_SLAB_SINGLETHREADED
);
495 util_slab_create(&ctx
->sample_period_pool
, sizeof(struct fd_hw_sample_period
),
496 16, UTIL_SLAB_SINGLETHREADED
);
497 list_inithead(&ctx
->active_queries
);
498 list_inithead(&ctx
->current_queries
);
502 fd_hw_query_fini(struct pipe_context
*pctx
)
504 struct fd_context
*ctx
= fd_context(pctx
);
506 util_slab_destroy(&ctx
->sample_pool
);
507 util_slab_destroy(&ctx
->sample_period_pool
);