1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
4 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * Rob Clark <robclark@freedesktop.org>
29 #include "pipe/p_state.h"
30 #include "util/u_memory.h"
31 #include "util/u_inlines.h"
33 #include "freedreno_query_hw.h"
34 #include "freedreno_context.h"
35 #include "freedreno_resource.h"
36 #include "freedreno_util.h"
38 struct fd_hw_sample_period
{
39 struct fd_hw_sample
*start
, *end
;
40 struct list_head list
;
43 /* maps query_type to sample provider idx: */
44 static int pidx(unsigned query_type
)
47 case PIPE_QUERY_OCCLUSION_COUNTER
:
49 case PIPE_QUERY_OCCLUSION_PREDICATE
:
51 /* TODO currently queries only emitted in main pass (not in binning pass)..
52 * which is fine for occlusion query, but pretty much not anything else.
54 case PIPE_QUERY_TIME_ELAPSED
:
56 case PIPE_QUERY_TIMESTAMP
:
63 static struct fd_hw_sample
*
64 get_sample(struct fd_batch
*batch
, struct fd_ringbuffer
*ring
,
67 struct fd_context
*ctx
= batch
->ctx
;
68 struct fd_hw_sample
*samp
= NULL
;
69 int idx
= pidx(query_type
);
71 assume(idx
>= 0); /* query never would have been created otherwise */
73 if (!batch
->sample_cache
[idx
]) {
74 struct fd_hw_sample
*new_samp
=
75 ctx
->sample_providers
[idx
]->get_sample(batch
, ring
);
76 fd_hw_sample_reference(ctx
, &batch
->sample_cache
[idx
], new_samp
);
77 util_dynarray_append(&batch
->samples
, struct fd_hw_sample
*, new_samp
);
78 batch
->needs_flush
= true;
81 fd_hw_sample_reference(ctx
, &samp
, batch
->sample_cache
[idx
]);
87 clear_sample_cache(struct fd_batch
*batch
)
91 for (i
= 0; i
< ARRAY_SIZE(batch
->sample_cache
); i
++)
92 fd_hw_sample_reference(batch
->ctx
, &batch
->sample_cache
[i
], NULL
);
96 is_active(struct fd_hw_query
*hq
, enum fd_render_stage stage
)
98 return !!(hq
->provider
->active
& stage
);
103 resume_query(struct fd_batch
*batch
, struct fd_hw_query
*hq
,
104 struct fd_ringbuffer
*ring
)
106 int idx
= pidx(hq
->provider
->query_type
);
108 assert(idx
>= 0); /* query never would have been created otherwise */
110 batch
->active_providers
|= (1 << idx
);
111 hq
->period
= slab_alloc_st(&batch
->ctx
->sample_period_pool
);
112 list_inithead(&hq
->period
->list
);
113 hq
->period
->start
= get_sample(batch
, ring
, hq
->base
.type
);
114 /* NOTE: slab_alloc_st() does not zero out the buffer: */
115 hq
->period
->end
= NULL
;
119 pause_query(struct fd_batch
*batch
, struct fd_hw_query
*hq
,
120 struct fd_ringbuffer
*ring
)
122 int idx
= pidx(hq
->provider
->query_type
);
124 assert(idx
>= 0); /* query never would have been created otherwise */
125 assert(hq
->period
&& !hq
->period
->end
);
126 assert(batch
->active_providers
& (1 << idx
));
127 hq
->period
->end
= get_sample(batch
, ring
, hq
->base
.type
);
128 list_addtail(&hq
->period
->list
, &hq
->periods
);
133 destroy_periods(struct fd_context
*ctx
, struct fd_hw_query
*hq
)
135 struct fd_hw_sample_period
*period
, *s
;
136 LIST_FOR_EACH_ENTRY_SAFE(period
, s
, &hq
->periods
, list
) {
137 fd_hw_sample_reference(ctx
, &period
->start
, NULL
);
138 fd_hw_sample_reference(ctx
, &period
->end
, NULL
);
139 list_del(&period
->list
);
140 slab_free_st(&ctx
->sample_period_pool
, period
);
145 fd_hw_destroy_query(struct fd_context
*ctx
, struct fd_query
*q
)
147 struct fd_hw_query
*hq
= fd_hw_query(q
);
149 DBG("%p: active=%d", q
, q
->active
);
151 destroy_periods(ctx
, hq
);
158 fd_hw_begin_query(struct fd_context
*ctx
, struct fd_query
*q
)
160 struct fd_batch
*batch
= ctx
->batch
;
161 struct fd_hw_query
*hq
= fd_hw_query(q
);
163 DBG("%p: active=%d", q
, q
->active
);
165 /* begin_query() should clear previous results: */
166 destroy_periods(ctx
, hq
);
168 if (batch
&& is_active(hq
, batch
->stage
))
169 resume_query(batch
, hq
, batch
->draw
);
171 /* add to active list: */
172 assert(list_empty(&hq
->list
));
173 list_addtail(&hq
->list
, &ctx
->active_queries
);
179 fd_hw_end_query(struct fd_context
*ctx
, struct fd_query
*q
)
181 struct fd_batch
*batch
= ctx
->batch
;
182 struct fd_hw_query
*hq
= fd_hw_query(q
);
184 DBG("%p: active=%d", q
, q
->active
);
186 if (batch
&& is_active(hq
, batch
->stage
))
187 pause_query(batch
, hq
, batch
->draw
);
189 /* remove from active list: */
190 list_delinit(&hq
->list
);
193 /* helper to get ptr to specified sample: */
194 static void * sampptr(struct fd_hw_sample
*samp
, uint32_t n
, void *ptr
)
196 return ((char *)ptr
) + (samp
->tile_stride
* n
) + samp
->offset
;
200 fd_hw_get_query_result(struct fd_context
*ctx
, struct fd_query
*q
,
201 boolean wait
, union pipe_query_result
*result
)
203 struct fd_hw_query
*hq
= fd_hw_query(q
);
204 const struct fd_hw_sample_provider
*p
= hq
->provider
;
205 struct fd_hw_sample_period
*period
;
207 DBG("%p: wait=%d, active=%d", q
, wait
, q
->active
);
209 if (LIST_IS_EMPTY(&hq
->periods
))
212 assert(LIST_IS_EMPTY(&hq
->list
));
215 /* if !wait, then check the last sample (the one most likely to
216 * not be ready yet) and bail if it is not ready:
221 period
= LIST_ENTRY(struct fd_hw_sample_period
,
222 hq
->periods
.prev
, list
);
224 struct fd_resource
*rsc
= fd_resource(period
->end
->prsc
);
226 if (pending(rsc
, false)) {
227 /* piglit spec@arb_occlusion_query@occlusion_query_conform
228 * test, and silly apps perhaps, get stuck in a loop trying
229 * to get query result forever with wait==false.. we don't
230 * wait to flush unnecessarily but we also don't want to
233 if (hq
->no_wait_cnt
++ > 5)
234 fd_batch_flush(rsc
->write_batch
, false);
241 ret
= fd_bo_cpu_prep(rsc
->bo
, ctx
->screen
->pipe
,
242 DRM_FREEDRENO_PREP_READ
| DRM_FREEDRENO_PREP_NOSYNC
);
246 fd_bo_cpu_fini(rsc
->bo
);
249 /* sum the result across all sample periods: */
250 LIST_FOR_EACH_ENTRY(period
, &hq
->periods
, list
) {
251 struct fd_hw_sample
*start
= period
->start
;
252 struct fd_hw_sample
*end
= period
->end
;
255 /* start and end samples should be from same batch: */
256 assert(start
->prsc
== end
->prsc
);
257 assert(start
->num_tiles
== end
->num_tiles
);
259 struct fd_resource
*rsc
= fd_resource(start
->prsc
);
261 if (rsc
->write_batch
)
262 fd_batch_flush(rsc
->write_batch
, true);
264 /* some piglit tests at least do query with no draws, I guess: */
268 fd_bo_cpu_prep(rsc
->bo
, ctx
->screen
->pipe
, DRM_FREEDRENO_PREP_READ
);
270 void *ptr
= fd_bo_map(rsc
->bo
);
272 for (i
= 0; i
< start
->num_tiles
; i
++) {
273 p
->accumulate_result(ctx
, sampptr(period
->start
, i
, ptr
),
274 sampptr(period
->end
, i
, ptr
), result
);
277 fd_bo_cpu_fini(rsc
->bo
);
283 static const struct fd_query_funcs hw_query_funcs
= {
284 .destroy_query
= fd_hw_destroy_query
,
285 .begin_query
= fd_hw_begin_query
,
286 .end_query
= fd_hw_end_query
,
287 .get_query_result
= fd_hw_get_query_result
,
291 fd_hw_create_query(struct fd_context
*ctx
, unsigned query_type
)
293 struct fd_hw_query
*hq
;
295 int idx
= pidx(query_type
);
297 if ((idx
< 0) || !ctx
->sample_providers
[idx
])
300 hq
= CALLOC_STRUCT(fd_hw_query
);
304 DBG("%p: query_type=%u", hq
, query_type
);
306 hq
->provider
= ctx
->sample_providers
[idx
];
308 list_inithead(&hq
->periods
);
309 list_inithead(&hq
->list
);
312 q
->funcs
= &hw_query_funcs
;
313 q
->type
= query_type
;
318 struct fd_hw_sample
*
319 fd_hw_sample_init(struct fd_batch
*batch
, uint32_t size
)
321 struct fd_hw_sample
*samp
= slab_alloc_st(&batch
->ctx
->sample_pool
);
322 pipe_reference_init(&samp
->reference
, 1);
324 debug_assert(util_is_power_of_two(size
));
325 batch
->next_sample_offset
= align(batch
->next_sample_offset
, size
);
326 samp
->offset
= batch
->next_sample_offset
;
327 /* NOTE: slab_alloc_st() does not zero out the buffer: */
330 samp
->tile_stride
= 0;
331 batch
->next_sample_offset
+= size
;
333 if (!batch
->query_buf
) {
334 struct pipe_screen
*pscreen
= &batch
->ctx
->screen
->base
;
335 struct pipe_resource templ
= {
336 .target
= PIPE_BUFFER
,
337 .format
= PIPE_FORMAT_R8_UNORM
,
338 .bind
= PIPE_BIND_QUERY_BUFFER
,
339 .width0
= 0, /* create initially zero size buffer */
346 batch
->query_buf
= pscreen
->resource_create(pscreen
, &templ
);
349 pipe_resource_reference(&samp
->prsc
, batch
->query_buf
);
355 __fd_hw_sample_destroy(struct fd_context
*ctx
, struct fd_hw_sample
*samp
)
357 pipe_resource_reference(&samp
->prsc
, NULL
);
358 slab_free_st(&ctx
->sample_pool
, samp
);
361 /* called from gmem code once total storage requirements are known (ie.
362 * number of samples times number of tiles)
365 fd_hw_query_prepare(struct fd_batch
*batch
, uint32_t num_tiles
)
367 uint32_t tile_stride
= batch
->next_sample_offset
;
370 fd_resource_resize(batch
->query_buf
, tile_stride
* num_tiles
);
372 batch
->query_tile_stride
= tile_stride
;
374 while (batch
->samples
.size
> 0) {
375 struct fd_hw_sample
*samp
=
376 util_dynarray_pop(&batch
->samples
, struct fd_hw_sample
*);
377 samp
->num_tiles
= num_tiles
;
378 samp
->tile_stride
= tile_stride
;
379 fd_hw_sample_reference(batch
->ctx
, &samp
, NULL
);
382 /* reset things for next batch: */
383 batch
->next_sample_offset
= 0;
387 fd_hw_query_prepare_tile(struct fd_batch
*batch
, uint32_t n
,
388 struct fd_ringbuffer
*ring
)
390 uint32_t tile_stride
= batch
->query_tile_stride
;
391 uint32_t offset
= tile_stride
* n
;
393 /* bail if no queries: */
394 if (tile_stride
== 0)
398 OUT_PKT0 (ring
, HW_QUERY_BASE_REG
, 1);
399 OUT_RELOCW(ring
, fd_resource(batch
->query_buf
)->bo
, offset
, 0, 0);
403 fd_hw_query_set_stage(struct fd_batch
*batch
, struct fd_ringbuffer
*ring
,
404 enum fd_render_stage stage
)
406 if (stage
!= batch
->stage
) {
407 struct fd_hw_query
*hq
;
408 LIST_FOR_EACH_ENTRY(hq
, &batch
->ctx
->active_queries
, list
) {
409 bool was_active
= is_active(hq
, batch
->stage
);
410 bool now_active
= is_active(hq
, stage
);
412 if (now_active
&& !was_active
)
413 resume_query(batch
, hq
, ring
);
414 else if (was_active
&& !now_active
)
415 pause_query(batch
, hq
, ring
);
418 clear_sample_cache(batch
);
421 /* call the provider->enable() for all the hw queries that were active
422 * in the current batch. This sets up perfctr selector regs statically
423 * for the duration of the batch.
426 fd_hw_query_enable(struct fd_batch
*batch
, struct fd_ringbuffer
*ring
)
428 struct fd_context
*ctx
= batch
->ctx
;
429 for (int idx
= 0; idx
< MAX_HW_SAMPLE_PROVIDERS
; idx
++) {
430 if (batch
->active_providers
& (1 << idx
)) {
431 assert(ctx
->sample_providers
[idx
]);
432 if (ctx
->sample_providers
[idx
]->enable
)
433 ctx
->sample_providers
[idx
]->enable(ctx
, ring
);
436 batch
->active_providers
= 0; /* clear it for next frame */
440 fd_hw_query_register_provider(struct pipe_context
*pctx
,
441 const struct fd_hw_sample_provider
*provider
)
443 struct fd_context
*ctx
= fd_context(pctx
);
444 int idx
= pidx(provider
->query_type
);
446 assert((0 <= idx
) && (idx
< MAX_HW_SAMPLE_PROVIDERS
));
447 assert(!ctx
->sample_providers
[idx
]);
449 ctx
->sample_providers
[idx
] = provider
;
453 fd_hw_query_init(struct pipe_context
*pctx
)
455 struct fd_context
*ctx
= fd_context(pctx
);
457 slab_create(&ctx
->sample_pool
, sizeof(struct fd_hw_sample
),
459 slab_create(&ctx
->sample_period_pool
, sizeof(struct fd_hw_sample_period
),
461 list_inithead(&ctx
->active_queries
);
465 fd_hw_query_fini(struct pipe_context
*pctx
)
467 struct fd_context
*ctx
= fd_context(pctx
);
469 slab_destroy(&ctx
->sample_pool
);
470 slab_destroy(&ctx
->sample_period_pool
);