2 * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * Rob Clark <robclark@freedesktop.org>
27 #include "util/list.h"
29 #include "util/hash_table.h"
30 #include "util/u_string.h"
32 #include "freedreno_batch.h"
33 #include "freedreno_context.h"
34 #include "freedreno_fence.h"
35 #include "freedreno_resource.h"
36 #include "freedreno_query_hw.h"
39 batch_init(struct fd_batch
*batch
)
41 struct fd_context
*ctx
= batch
->ctx
;
44 if (ctx
->screen
->reorder
)
45 util_queue_fence_init(&batch
->flush_fence
);
47 /* if kernel is too old to support unlimited # of cmd buffers, we
48 * have no option but to allocate large worst-case sizes so that
49 * we don't need to grow the ringbuffer. Performance is likely to
50 * suffer, but there is no good alternative.
52 * XXX I think we can just require new enough kernel for this?
54 if ((fd_device_version(ctx
->screen
->dev
) < FD_VERSION_UNLIMITED_CMDS
) ||
55 (fd_mesa_debug
& FD_DBG_NOGROW
)){
59 batch
->submit
= fd_submit_new(ctx
->pipe
);
61 batch
->draw
= fd_submit_new_ringbuffer(batch
->submit
, size
,
62 FD_RINGBUFFER_PRIMARY
| FD_RINGBUFFER_GROWABLE
);
64 batch
->gmem
= fd_submit_new_ringbuffer(batch
->submit
, size
,
65 FD_RINGBUFFER_PRIMARY
| FD_RINGBUFFER_GROWABLE
);
66 batch
->draw
= fd_submit_new_ringbuffer(batch
->submit
, size
,
67 FD_RINGBUFFER_GROWABLE
);
69 if (ctx
->screen
->gpu_id
< 600) {
70 batch
->binning
= fd_submit_new_ringbuffer(batch
->submit
,
71 size
, FD_RINGBUFFER_GROWABLE
);
75 batch
->in_fence_fd
= -1;
76 batch
->fence
= fd_fence_create(batch
);
79 batch
->invalidated
= 0;
80 batch
->restore
= batch
->resolve
= 0;
81 batch
->needs_flush
= false;
82 batch
->flushed
= false;
83 batch
->gmem_reason
= 0;
85 batch
->stage
= FD_STAGE_NULL
;
89 util_dynarray_init(&batch
->draw_patches
, NULL
);
91 if (is_a3xx(ctx
->screen
))
92 util_dynarray_init(&batch
->rbrc_patches
, NULL
);
94 util_dynarray_init(&batch
->gmem_patches
, NULL
);
96 assert(batch
->resources
->entries
== 0);
98 util_dynarray_init(&batch
->samples
, NULL
);
102 fd_batch_create(struct fd_context
*ctx
, bool nondraw
)
104 struct fd_batch
*batch
= CALLOC_STRUCT(fd_batch
);
111 pipe_reference_init(&batch
->reference
, 1);
113 batch
->nondraw
= nondraw
;
115 batch
->resources
= _mesa_set_create(NULL
, _mesa_hash_pointer
,
116 _mesa_key_pointer_equal
);
124 batch_fini(struct fd_batch
*batch
)
128 pipe_resource_reference(&batch
->query_buf
, NULL
);
130 if (batch
->in_fence_fd
!= -1)
131 close(batch
->in_fence_fd
);
133 /* in case batch wasn't flushed but fence was created: */
134 fd_fence_populate(batch
->fence
, 0, -1);
136 fd_fence_ref(NULL
, &batch
->fence
, NULL
);
138 fd_ringbuffer_del(batch
->draw
);
139 if (!batch
->nondraw
) {
141 fd_ringbuffer_del(batch
->binning
);
142 fd_ringbuffer_del(batch
->gmem
);
144 debug_assert(!batch
->binning
);
145 debug_assert(!batch
->gmem
);
147 if (batch
->lrz_clear
) {
148 fd_ringbuffer_del(batch
->lrz_clear
);
149 batch
->lrz_clear
= NULL
;
152 fd_submit_del(batch
->submit
);
154 util_dynarray_fini(&batch
->draw_patches
);
156 if (is_a3xx(batch
->ctx
->screen
))
157 util_dynarray_fini(&batch
->rbrc_patches
);
159 util_dynarray_fini(&batch
->gmem_patches
);
161 while (batch
->samples
.size
> 0) {
162 struct fd_hw_sample
*samp
=
163 util_dynarray_pop(&batch
->samples
, struct fd_hw_sample
*);
164 fd_hw_sample_reference(batch
->ctx
, &samp
, NULL
);
166 util_dynarray_fini(&batch
->samples
);
168 if (batch
->ctx
->screen
->reorder
)
169 util_queue_fence_destroy(&batch
->flush_fence
);
173 batch_flush_reset_dependencies(struct fd_batch
*batch
, bool flush
)
175 struct fd_batch_cache
*cache
= &batch
->ctx
->screen
->batch_cache
;
176 struct fd_batch
*dep
;
178 foreach_batch(dep
, cache
, batch
->dependents_mask
) {
180 fd_batch_flush(dep
, false, false);
181 fd_batch_reference(&dep
, NULL
);
184 batch
->dependents_mask
= 0;
188 batch_reset_resources_locked(struct fd_batch
*batch
)
190 pipe_mutex_assert_locked(batch
->ctx
->screen
->lock
);
192 set_foreach(batch
->resources
, entry
) {
193 struct fd_resource
*rsc
= (struct fd_resource
*)entry
->key
;
194 _mesa_set_remove(batch
->resources
, entry
);
195 debug_assert(rsc
->batch_mask
& (1 << batch
->idx
));
196 rsc
->batch_mask
&= ~(1 << batch
->idx
);
197 if (rsc
->write_batch
== batch
)
198 fd_batch_reference_locked(&rsc
->write_batch
, NULL
);
203 batch_reset_resources(struct fd_batch
*batch
)
205 mtx_lock(&batch
->ctx
->screen
->lock
);
206 batch_reset_resources_locked(batch
);
207 mtx_unlock(&batch
->ctx
->screen
->lock
);
211 batch_reset(struct fd_batch
*batch
)
215 fd_batch_sync(batch
);
217 batch_flush_reset_dependencies(batch
, false);
218 batch_reset_resources(batch
);
225 fd_batch_reset(struct fd_batch
*batch
)
227 if (batch
->needs_flush
)
232 __fd_batch_destroy(struct fd_batch
*batch
)
234 struct fd_context
*ctx
= batch
->ctx
;
238 fd_context_assert_locked(batch
->ctx
);
240 fd_bc_invalidate_batch(batch
, true);
242 batch_reset_resources_locked(batch
);
243 debug_assert(batch
->resources
->entries
== 0);
244 _mesa_set_destroy(batch
->resources
, NULL
);
246 fd_context_unlock(ctx
);
247 batch_flush_reset_dependencies(batch
, false);
248 debug_assert(batch
->dependents_mask
== 0);
250 util_copy_framebuffer_state(&batch
->framebuffer
, NULL
);
253 fd_context_lock(ctx
);
257 __fd_batch_describe(char* buf
, const struct fd_batch
*batch
)
259 util_sprintf(buf
, "fd_batch<%u>", batch
->seqno
);
263 fd_batch_sync(struct fd_batch
*batch
)
265 if (!batch
->ctx
->screen
->reorder
)
267 util_queue_fence_wait(&batch
->flush_fence
);
271 batch_flush_func(void *job
, int id
)
273 struct fd_batch
*batch
= job
;
277 fd_gmem_render_tiles(batch
);
278 batch_reset_resources(batch
);
282 batch_cleanup_func(void *job
, int id
)
284 struct fd_batch
*batch
= job
;
285 fd_batch_reference(&batch
, NULL
);
289 batch_flush(struct fd_batch
*batch
, bool force
)
291 DBG("%p: needs_flush=%d", batch
, batch
->needs_flush
);
296 batch
->needs_flush
= false;
298 /* close out the draw cmds by making sure any active queries are
301 fd_batch_set_stage(batch
, FD_STAGE_NULL
);
303 batch_flush_reset_dependencies(batch
, true);
305 batch
->flushed
= true;
307 if (batch
->ctx
->screen
->reorder
) {
308 struct fd_batch
*tmp
= NULL
;
309 fd_batch_reference(&tmp
, batch
);
311 if (!util_queue_is_initialized(&batch
->ctx
->flush_queue
))
312 util_queue_init(&batch
->ctx
->flush_queue
, "flush_queue", 16, 1, 0);
314 util_queue_add_job(&batch
->ctx
->flush_queue
,
315 batch
, &batch
->flush_fence
,
316 batch_flush_func
, batch_cleanup_func
);
318 fd_gmem_render_tiles(batch
);
319 batch_reset_resources(batch
);
322 debug_assert(batch
->reference
.count
> 0);
324 mtx_lock(&batch
->ctx
->screen
->lock
);
325 fd_bc_invalidate_batch(batch
, false);
326 mtx_unlock(&batch
->ctx
->screen
->lock
);
329 /* NOTE: could drop the last ref to batch
331 * @sync: synchronize with flush_queue, ensures batch is *actually* flushed
332 * to kernel before this returns, as opposed to just being queued to be
334 * @force: force a flush even if no rendering, mostly useful if you need
338 fd_batch_flush(struct fd_batch
*batch
, bool sync
, bool force
)
340 struct fd_batch
*tmp
= NULL
;
341 bool newbatch
= false;
343 /* NOTE: we need to hold an extra ref across the body of flush,
344 * since the last ref to this batch could be dropped when cleaning
347 fd_batch_reference(&tmp
, batch
);
349 if (batch
== batch
->ctx
->batch
) {
350 batch
->ctx
->batch
= NULL
;
354 batch_flush(tmp
, force
);
357 struct fd_context
*ctx
= batch
->ctx
;
358 struct fd_batch
*new_batch
;
360 if (ctx
->screen
->reorder
) {
361 /* defer allocating new batch until one is needed for rendering
362 * to avoid unused batches for apps that create many contexts
366 new_batch
= fd_bc_alloc_batch(&ctx
->screen
->batch_cache
, ctx
, false);
367 util_copy_framebuffer_state(&new_batch
->framebuffer
, &batch
->framebuffer
);
370 fd_batch_reference(&batch
, NULL
);
371 ctx
->batch
= new_batch
;
372 fd_context_all_dirty(ctx
);
378 fd_batch_reference(&tmp
, NULL
);
381 /* does 'batch' depend directly or indirectly on 'other' ? */
383 batch_depends_on(struct fd_batch
*batch
, struct fd_batch
*other
)
385 struct fd_batch_cache
*cache
= &batch
->ctx
->screen
->batch_cache
;
386 struct fd_batch
*dep
;
388 if (batch
->dependents_mask
& (1 << other
->idx
))
391 foreach_batch(dep
, cache
, batch
->dependents_mask
)
392 if (batch_depends_on(batch
, dep
))
399 fd_batch_add_dep(struct fd_batch
*batch
, struct fd_batch
*dep
)
401 if (batch
->dependents_mask
& (1 << dep
->idx
))
404 /* a loop should not be possible */
405 debug_assert(!batch_depends_on(dep
, batch
));
407 struct fd_batch
*other
= NULL
;
408 fd_batch_reference_locked(&other
, dep
);
409 batch
->dependents_mask
|= (1 << dep
->idx
);
410 DBG("%p: added dependency on %p", batch
, dep
);
414 flush_write_batch(struct fd_resource
*rsc
)
416 struct fd_batch
*b
= NULL
;
417 fd_batch_reference(&b
, rsc
->write_batch
);
419 mtx_unlock(&b
->ctx
->screen
->lock
);
420 fd_batch_flush(b
, true, false);
421 mtx_lock(&b
->ctx
->screen
->lock
);
423 fd_bc_invalidate_batch(b
, false);
424 fd_batch_reference_locked(&b
, NULL
);
428 fd_batch_resource_used(struct fd_batch
*batch
, struct fd_resource
*rsc
, bool write
)
430 pipe_mutex_assert_locked(batch
->ctx
->screen
->lock
);
433 fd_batch_resource_used(batch
, rsc
->stencil
, write
);
435 DBG("%p: %s %p", batch
, write
? "write" : "read", rsc
);
440 /* note, invalidate write batch, to avoid further writes to rsc
441 * resulting in a write-after-read hazard.
445 /* if we are pending read or write by any other batch: */
446 if (rsc
->batch_mask
& ~(1 << batch
->idx
)) {
447 struct fd_batch_cache
*cache
= &batch
->ctx
->screen
->batch_cache
;
448 struct fd_batch
*dep
;
450 if (rsc
->write_batch
&& rsc
->write_batch
!= batch
)
451 flush_write_batch(rsc
);
453 foreach_batch(dep
, cache
, rsc
->batch_mask
) {
454 struct fd_batch
*b
= NULL
;
457 /* note that batch_add_dep could flush and unref dep, so
458 * we need to hold a reference to keep it live for the
459 * fd_bc_invalidate_batch()
461 fd_batch_reference(&b
, dep
);
462 fd_batch_add_dep(batch
, b
);
463 fd_bc_invalidate_batch(b
, false);
464 fd_batch_reference_locked(&b
, NULL
);
467 fd_batch_reference_locked(&rsc
->write_batch
, batch
);
469 /* If reading a resource pending a write, go ahead and flush the
470 * writer. This avoids situations where we end up having to
471 * flush the current batch in _resource_used()
473 if (rsc
->write_batch
&& rsc
->write_batch
!= batch
)
474 flush_write_batch(rsc
);
477 if (rsc
->batch_mask
& (1 << batch
->idx
)) {
478 debug_assert(_mesa_set_search(batch
->resources
, rsc
));
482 debug_assert(!_mesa_set_search(batch
->resources
, rsc
));
484 _mesa_set_add(batch
->resources
, rsc
);
485 rsc
->batch_mask
|= (1 << batch
->idx
);
489 fd_batch_check_size(struct fd_batch
*batch
)
491 debug_assert(!batch
->flushed
);
493 if (unlikely(fd_mesa_debug
& FD_DBG_FLUSH
)) {
494 fd_batch_flush(batch
, true, false);
498 if (fd_device_version(batch
->ctx
->screen
->dev
) >= FD_VERSION_UNLIMITED_CMDS
)
501 struct fd_ringbuffer
*ring
= batch
->draw
;
502 if ((ring
->cur
- ring
->start
) > (ring
->size
/4 - 0x1000))
503 fd_batch_flush(batch
, true, false);
506 /* emit a WAIT_FOR_IDLE only if needed, ie. if there has not already
507 * been one since last draw:
510 fd_wfi(struct fd_batch
*batch
, struct fd_ringbuffer
*ring
)
512 if (batch
->needs_wfi
) {
513 if (batch
->ctx
->screen
->gpu_id
>= 500)
517 batch
->needs_wfi
= false;