2 * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * Rob Clark <robclark@freedesktop.org>
27 #include "util/list.h"
29 #include "util/hash_table.h"
30 #include "util/u_string.h"
32 #include "freedreno_batch.h"
33 #include "freedreno_context.h"
34 #include "freedreno_fence.h"
35 #include "freedreno_resource.h"
36 #include "freedreno_query_hw.h"
38 static struct fd_ringbuffer
*
39 alloc_ring(struct fd_batch
*batch
, unsigned sz
, enum fd_ringbuffer_flags flags
)
41 struct fd_context
*ctx
= batch
->ctx
;
43 /* if kernel is too old to support unlimited # of cmd buffers, we
44 * have no option but to allocate large worst-case sizes so that
45 * we don't need to grow the ringbuffer. Performance is likely to
46 * suffer, but there is no good alternative.
48 * Otherwise if supported, allocate a growable ring with initial
51 if ((fd_device_version(ctx
->screen
->dev
) >= FD_VERSION_UNLIMITED_CMDS
) &&
52 !(fd_mesa_debug
& FD_DBG_NOGROW
)){
53 flags
|= FD_RINGBUFFER_GROWABLE
;
57 return fd_submit_new_ringbuffer(batch
->submit
, sz
, flags
);
61 batch_init(struct fd_batch
*batch
)
63 struct fd_context
*ctx
= batch
->ctx
;
65 batch
->submit
= fd_submit_new(ctx
->pipe
);
67 batch
->gmem
= alloc_ring(batch
, 0x1000, FD_RINGBUFFER_PRIMARY
);
68 batch
->draw
= alloc_ring(batch
, 0x100000, 0);
70 batch
->gmem
= alloc_ring(batch
, 0x100000, FD_RINGBUFFER_PRIMARY
);
71 batch
->draw
= alloc_ring(batch
, 0x100000, 0);
73 /* a6xx+ re-uses draw rb for both draw and binning pass: */
74 if (ctx
->screen
->gpu_id
< 600) {
75 batch
->binning
= alloc_ring(batch
, 0x100000, 0);
79 batch
->in_fence_fd
= -1;
80 batch
->fence
= fd_fence_create(batch
);
83 batch
->fast_cleared
= 0;
84 batch
->invalidated
= 0;
85 batch
->restore
= batch
->resolve
= 0;
86 batch
->needs_flush
= false;
87 batch
->flushed
= false;
88 batch
->gmem_reason
= 0;
90 batch
->num_vertices
= 0;
91 batch
->num_bins_per_pipe
= 0;
92 batch
->prim_strm_bits
= 0;
93 batch
->draw_strm_bits
= 0;
94 batch
->stage
= FD_STAGE_NULL
;
98 util_dynarray_init(&batch
->draw_patches
, NULL
);
99 util_dynarray_init(&batch
->fb_read_patches
, NULL
);
101 if (is_a2xx(ctx
->screen
)) {
102 util_dynarray_init(&batch
->shader_patches
, NULL
);
103 util_dynarray_init(&batch
->gmem_patches
, NULL
);
106 if (is_a3xx(ctx
->screen
))
107 util_dynarray_init(&batch
->rbrc_patches
, NULL
);
109 assert(batch
->resources
->entries
== 0);
111 util_dynarray_init(&batch
->samples
, NULL
);
113 list_inithead(&batch
->log_chunks
);
117 fd_batch_create(struct fd_context
*ctx
, bool nondraw
)
119 struct fd_batch
*batch
= CALLOC_STRUCT(fd_batch
);
126 pipe_reference_init(&batch
->reference
, 1);
128 batch
->nondraw
= nondraw
;
130 batch
->resources
= _mesa_set_create(NULL
, _mesa_hash_pointer
,
131 _mesa_key_pointer_equal
);
135 fd_screen_assert_locked(ctx
->screen
);
137 _mesa_set_add(ctx
->screen
->live_batches
, batch
);
144 batch_fini(struct fd_batch
*batch
)
148 pipe_resource_reference(&batch
->query_buf
, NULL
);
150 if (batch
->in_fence_fd
!= -1)
151 close(batch
->in_fence_fd
);
153 /* in case batch wasn't flushed but fence was created: */
154 fd_fence_populate(batch
->fence
, 0, -1);
156 fd_fence_ref(&batch
->fence
, NULL
);
158 fd_ringbuffer_del(batch
->draw
);
159 fd_ringbuffer_del(batch
->gmem
);
161 if (batch
->binning
) {
162 fd_ringbuffer_del(batch
->binning
);
163 batch
->binning
= NULL
;
166 if (batch
->prologue
) {
167 fd_ringbuffer_del(batch
->prologue
);
168 batch
->prologue
= NULL
;
171 if (batch
->epilogue
) {
172 fd_ringbuffer_del(batch
->epilogue
);
173 batch
->epilogue
= NULL
;
176 if (batch
->tile_setup
) {
177 fd_ringbuffer_del(batch
->tile_setup
);
178 batch
->tile_setup
= NULL
;
181 if (batch
->tile_fini
) {
182 fd_ringbuffer_del(batch
->tile_fini
);
183 batch
->tile_fini
= NULL
;
186 if (batch
->tessellation
) {
187 fd_bo_del(batch
->tessfactor_bo
);
188 fd_bo_del(batch
->tessparam_bo
);
189 fd_ringbuffer_del(batch
->tess_addrs_constobj
);
192 fd_submit_del(batch
->submit
);
194 util_dynarray_fini(&batch
->draw_patches
);
195 util_dynarray_fini(&batch
->fb_read_patches
);
197 if (is_a2xx(batch
->ctx
->screen
)) {
198 util_dynarray_fini(&batch
->shader_patches
);
199 util_dynarray_fini(&batch
->gmem_patches
);
202 if (is_a3xx(batch
->ctx
->screen
))
203 util_dynarray_fini(&batch
->rbrc_patches
);
205 while (batch
->samples
.size
> 0) {
206 struct fd_hw_sample
*samp
=
207 util_dynarray_pop(&batch
->samples
, struct fd_hw_sample
*);
208 fd_hw_sample_reference(batch
->ctx
, &samp
, NULL
);
210 util_dynarray_fini(&batch
->samples
);
212 assert(list_is_empty(&batch
->log_chunks
));
216 batch_flush_reset_dependencies(struct fd_batch
*batch
, bool flush
)
218 struct fd_batch_cache
*cache
= &batch
->ctx
->screen
->batch_cache
;
219 struct fd_batch
*dep
;
221 foreach_batch(dep
, cache
, batch
->dependents_mask
) {
224 fd_batch_reference(&dep
, NULL
);
227 batch
->dependents_mask
= 0;
231 batch_reset_resources_locked(struct fd_batch
*batch
)
233 fd_screen_assert_locked(batch
->ctx
->screen
);
235 set_foreach(batch
->resources
, entry
) {
236 struct fd_resource
*rsc
= (struct fd_resource
*)entry
->key
;
237 _mesa_set_remove(batch
->resources
, entry
);
238 debug_assert(rsc
->batch_mask
& (1 << batch
->idx
));
239 rsc
->batch_mask
&= ~(1 << batch
->idx
);
240 if (rsc
->write_batch
== batch
)
241 fd_batch_reference_locked(&rsc
->write_batch
, NULL
);
246 batch_reset_resources(struct fd_batch
*batch
)
248 fd_screen_lock(batch
->ctx
->screen
);
249 batch_reset_resources_locked(batch
);
250 fd_screen_unlock(batch
->ctx
->screen
);
254 batch_reset(struct fd_batch
*batch
)
258 batch_flush_reset_dependencies(batch
, false);
259 batch_reset_resources(batch
);
266 fd_batch_reset(struct fd_batch
*batch
)
268 if (batch
->needs_flush
)
273 __fd_batch_destroy(struct fd_batch
*batch
)
275 struct fd_context
*ctx
= batch
->ctx
;
279 fd_context_assert_locked(batch
->ctx
);
282 _mesa_set_remove_key(ctx
->screen
->live_batches
, batch
);
285 fd_bc_invalidate_batch(batch
, true);
287 batch_reset_resources_locked(batch
);
288 debug_assert(batch
->resources
->entries
== 0);
289 _mesa_set_destroy(batch
->resources
, NULL
);
291 fd_context_unlock(ctx
);
292 batch_flush_reset_dependencies(batch
, false);
293 debug_assert(batch
->dependents_mask
== 0);
295 util_copy_framebuffer_state(&batch
->framebuffer
, NULL
);
298 fd_context_lock(ctx
);
302 __fd_batch_describe(char* buf
, const struct fd_batch
*batch
)
304 sprintf(buf
, "fd_batch<%u>", batch
->seqno
);
308 batch_flush(struct fd_batch
*batch
)
310 DBG("%p: needs_flush=%d", batch
, batch
->needs_flush
);
315 batch
->needs_flush
= false;
317 /* close out the draw cmds by making sure any active queries are
320 fd_batch_set_stage(batch
, FD_STAGE_NULL
);
322 batch_flush_reset_dependencies(batch
, true);
324 batch
->flushed
= true;
326 fd_fence_ref(&batch
->ctx
->last_fence
, batch
->fence
);
328 fd_gmem_render_tiles(batch
);
329 batch_reset_resources(batch
);
331 debug_assert(batch
->reference
.count
> 0);
333 fd_screen_lock(batch
->ctx
->screen
);
334 fd_bc_invalidate_batch(batch
, false);
335 fd_screen_unlock(batch
->ctx
->screen
);
338 /* Get per-batch prologue */
339 struct fd_ringbuffer
*
340 fd_batch_get_prologue(struct fd_batch
*batch
)
342 if (!batch
->prologue
)
343 batch
->prologue
= alloc_ring(batch
, 0x1000, 0);
344 return batch
->prologue
;
347 /* NOTE: could drop the last ref to batch
349 * @sync: synchronize with flush_queue, ensures batch is *actually* flushed
350 * to kernel before this returns, as opposed to just being queued to be
352 * @force: force a flush even if no rendering, mostly useful if you need
356 fd_batch_flush(struct fd_batch
*batch
)
358 struct fd_batch
*tmp
= NULL
;
360 /* NOTE: we need to hold an extra ref across the body of flush,
361 * since the last ref to this batch could be dropped when cleaning
364 fd_batch_reference(&tmp
, batch
);
368 if (batch
== batch
->ctx
->batch
) {
369 fd_batch_reference(&batch
->ctx
->batch
, NULL
);
372 fd_batch_reference(&tmp
, NULL
);
375 /* find a batches dependents mask, including recursive dependencies: */
377 recursive_dependents_mask(struct fd_batch
*batch
)
379 struct fd_batch_cache
*cache
= &batch
->ctx
->screen
->batch_cache
;
380 struct fd_batch
*dep
;
381 uint32_t dependents_mask
= batch
->dependents_mask
;
383 foreach_batch(dep
, cache
, batch
->dependents_mask
)
384 dependents_mask
|= recursive_dependents_mask(dep
);
386 return dependents_mask
;
390 fd_batch_add_dep(struct fd_batch
*batch
, struct fd_batch
*dep
)
392 fd_screen_assert_locked(batch
->ctx
->screen
);
394 if (batch
->dependents_mask
& (1 << dep
->idx
))
397 /* a loop should not be possible */
398 debug_assert(!((1 << batch
->idx
) & recursive_dependents_mask(dep
)));
400 struct fd_batch
*other
= NULL
;
401 fd_batch_reference_locked(&other
, dep
);
402 batch
->dependents_mask
|= (1 << dep
->idx
);
403 DBG("%p: added dependency on %p", batch
, dep
);
407 flush_write_batch(struct fd_resource
*rsc
)
409 struct fd_batch
*b
= NULL
;
410 fd_batch_reference_locked(&b
, rsc
->write_batch
);
412 fd_screen_unlock(b
->ctx
->screen
);
414 fd_screen_lock(b
->ctx
->screen
);
416 fd_bc_invalidate_batch(b
, false);
417 fd_batch_reference_locked(&b
, NULL
);
421 fd_batch_add_resource(struct fd_batch
*batch
, struct fd_resource
*rsc
)
424 if (likely(fd_batch_references_resource(batch
, rsc
))) {
425 debug_assert(_mesa_set_search(batch
->resources
, rsc
));
429 debug_assert(!_mesa_set_search(batch
->resources
, rsc
));
431 _mesa_set_add(batch
->resources
, rsc
);
432 rsc
->batch_mask
|= (1 << batch
->idx
);
436 fd_batch_resource_write(struct fd_batch
*batch
, struct fd_resource
*rsc
)
438 fd_screen_assert_locked(batch
->ctx
->screen
);
440 fd_batch_write_prep(batch
, rsc
);
443 fd_batch_resource_write(batch
, rsc
->stencil
);
445 DBG("%p: write %p", batch
, rsc
);
449 /* note, invalidate write batch, to avoid further writes to rsc
450 * resulting in a write-after-read hazard.
452 /* if we are pending read or write by any other batch: */
453 if (unlikely(rsc
->batch_mask
& ~(1 << batch
->idx
))) {
454 struct fd_batch_cache
*cache
= &batch
->ctx
->screen
->batch_cache
;
455 struct fd_batch
*dep
;
457 if (rsc
->write_batch
&& rsc
->write_batch
!= batch
)
458 flush_write_batch(rsc
);
460 foreach_batch(dep
, cache
, rsc
->batch_mask
) {
461 struct fd_batch
*b
= NULL
;
464 /* note that batch_add_dep could flush and unref dep, so
465 * we need to hold a reference to keep it live for the
466 * fd_bc_invalidate_batch()
468 fd_batch_reference(&b
, dep
);
469 fd_batch_add_dep(batch
, b
);
470 fd_bc_invalidate_batch(b
, false);
471 fd_batch_reference_locked(&b
, NULL
);
474 fd_batch_reference_locked(&rsc
->write_batch
, batch
);
476 fd_batch_add_resource(batch
, rsc
);
480 fd_batch_resource_read_slowpath(struct fd_batch
*batch
, struct fd_resource
*rsc
)
482 fd_screen_assert_locked(batch
->ctx
->screen
);
485 fd_batch_resource_read(batch
, rsc
->stencil
);
487 DBG("%p: read %p", batch
, rsc
);
489 /* If reading a resource pending a write, go ahead and flush the
490 * writer. This avoids situations where we end up having to
491 * flush the current batch in _resource_used()
493 if (unlikely(rsc
->write_batch
&& rsc
->write_batch
!= batch
))
494 flush_write_batch(rsc
);
496 fd_batch_add_resource(batch
, rsc
);
500 fd_batch_check_size(struct fd_batch
*batch
)
502 debug_assert(!batch
->flushed
);
504 if (unlikely(fd_mesa_debug
& FD_DBG_FLUSH
)) {
505 fd_batch_flush(batch
);
509 if (fd_device_version(batch
->ctx
->screen
->dev
) >= FD_VERSION_UNLIMITED_CMDS
)
512 struct fd_ringbuffer
*ring
= batch
->draw
;
513 if ((ring
->cur
- ring
->start
) > (ring
->size
/4 - 0x1000))
514 fd_batch_flush(batch
);
517 /* emit a WAIT_FOR_IDLE only if needed, ie. if there has not already
518 * been one since last draw:
521 fd_wfi(struct fd_batch
*batch
, struct fd_ringbuffer
*ring
)
523 if (batch
->needs_wfi
) {
524 if (batch
->ctx
->screen
->gpu_id
>= 500)
528 batch
->needs_wfi
= false;