1 /**************************************************************************
3 * Copyright 2017 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * on the rights to use, copy, modify, merge, publish, distribute, sub
10 * license, and/or sell copies of the Software, and to permit persons to whom
11 * the Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
23 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 **************************************************************************/
27 #include "util/u_threaded_context.h"
28 #include "util/u_cpu_detect.h"
29 #include "util/format/u_format.h"
30 #include "util/u_inlines.h"
31 #include "util/u_memory.h"
32 #include "util/u_upload_mgr.h"
34 /* 0 = disabled, 1 = assertions, 2 = printfs */
38 #define tc_assert assert
44 #define tc_printf printf
45 #define tc_asprintf asprintf
46 #define tc_strcmp strcmp
48 #define tc_printf(...)
49 #define tc_asprintf(...) 0
50 #define tc_strcmp(...) 0
53 #define TC_SENTINEL 0x5ca1ab1e
56 #define CALL(name) TC_CALL_##name,
57 #include "u_threaded_context_calls.h"
62 typedef void (*tc_execute
)(struct pipe_context
*pipe
, union tc_payload
*payload
);
64 static const tc_execute execute_func
[TC_NUM_CALLS
];
67 tc_batch_check(UNUSED
struct tc_batch
*batch
)
69 tc_assert(batch
->sentinel
== TC_SENTINEL
);
70 tc_assert(batch
->num_total_call_slots
<= TC_CALLS_PER_BATCH
);
74 tc_debug_check(struct threaded_context
*tc
)
76 for (unsigned i
= 0; i
< TC_MAX_BATCHES
; i
++) {
77 tc_batch_check(&tc
->batch_slots
[i
]);
78 tc_assert(tc
->batch_slots
[i
].pipe
== tc
->pipe
);
83 tc_batch_execute(void *job
, UNUSED
int thread_index
)
85 struct tc_batch
*batch
= job
;
86 struct pipe_context
*pipe
= batch
->pipe
;
87 struct tc_call
*last
= &batch
->call
[batch
->num_total_call_slots
];
89 tc_batch_check(batch
);
91 assert(!batch
->token
);
93 for (struct tc_call
*iter
= batch
->call
; iter
!= last
;
94 iter
+= iter
->num_call_slots
) {
95 tc_assert(iter
->sentinel
== TC_SENTINEL
);
96 execute_func
[iter
->call_id
](pipe
, &iter
->payload
);
99 tc_batch_check(batch
);
100 batch
->num_total_call_slots
= 0;
104 tc_batch_flush(struct threaded_context
*tc
)
106 struct tc_batch
*next
= &tc
->batch_slots
[tc
->next
];
108 tc_assert(next
->num_total_call_slots
!= 0);
109 tc_batch_check(next
);
111 tc
->bytes_mapped_estimate
= 0;
112 p_atomic_add(&tc
->num_offloaded_slots
, next
->num_total_call_slots
);
115 next
->token
->tc
= NULL
;
116 tc_unflushed_batch_token_reference(&next
->token
, NULL
);
119 util_queue_add_job(&tc
->queue
, next
, &next
->fence
, tc_batch_execute
,
122 tc
->next
= (tc
->next
+ 1) % TC_MAX_BATCHES
;
125 /* This is the function that adds variable-sized calls into the current
126 * batch. It also flushes the batch if there is not enough space there.
127 * All other higher-level "add" functions use it.
129 static union tc_payload
*
130 tc_add_sized_call(struct threaded_context
*tc
, enum tc_call_id id
,
131 unsigned payload_size
)
133 struct tc_batch
*next
= &tc
->batch_slots
[tc
->next
];
134 unsigned total_size
= offsetof(struct tc_call
, payload
) + payload_size
;
135 unsigned num_call_slots
= DIV_ROUND_UP(total_size
, sizeof(struct tc_call
));
139 if (unlikely(next
->num_total_call_slots
+ num_call_slots
> TC_CALLS_PER_BATCH
)) {
141 next
= &tc
->batch_slots
[tc
->next
];
142 tc_assert(next
->num_total_call_slots
== 0);
145 tc_assert(util_queue_fence_is_signalled(&next
->fence
));
147 struct tc_call
*call
= &next
->call
[next
->num_total_call_slots
];
148 next
->num_total_call_slots
+= num_call_slots
;
150 call
->sentinel
= TC_SENTINEL
;
152 call
->num_call_slots
= num_call_slots
;
155 return &call
->payload
;
158 #define tc_add_struct_typed_call(tc, execute, type) \
159 ((struct type*)tc_add_sized_call(tc, execute, sizeof(struct type)))
161 #define tc_add_slot_based_call(tc, execute, type, num_slots) \
162 ((struct type*)tc_add_sized_call(tc, execute, \
163 sizeof(struct type) + \
164 sizeof(((struct type*)NULL)->slot[0]) * \
167 static union tc_payload
*
168 tc_add_small_call(struct threaded_context
*tc
, enum tc_call_id id
)
170 return tc_add_sized_call(tc
, id
, 0);
174 tc_is_sync(struct threaded_context
*tc
)
176 struct tc_batch
*last
= &tc
->batch_slots
[tc
->last
];
177 struct tc_batch
*next
= &tc
->batch_slots
[tc
->next
];
179 return util_queue_fence_is_signalled(&last
->fence
) &&
180 !next
->num_total_call_slots
;
184 _tc_sync(struct threaded_context
*tc
, UNUSED
const char *info
, UNUSED
const char *func
)
186 struct tc_batch
*last
= &tc
->batch_slots
[tc
->last
];
187 struct tc_batch
*next
= &tc
->batch_slots
[tc
->next
];
192 /* Only wait for queued calls... */
193 if (!util_queue_fence_is_signalled(&last
->fence
)) {
194 util_queue_fence_wait(&last
->fence
);
201 next
->token
->tc
= NULL
;
202 tc_unflushed_batch_token_reference(&next
->token
, NULL
);
205 /* .. and execute unflushed calls directly. */
206 if (next
->num_total_call_slots
) {
207 p_atomic_add(&tc
->num_direct_slots
, next
->num_total_call_slots
);
208 tc
->bytes_mapped_estimate
= 0;
209 tc_batch_execute(next
, 0);
214 p_atomic_inc(&tc
->num_syncs
);
216 if (tc_strcmp(func
, "tc_destroy") != 0) {
217 tc_printf("sync %s %s\n", func
, info
);
224 #define tc_sync(tc) _tc_sync(tc, "", __func__)
225 #define tc_sync_msg(tc, info) _tc_sync(tc, info, __func__)
228 * Call this from fence_finish for same-context fence waits of deferred fences
229 * that haven't been flushed yet.
231 * The passed pipe_context must be the one passed to pipe_screen::fence_finish,
232 * i.e., the wrapped one.
235 threaded_context_flush(struct pipe_context
*_pipe
,
236 struct tc_unflushed_batch_token
*token
,
239 struct threaded_context
*tc
= threaded_context(_pipe
);
241 /* This is called from the state-tracker / application thread. */
242 if (token
->tc
&& token
->tc
== tc
) {
243 struct tc_batch
*last
= &tc
->batch_slots
[tc
->last
];
245 /* Prefer to do the flush in the driver thread if it is already
246 * running. That should be better for cache locality.
248 if (prefer_async
|| !util_queue_fence_is_signalled(&last
->fence
))
256 tc_set_resource_reference(struct pipe_resource
**dst
, struct pipe_resource
*src
)
259 pipe_resource_reference(dst
, src
);
263 threaded_resource_init(struct pipe_resource
*res
)
265 struct threaded_resource
*tres
= threaded_resource(res
);
267 tres
->latest
= &tres
->b
;
268 util_range_init(&tres
->valid_buffer_range
);
269 tres
->base_valid_buffer_range
= &tres
->valid_buffer_range
;
270 tres
->is_shared
= false;
271 tres
->is_user_ptr
= false;
275 threaded_resource_deinit(struct pipe_resource
*res
)
277 struct threaded_resource
*tres
= threaded_resource(res
);
279 if (tres
->latest
!= &tres
->b
)
280 pipe_resource_reference(&tres
->latest
, NULL
);
281 util_range_destroy(&tres
->valid_buffer_range
);
284 struct pipe_context
*
285 threaded_context_unwrap_sync(struct pipe_context
*pipe
)
287 if (!pipe
|| !pipe
->priv
)
290 tc_sync(threaded_context(pipe
));
291 return (struct pipe_context
*)pipe
->priv
;
295 /********************************************************************
299 #define TC_FUNC1(func, m_payload, qualifier, type, deref, deref2) \
301 tc_call_##func(struct pipe_context *pipe, union tc_payload *payload) \
303 pipe->func(pipe, deref2((type*)payload)); \
307 tc_##func(struct pipe_context *_pipe, qualifier type deref param) \
309 struct threaded_context *tc = threaded_context(_pipe); \
310 type *p = (type*)tc_add_sized_call(tc, TC_CALL_##func, sizeof(type)); \
314 TC_FUNC1(set_active_query_state
, flags
, , bool, , *)
316 TC_FUNC1(set_blend_color
, blend_color
, const, struct pipe_blend_color
, *, )
317 TC_FUNC1(set_stencil_ref
, stencil_ref
, const, struct pipe_stencil_ref
, *, )
318 TC_FUNC1(set_clip_state
, clip_state
, const, struct pipe_clip_state
, *, )
319 TC_FUNC1(set_sample_mask
, sample_mask
, , unsigned, , *)
320 TC_FUNC1(set_min_samples
, min_samples
, , unsigned, , *)
321 TC_FUNC1(set_polygon_stipple
, polygon_stipple
, const, struct pipe_poly_stipple
, *, )
323 TC_FUNC1(texture_barrier
, flags
, , unsigned, , *)
324 TC_FUNC1(memory_barrier
, flags
, , unsigned, , *)
327 /********************************************************************
331 static struct pipe_query
*
332 tc_create_query(struct pipe_context
*_pipe
, unsigned query_type
,
335 struct threaded_context
*tc
= threaded_context(_pipe
);
336 struct pipe_context
*pipe
= tc
->pipe
;
338 return pipe
->create_query(pipe
, query_type
, index
);
341 static struct pipe_query
*
342 tc_create_batch_query(struct pipe_context
*_pipe
, unsigned num_queries
,
343 unsigned *query_types
)
345 struct threaded_context
*tc
= threaded_context(_pipe
);
346 struct pipe_context
*pipe
= tc
->pipe
;
348 return pipe
->create_batch_query(pipe
, num_queries
, query_types
);
352 tc_call_destroy_query(struct pipe_context
*pipe
, union tc_payload
*payload
)
354 struct threaded_query
*tq
= threaded_query(payload
->query
);
356 if (tq
->head_unflushed
.next
)
357 list_del(&tq
->head_unflushed
);
359 pipe
->destroy_query(pipe
, payload
->query
);
363 tc_destroy_query(struct pipe_context
*_pipe
, struct pipe_query
*query
)
365 struct threaded_context
*tc
= threaded_context(_pipe
);
367 tc_add_small_call(tc
, TC_CALL_destroy_query
)->query
= query
;
371 tc_call_begin_query(struct pipe_context
*pipe
, union tc_payload
*payload
)
373 pipe
->begin_query(pipe
, payload
->query
);
377 tc_begin_query(struct pipe_context
*_pipe
, struct pipe_query
*query
)
379 struct threaded_context
*tc
= threaded_context(_pipe
);
380 union tc_payload
*payload
= tc_add_small_call(tc
, TC_CALL_begin_query
);
382 payload
->query
= query
;
383 return true; /* we don't care about the return value for this call */
386 struct tc_end_query_payload
{
387 struct threaded_context
*tc
;
388 struct pipe_query
*query
;
392 tc_call_end_query(struct pipe_context
*pipe
, union tc_payload
*payload
)
394 struct tc_end_query_payload
*p
= (struct tc_end_query_payload
*)payload
;
395 struct threaded_query
*tq
= threaded_query(p
->query
);
397 if (!tq
->head_unflushed
.next
)
398 list_add(&tq
->head_unflushed
, &p
->tc
->unflushed_queries
);
400 pipe
->end_query(pipe
, p
->query
);
404 tc_end_query(struct pipe_context
*_pipe
, struct pipe_query
*query
)
406 struct threaded_context
*tc
= threaded_context(_pipe
);
407 struct threaded_query
*tq
= threaded_query(query
);
408 struct tc_end_query_payload
*payload
=
409 tc_add_struct_typed_call(tc
, TC_CALL_end_query
, tc_end_query_payload
);
412 payload
->query
= query
;
416 return true; /* we don't care about the return value for this call */
420 tc_get_query_result(struct pipe_context
*_pipe
,
421 struct pipe_query
*query
, bool wait
,
422 union pipe_query_result
*result
)
424 struct threaded_context
*tc
= threaded_context(_pipe
);
425 struct threaded_query
*tq
= threaded_query(query
);
426 struct pipe_context
*pipe
= tc
->pipe
;
429 tc_sync_msg(tc
, wait
? "wait" : "nowait");
431 bool success
= pipe
->get_query_result(pipe
, query
, wait
, result
);
435 if (tq
->head_unflushed
.next
) {
436 /* This is safe because it can only happen after we sync'd. */
437 list_del(&tq
->head_unflushed
);
443 struct tc_query_result_resource
{
444 struct pipe_query
*query
;
446 enum pipe_query_value_type result_type
;
448 struct pipe_resource
*resource
;
453 tc_call_get_query_result_resource(struct pipe_context
*pipe
,
454 union tc_payload
*payload
)
456 struct tc_query_result_resource
*p
= (struct tc_query_result_resource
*)payload
;
458 pipe
->get_query_result_resource(pipe
, p
->query
, p
->wait
, p
->result_type
,
459 p
->index
, p
->resource
, p
->offset
);
460 pipe_resource_reference(&p
->resource
, NULL
);
464 tc_get_query_result_resource(struct pipe_context
*_pipe
,
465 struct pipe_query
*query
, bool wait
,
466 enum pipe_query_value_type result_type
, int index
,
467 struct pipe_resource
*resource
, unsigned offset
)
469 struct threaded_context
*tc
= threaded_context(_pipe
);
470 struct tc_query_result_resource
*p
=
471 tc_add_struct_typed_call(tc
, TC_CALL_get_query_result_resource
,
472 tc_query_result_resource
);
476 p
->result_type
= result_type
;
478 tc_set_resource_reference(&p
->resource
, resource
);
482 struct tc_render_condition
{
483 struct pipe_query
*query
;
489 tc_call_render_condition(struct pipe_context
*pipe
, union tc_payload
*payload
)
491 struct tc_render_condition
*p
= (struct tc_render_condition
*)payload
;
492 pipe
->render_condition(pipe
, p
->query
, p
->condition
, p
->mode
);
496 tc_render_condition(struct pipe_context
*_pipe
,
497 struct pipe_query
*query
, bool condition
,
498 enum pipe_render_cond_flag mode
)
500 struct threaded_context
*tc
= threaded_context(_pipe
);
501 struct tc_render_condition
*p
=
502 tc_add_struct_typed_call(tc
, TC_CALL_render_condition
, tc_render_condition
);
505 p
->condition
= condition
;
510 /********************************************************************
511 * constant (immutable) states
514 #define TC_CSO_CREATE(name, sname) \
516 tc_create_##name##_state(struct pipe_context *_pipe, \
517 const struct pipe_##sname##_state *state) \
519 struct pipe_context *pipe = threaded_context(_pipe)->pipe; \
520 return pipe->create_##name##_state(pipe, state); \
523 #define TC_CSO_BIND(name) TC_FUNC1(bind_##name##_state, cso, , void *, , *)
524 #define TC_CSO_DELETE(name) TC_FUNC1(delete_##name##_state, cso, , void *, , *)
526 #define TC_CSO_WHOLE2(name, sname) \
527 TC_CSO_CREATE(name, sname) \
531 #define TC_CSO_WHOLE(name) TC_CSO_WHOLE2(name, name)
534 TC_CSO_WHOLE(rasterizer
)
535 TC_CSO_WHOLE(depth_stencil_alpha
)
536 TC_CSO_WHOLE(compute
)
537 TC_CSO_WHOLE2(fs
, shader
)
538 TC_CSO_WHOLE2(vs
, shader
)
539 TC_CSO_WHOLE2(gs
, shader
)
540 TC_CSO_WHOLE2(tcs
, shader
)
541 TC_CSO_WHOLE2(tes
, shader
)
542 TC_CSO_CREATE(sampler
, sampler
)
543 TC_CSO_DELETE(sampler
)
544 TC_CSO_BIND(vertex_elements
)
545 TC_CSO_DELETE(vertex_elements
)
548 tc_create_vertex_elements_state(struct pipe_context
*_pipe
, unsigned count
,
549 const struct pipe_vertex_element
*elems
)
551 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
553 return pipe
->create_vertex_elements_state(pipe
, count
, elems
);
556 struct tc_sampler_states
{
557 ubyte shader
, start
, count
;
558 void *slot
[0]; /* more will be allocated if needed */
562 tc_call_bind_sampler_states(struct pipe_context
*pipe
, union tc_payload
*payload
)
564 struct tc_sampler_states
*p
= (struct tc_sampler_states
*)payload
;
565 pipe
->bind_sampler_states(pipe
, p
->shader
, p
->start
, p
->count
, p
->slot
);
569 tc_bind_sampler_states(struct pipe_context
*_pipe
,
570 enum pipe_shader_type shader
,
571 unsigned start
, unsigned count
, void **states
)
576 struct threaded_context
*tc
= threaded_context(_pipe
);
577 struct tc_sampler_states
*p
=
578 tc_add_slot_based_call(tc
, TC_CALL_bind_sampler_states
, tc_sampler_states
, count
);
583 memcpy(p
->slot
, states
, count
* sizeof(states
[0]));
587 /********************************************************************
592 tc_call_set_framebuffer_state(struct pipe_context
*pipe
, union tc_payload
*payload
)
594 struct pipe_framebuffer_state
*p
= (struct pipe_framebuffer_state
*)payload
;
596 pipe
->set_framebuffer_state(pipe
, p
);
598 unsigned nr_cbufs
= p
->nr_cbufs
;
599 for (unsigned i
= 0; i
< nr_cbufs
; i
++)
600 pipe_surface_reference(&p
->cbufs
[i
], NULL
);
601 pipe_surface_reference(&p
->zsbuf
, NULL
);
605 tc_set_framebuffer_state(struct pipe_context
*_pipe
,
606 const struct pipe_framebuffer_state
*fb
)
608 struct threaded_context
*tc
= threaded_context(_pipe
);
609 struct pipe_framebuffer_state
*p
=
610 tc_add_struct_typed_call(tc
, TC_CALL_set_framebuffer_state
,
611 pipe_framebuffer_state
);
612 unsigned nr_cbufs
= fb
->nr_cbufs
;
614 p
->width
= fb
->width
;
615 p
->height
= fb
->height
;
616 p
->samples
= fb
->samples
;
617 p
->layers
= fb
->layers
;
618 p
->nr_cbufs
= nr_cbufs
;
620 for (unsigned i
= 0; i
< nr_cbufs
; i
++) {
622 pipe_surface_reference(&p
->cbufs
[i
], fb
->cbufs
[i
]);
625 pipe_surface_reference(&p
->zsbuf
, fb
->zsbuf
);
629 tc_call_set_tess_state(struct pipe_context
*pipe
, union tc_payload
*payload
)
631 float *p
= (float*)payload
;
632 pipe
->set_tess_state(pipe
, p
, p
+ 4);
636 tc_set_tess_state(struct pipe_context
*_pipe
,
637 const float default_outer_level
[4],
638 const float default_inner_level
[2])
640 struct threaded_context
*tc
= threaded_context(_pipe
);
641 float *p
= (float*)tc_add_sized_call(tc
, TC_CALL_set_tess_state
,
644 memcpy(p
, default_outer_level
, 4 * sizeof(float));
645 memcpy(p
+ 4, default_inner_level
, 2 * sizeof(float));
648 struct tc_constant_buffer
{
650 struct pipe_constant_buffer cb
;
654 tc_call_set_constant_buffer(struct pipe_context
*pipe
, union tc_payload
*payload
)
656 struct tc_constant_buffer
*p
= (struct tc_constant_buffer
*)payload
;
658 pipe
->set_constant_buffer(pipe
,
662 pipe_resource_reference(&p
->cb
.buffer
, NULL
);
666 tc_set_constant_buffer(struct pipe_context
*_pipe
,
667 enum pipe_shader_type shader
, uint index
,
668 const struct pipe_constant_buffer
*cb
)
670 struct threaded_context
*tc
= threaded_context(_pipe
);
671 struct pipe_resource
*buffer
= NULL
;
674 /* This must be done before adding set_constant_buffer, because it could
675 * generate e.g. transfer_unmap and flush partially-uninitialized
676 * set_constant_buffer to the driver if it was done afterwards.
678 if (cb
&& cb
->user_buffer
) {
679 u_upload_data(tc
->base
.const_uploader
, 0, cb
->buffer_size
, 64,
680 cb
->user_buffer
, &offset
, &buffer
);
681 u_upload_unmap(tc
->base
.const_uploader
);
684 struct tc_constant_buffer
*p
=
685 tc_add_struct_typed_call(tc
, TC_CALL_set_constant_buffer
,
691 if (cb
->user_buffer
) {
692 p
->cb
.buffer_size
= cb
->buffer_size
;
693 p
->cb
.user_buffer
= NULL
;
694 p
->cb
.buffer_offset
= offset
;
695 p
->cb
.buffer
= buffer
;
697 tc_set_resource_reference(&p
->cb
.buffer
,
699 memcpy(&p
->cb
, cb
, sizeof(*cb
));
702 memset(&p
->cb
, 0, sizeof(*cb
));
708 struct pipe_scissor_state slot
[0]; /* more will be allocated if needed */
712 tc_call_set_scissor_states(struct pipe_context
*pipe
, union tc_payload
*payload
)
714 struct tc_scissors
*p
= (struct tc_scissors
*)payload
;
715 pipe
->set_scissor_states(pipe
, p
->start
, p
->count
, p
->slot
);
719 tc_set_scissor_states(struct pipe_context
*_pipe
,
720 unsigned start
, unsigned count
,
721 const struct pipe_scissor_state
*states
)
723 struct threaded_context
*tc
= threaded_context(_pipe
);
724 struct tc_scissors
*p
=
725 tc_add_slot_based_call(tc
, TC_CALL_set_scissor_states
, tc_scissors
, count
);
729 memcpy(&p
->slot
, states
, count
* sizeof(states
[0]));
732 struct tc_viewports
{
734 struct pipe_viewport_state slot
[0]; /* more will be allocated if needed */
738 tc_call_set_viewport_states(struct pipe_context
*pipe
, union tc_payload
*payload
)
740 struct tc_viewports
*p
= (struct tc_viewports
*)payload
;
741 pipe
->set_viewport_states(pipe
, p
->start
, p
->count
, p
->slot
);
745 tc_set_viewport_states(struct pipe_context
*_pipe
,
746 unsigned start
, unsigned count
,
747 const struct pipe_viewport_state
*states
)
752 struct threaded_context
*tc
= threaded_context(_pipe
);
753 struct tc_viewports
*p
=
754 tc_add_slot_based_call(tc
, TC_CALL_set_viewport_states
, tc_viewports
, count
);
758 memcpy(&p
->slot
, states
, count
* sizeof(states
[0]));
761 struct tc_window_rects
{
764 struct pipe_scissor_state slot
[0]; /* more will be allocated if needed */
768 tc_call_set_window_rectangles(struct pipe_context
*pipe
,
769 union tc_payload
*payload
)
771 struct tc_window_rects
*p
= (struct tc_window_rects
*)payload
;
772 pipe
->set_window_rectangles(pipe
, p
->include
, p
->count
, p
->slot
);
776 tc_set_window_rectangles(struct pipe_context
*_pipe
, bool include
,
778 const struct pipe_scissor_state
*rects
)
780 struct threaded_context
*tc
= threaded_context(_pipe
);
781 struct tc_window_rects
*p
=
782 tc_add_slot_based_call(tc
, TC_CALL_set_window_rectangles
, tc_window_rects
, count
);
784 p
->include
= include
;
786 memcpy(p
->slot
, rects
, count
* sizeof(rects
[0]));
789 struct tc_sampler_views
{
790 ubyte shader
, start
, count
;
791 struct pipe_sampler_view
*slot
[0]; /* more will be allocated if needed */
795 tc_call_set_sampler_views(struct pipe_context
*pipe
, union tc_payload
*payload
)
797 struct tc_sampler_views
*p
= (struct tc_sampler_views
*)payload
;
798 unsigned count
= p
->count
;
800 pipe
->set_sampler_views(pipe
, p
->shader
, p
->start
, p
->count
, p
->slot
);
801 for (unsigned i
= 0; i
< count
; i
++)
802 pipe_sampler_view_reference(&p
->slot
[i
], NULL
);
806 tc_set_sampler_views(struct pipe_context
*_pipe
,
807 enum pipe_shader_type shader
,
808 unsigned start
, unsigned count
,
809 struct pipe_sampler_view
**views
)
814 struct threaded_context
*tc
= threaded_context(_pipe
);
815 struct tc_sampler_views
*p
=
816 tc_add_slot_based_call(tc
, TC_CALL_set_sampler_views
, tc_sampler_views
, count
);
823 for (unsigned i
= 0; i
< count
; i
++) {
825 pipe_sampler_view_reference(&p
->slot
[i
], views
[i
]);
828 memset(p
->slot
, 0, count
* sizeof(views
[0]));
832 struct tc_shader_images
{
833 ubyte shader
, start
, count
;
835 struct pipe_image_view slot
[0]; /* more will be allocated if needed */
839 tc_call_set_shader_images(struct pipe_context
*pipe
, union tc_payload
*payload
)
841 struct tc_shader_images
*p
= (struct tc_shader_images
*)payload
;
842 unsigned count
= p
->count
;
845 pipe
->set_shader_images(pipe
, p
->shader
, p
->start
, p
->count
, NULL
);
849 pipe
->set_shader_images(pipe
, p
->shader
, p
->start
, p
->count
, p
->slot
);
851 for (unsigned i
= 0; i
< count
; i
++)
852 pipe_resource_reference(&p
->slot
[i
].resource
, NULL
);
856 tc_set_shader_images(struct pipe_context
*_pipe
,
857 enum pipe_shader_type shader
,
858 unsigned start
, unsigned count
,
859 const struct pipe_image_view
*images
)
864 struct threaded_context
*tc
= threaded_context(_pipe
);
865 struct tc_shader_images
*p
=
866 tc_add_slot_based_call(tc
, TC_CALL_set_shader_images
, tc_shader_images
,
872 p
->unbind
= images
== NULL
;
875 for (unsigned i
= 0; i
< count
; i
++) {
876 tc_set_resource_reference(&p
->slot
[i
].resource
, images
[i
].resource
);
878 if (images
[i
].access
& PIPE_IMAGE_ACCESS_WRITE
&&
879 images
[i
].resource
&&
880 images
[i
].resource
->target
== PIPE_BUFFER
) {
881 struct threaded_resource
*tres
=
882 threaded_resource(images
[i
].resource
);
884 util_range_add(&tres
->b
, &tres
->valid_buffer_range
,
885 images
[i
].u
.buf
.offset
,
886 images
[i
].u
.buf
.offset
+ images
[i
].u
.buf
.size
);
889 memcpy(p
->slot
, images
, count
* sizeof(images
[0]));
893 struct tc_shader_buffers
{
894 ubyte shader
, start
, count
;
896 unsigned writable_bitmask
;
897 struct pipe_shader_buffer slot
[0]; /* more will be allocated if needed */
901 tc_call_set_shader_buffers(struct pipe_context
*pipe
, union tc_payload
*payload
)
903 struct tc_shader_buffers
*p
= (struct tc_shader_buffers
*)payload
;
904 unsigned count
= p
->count
;
907 pipe
->set_shader_buffers(pipe
, p
->shader
, p
->start
, p
->count
, NULL
, 0);
911 pipe
->set_shader_buffers(pipe
, p
->shader
, p
->start
, p
->count
, p
->slot
,
912 p
->writable_bitmask
);
914 for (unsigned i
= 0; i
< count
; i
++)
915 pipe_resource_reference(&p
->slot
[i
].buffer
, NULL
);
919 tc_set_shader_buffers(struct pipe_context
*_pipe
,
920 enum pipe_shader_type shader
,
921 unsigned start
, unsigned count
,
922 const struct pipe_shader_buffer
*buffers
,
923 unsigned writable_bitmask
)
928 struct threaded_context
*tc
= threaded_context(_pipe
);
929 struct tc_shader_buffers
*p
=
930 tc_add_slot_based_call(tc
, TC_CALL_set_shader_buffers
, tc_shader_buffers
,
931 buffers
? count
: 0);
936 p
->unbind
= buffers
== NULL
;
937 p
->writable_bitmask
= writable_bitmask
;
940 for (unsigned i
= 0; i
< count
; i
++) {
941 struct pipe_shader_buffer
*dst
= &p
->slot
[i
];
942 const struct pipe_shader_buffer
*src
= buffers
+ i
;
944 tc_set_resource_reference(&dst
->buffer
, src
->buffer
);
945 dst
->buffer_offset
= src
->buffer_offset
;
946 dst
->buffer_size
= src
->buffer_size
;
949 struct threaded_resource
*tres
= threaded_resource(src
->buffer
);
951 util_range_add(&tres
->b
, &tres
->valid_buffer_range
,
953 src
->buffer_offset
+ src
->buffer_size
);
959 struct tc_vertex_buffers
{
962 struct pipe_vertex_buffer slot
[0]; /* more will be allocated if needed */
966 tc_call_set_vertex_buffers(struct pipe_context
*pipe
, union tc_payload
*payload
)
968 struct tc_vertex_buffers
*p
= (struct tc_vertex_buffers
*)payload
;
969 unsigned count
= p
->count
;
972 pipe
->set_vertex_buffers(pipe
, p
->start
, count
, NULL
);
976 for (unsigned i
= 0; i
< count
; i
++)
977 tc_assert(!p
->slot
[i
].is_user_buffer
);
979 pipe
->set_vertex_buffers(pipe
, p
->start
, count
, p
->slot
);
980 for (unsigned i
= 0; i
< count
; i
++)
981 pipe_resource_reference(&p
->slot
[i
].buffer
.resource
, NULL
);
985 tc_set_vertex_buffers(struct pipe_context
*_pipe
,
986 unsigned start
, unsigned count
,
987 const struct pipe_vertex_buffer
*buffers
)
989 struct threaded_context
*tc
= threaded_context(_pipe
);
995 struct tc_vertex_buffers
*p
=
996 tc_add_slot_based_call(tc
, TC_CALL_set_vertex_buffers
, tc_vertex_buffers
, count
);
1001 for (unsigned i
= 0; i
< count
; i
++) {
1002 struct pipe_vertex_buffer
*dst
= &p
->slot
[i
];
1003 const struct pipe_vertex_buffer
*src
= buffers
+ i
;
1005 tc_assert(!src
->is_user_buffer
);
1006 dst
->stride
= src
->stride
;
1007 dst
->is_user_buffer
= false;
1008 tc_set_resource_reference(&dst
->buffer
.resource
,
1009 src
->buffer
.resource
);
1010 dst
->buffer_offset
= src
->buffer_offset
;
1013 struct tc_vertex_buffers
*p
=
1014 tc_add_slot_based_call(tc
, TC_CALL_set_vertex_buffers
, tc_vertex_buffers
, 0);
1021 struct tc_stream_outputs
{
1023 struct pipe_stream_output_target
*targets
[PIPE_MAX_SO_BUFFERS
];
1024 unsigned offsets
[PIPE_MAX_SO_BUFFERS
];
1028 tc_call_set_stream_output_targets(struct pipe_context
*pipe
, union tc_payload
*payload
)
1030 struct tc_stream_outputs
*p
= (struct tc_stream_outputs
*)payload
;
1031 unsigned count
= p
->count
;
1033 pipe
->set_stream_output_targets(pipe
, count
, p
->targets
, p
->offsets
);
1034 for (unsigned i
= 0; i
< count
; i
++)
1035 pipe_so_target_reference(&p
->targets
[i
], NULL
);
1039 tc_set_stream_output_targets(struct pipe_context
*_pipe
,
1041 struct pipe_stream_output_target
**tgs
,
1042 const unsigned *offsets
)
1044 struct threaded_context
*tc
= threaded_context(_pipe
);
1045 struct tc_stream_outputs
*p
=
1046 tc_add_struct_typed_call(tc
, TC_CALL_set_stream_output_targets
,
1049 for (unsigned i
= 0; i
< count
; i
++) {
1050 p
->targets
[i
] = NULL
;
1051 pipe_so_target_reference(&p
->targets
[i
], tgs
[i
]);
1054 memcpy(p
->offsets
, offsets
, count
* sizeof(unsigned));
1058 tc_set_compute_resources(struct pipe_context
*_pipe
, unsigned start
,
1059 unsigned count
, struct pipe_surface
**resources
)
1061 struct threaded_context
*tc
= threaded_context(_pipe
);
1062 struct pipe_context
*pipe
= tc
->pipe
;
1065 pipe
->set_compute_resources(pipe
, start
, count
, resources
);
1069 tc_set_global_binding(struct pipe_context
*_pipe
, unsigned first
,
1070 unsigned count
, struct pipe_resource
**resources
,
1073 struct threaded_context
*tc
= threaded_context(_pipe
);
1074 struct pipe_context
*pipe
= tc
->pipe
;
1077 pipe
->set_global_binding(pipe
, first
, count
, resources
, handles
);
1081 /********************************************************************
1085 static struct pipe_surface
*
1086 tc_create_surface(struct pipe_context
*_pipe
,
1087 struct pipe_resource
*resource
,
1088 const struct pipe_surface
*surf_tmpl
)
1090 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
1091 struct pipe_surface
*view
=
1092 pipe
->create_surface(pipe
, resource
, surf_tmpl
);
1095 view
->context
= _pipe
;
1100 tc_surface_destroy(struct pipe_context
*_pipe
,
1101 struct pipe_surface
*surf
)
1103 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
1105 pipe
->surface_destroy(pipe
, surf
);
1108 static struct pipe_sampler_view
*
1109 tc_create_sampler_view(struct pipe_context
*_pipe
,
1110 struct pipe_resource
*resource
,
1111 const struct pipe_sampler_view
*templ
)
1113 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
1114 struct pipe_sampler_view
*view
=
1115 pipe
->create_sampler_view(pipe
, resource
, templ
);
1118 view
->context
= _pipe
;
1123 tc_sampler_view_destroy(struct pipe_context
*_pipe
,
1124 struct pipe_sampler_view
*view
)
1126 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
1128 pipe
->sampler_view_destroy(pipe
, view
);
1131 static struct pipe_stream_output_target
*
1132 tc_create_stream_output_target(struct pipe_context
*_pipe
,
1133 struct pipe_resource
*res
,
1134 unsigned buffer_offset
,
1135 unsigned buffer_size
)
1137 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
1138 struct threaded_resource
*tres
= threaded_resource(res
);
1139 struct pipe_stream_output_target
*view
;
1141 tc_sync(threaded_context(_pipe
));
1142 util_range_add(&tres
->b
, &tres
->valid_buffer_range
, buffer_offset
,
1143 buffer_offset
+ buffer_size
);
1145 view
= pipe
->create_stream_output_target(pipe
, res
, buffer_offset
,
1148 view
->context
= _pipe
;
1153 tc_stream_output_target_destroy(struct pipe_context
*_pipe
,
1154 struct pipe_stream_output_target
*target
)
1156 struct pipe_context
*pipe
= threaded_context(_pipe
)->pipe
;
1158 pipe
->stream_output_target_destroy(pipe
, target
);
1162 /********************************************************************
1167 tc_create_texture_handle(struct pipe_context
*_pipe
,
1168 struct pipe_sampler_view
*view
,
1169 const struct pipe_sampler_state
*state
)
1171 struct threaded_context
*tc
= threaded_context(_pipe
);
1172 struct pipe_context
*pipe
= tc
->pipe
;
1175 return pipe
->create_texture_handle(pipe
, view
, state
);
1179 tc_call_delete_texture_handle(struct pipe_context
*pipe
,
1180 union tc_payload
*payload
)
1182 pipe
->delete_texture_handle(pipe
, payload
->handle
);
1186 tc_delete_texture_handle(struct pipe_context
*_pipe
, uint64_t handle
)
1188 struct threaded_context
*tc
= threaded_context(_pipe
);
1189 union tc_payload
*payload
=
1190 tc_add_small_call(tc
, TC_CALL_delete_texture_handle
);
1192 payload
->handle
= handle
;
1195 struct tc_make_texture_handle_resident
1202 tc_call_make_texture_handle_resident(struct pipe_context
*pipe
,
1203 union tc_payload
*payload
)
1205 struct tc_make_texture_handle_resident
*p
=
1206 (struct tc_make_texture_handle_resident
*)payload
;
1208 pipe
->make_texture_handle_resident(pipe
, p
->handle
, p
->resident
);
1212 tc_make_texture_handle_resident(struct pipe_context
*_pipe
, uint64_t handle
,
1215 struct threaded_context
*tc
= threaded_context(_pipe
);
1216 struct tc_make_texture_handle_resident
*p
=
1217 tc_add_struct_typed_call(tc
, TC_CALL_make_texture_handle_resident
,
1218 tc_make_texture_handle_resident
);
1221 p
->resident
= resident
;
1225 tc_create_image_handle(struct pipe_context
*_pipe
,
1226 const struct pipe_image_view
*image
)
1228 struct threaded_context
*tc
= threaded_context(_pipe
);
1229 struct pipe_context
*pipe
= tc
->pipe
;
1232 return pipe
->create_image_handle(pipe
, image
);
1236 tc_call_delete_image_handle(struct pipe_context
*pipe
,
1237 union tc_payload
*payload
)
1239 pipe
->delete_image_handle(pipe
, payload
->handle
);
1243 tc_delete_image_handle(struct pipe_context
*_pipe
, uint64_t handle
)
1245 struct threaded_context
*tc
= threaded_context(_pipe
);
1246 union tc_payload
*payload
=
1247 tc_add_small_call(tc
, TC_CALL_delete_image_handle
);
1249 payload
->handle
= handle
;
1252 struct tc_make_image_handle_resident
1260 tc_call_make_image_handle_resident(struct pipe_context
*pipe
,
1261 union tc_payload
*payload
)
1263 struct tc_make_image_handle_resident
*p
=
1264 (struct tc_make_image_handle_resident
*)payload
;
1266 pipe
->make_image_handle_resident(pipe
, p
->handle
, p
->access
, p
->resident
);
1270 tc_make_image_handle_resident(struct pipe_context
*_pipe
, uint64_t handle
,
1271 unsigned access
, bool resident
)
1273 struct threaded_context
*tc
= threaded_context(_pipe
);
1274 struct tc_make_image_handle_resident
*p
=
1275 tc_add_struct_typed_call(tc
, TC_CALL_make_image_handle_resident
,
1276 tc_make_image_handle_resident
);
1280 p
->resident
= resident
;
1284 /********************************************************************
1288 struct tc_replace_buffer_storage
{
1289 struct pipe_resource
*dst
;
1290 struct pipe_resource
*src
;
1291 tc_replace_buffer_storage_func func
;
1295 tc_call_replace_buffer_storage(struct pipe_context
*pipe
,
1296 union tc_payload
*payload
)
1298 struct tc_replace_buffer_storage
*p
=
1299 (struct tc_replace_buffer_storage
*)payload
;
1301 p
->func(pipe
, p
->dst
, p
->src
);
1302 pipe_resource_reference(&p
->dst
, NULL
);
1303 pipe_resource_reference(&p
->src
, NULL
);
1307 tc_invalidate_buffer(struct threaded_context
*tc
,
1308 struct threaded_resource
*tbuf
)
1310 /* We can't check if the buffer is idle, so we invalidate it
1311 * unconditionally. */
1312 struct pipe_screen
*screen
= tc
->base
.screen
;
1313 struct pipe_resource
*new_buf
;
1315 /* Shared, pinned, and sparse buffers can't be reallocated. */
1316 if (tbuf
->is_shared
||
1317 tbuf
->is_user_ptr
||
1318 tbuf
->b
.flags
& PIPE_RESOURCE_FLAG_SPARSE
)
1321 /* Allocate a new one. */
1322 new_buf
= screen
->resource_create(screen
, &tbuf
->b
);
1326 /* Replace the "latest" pointer. */
1327 if (tbuf
->latest
!= &tbuf
->b
)
1328 pipe_resource_reference(&tbuf
->latest
, NULL
);
1330 tbuf
->latest
= new_buf
;
1331 util_range_set_empty(&tbuf
->valid_buffer_range
);
1333 /* The valid range should point to the original buffer. */
1334 threaded_resource(new_buf
)->base_valid_buffer_range
=
1335 &tbuf
->valid_buffer_range
;
1337 /* Enqueue storage replacement of the original buffer. */
1338 struct tc_replace_buffer_storage
*p
=
1339 tc_add_struct_typed_call(tc
, TC_CALL_replace_buffer_storage
,
1340 tc_replace_buffer_storage
);
1342 p
->func
= tc
->replace_buffer_storage
;
1343 tc_set_resource_reference(&p
->dst
, &tbuf
->b
);
1344 tc_set_resource_reference(&p
->src
, new_buf
);
1349 tc_improve_map_buffer_flags(struct threaded_context
*tc
,
1350 struct threaded_resource
*tres
, unsigned usage
,
1351 unsigned offset
, unsigned size
)
1353 /* Never invalidate inside the driver and never infer "unsynchronized". */
1354 unsigned tc_flags
= TC_TRANSFER_MAP_NO_INVALIDATE
|
1355 TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED
;
1357 /* Prevent a reentry. */
1358 if (usage
& tc_flags
)
1361 /* Use the staging upload if it's preferred. */
1362 if (usage
& (PIPE_TRANSFER_DISCARD_RANGE
|
1363 PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) &&
1364 !(usage
& PIPE_TRANSFER_PERSISTENT
) &&
1365 /* Try not to decrement the counter if it's not positive. Still racy,
1366 * but it makes it harder to wrap the counter from INT_MIN to INT_MAX. */
1367 tres
->max_forced_staging_uploads
> 0 &&
1368 p_atomic_dec_return(&tres
->max_forced_staging_uploads
) >= 0) {
1369 usage
&= ~(PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
|
1370 PIPE_TRANSFER_UNSYNCHRONIZED
);
1372 return usage
| tc_flags
| PIPE_TRANSFER_DISCARD_RANGE
;
1375 /* Sparse buffers can't be mapped directly and can't be reallocated
1376 * (fully invalidated). That may just be a radeonsi limitation, but
1377 * the threaded context must obey it with radeonsi.
1379 if (tres
->b
.flags
& PIPE_RESOURCE_FLAG_SPARSE
) {
1380 /* We can use DISCARD_RANGE instead of full discard. This is the only
1381 * fast path for sparse buffers that doesn't need thread synchronization.
1383 if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
)
1384 usage
|= PIPE_TRANSFER_DISCARD_RANGE
;
1386 /* Allow DISCARD_WHOLE_RESOURCE and infering UNSYNCHRONIZED in drivers.
1387 * The threaded context doesn't do unsychronized mappings and invalida-
1388 * tions of sparse buffers, therefore a correct driver behavior won't
1389 * result in an incorrect behavior with the threaded context.
1396 /* Handle CPU reads trivially. */
1397 if (usage
& PIPE_TRANSFER_READ
) {
1398 if (usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)
1399 usage
|= TC_TRANSFER_MAP_THREADED_UNSYNC
; /* don't sync */
1401 /* Drivers aren't allowed to do buffer invalidations. */
1402 return usage
& ~PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
;
1405 /* See if the buffer range being mapped has never been initialized,
1406 * in which case it can be mapped unsynchronized. */
1407 if (!(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) &&
1409 !util_ranges_intersect(&tres
->valid_buffer_range
, offset
, offset
+ size
))
1410 usage
|= PIPE_TRANSFER_UNSYNCHRONIZED
;
1412 if (!(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
1413 /* If discarding the entire range, discard the whole resource instead. */
1414 if (usage
& PIPE_TRANSFER_DISCARD_RANGE
&&
1415 offset
== 0 && size
== tres
->b
.width0
)
1416 usage
|= PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
;
1418 /* Discard the whole resource if needed. */
1419 if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
1420 if (tc_invalidate_buffer(tc
, tres
))
1421 usage
|= PIPE_TRANSFER_UNSYNCHRONIZED
;
1423 usage
|= PIPE_TRANSFER_DISCARD_RANGE
; /* fallback */
1427 /* We won't need this flag anymore. */
1428 /* TODO: We might not need TC_TRANSFER_MAP_NO_INVALIDATE with this. */
1429 usage
&= ~PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
;
1431 /* GL_AMD_pinned_memory and persistent mappings can't use staging
1433 if (usage
& (PIPE_TRANSFER_UNSYNCHRONIZED
|
1434 PIPE_TRANSFER_PERSISTENT
) ||
1436 usage
&= ~PIPE_TRANSFER_DISCARD_RANGE
;
1438 /* Unsychronized buffer mappings don't have to synchronize the thread. */
1439 if (usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) {
1440 usage
&= ~PIPE_TRANSFER_DISCARD_RANGE
;
1441 usage
|= TC_TRANSFER_MAP_THREADED_UNSYNC
; /* notify the driver */
1448 tc_transfer_map(struct pipe_context
*_pipe
,
1449 struct pipe_resource
*resource
, unsigned level
,
1450 unsigned usage
, const struct pipe_box
*box
,
1451 struct pipe_transfer
**transfer
)
1453 struct threaded_context
*tc
= threaded_context(_pipe
);
1454 struct threaded_resource
*tres
= threaded_resource(resource
);
1455 struct pipe_context
*pipe
= tc
->pipe
;
1457 if (resource
->target
== PIPE_BUFFER
) {
1458 usage
= tc_improve_map_buffer_flags(tc
, tres
, usage
, box
->x
, box
->width
);
1460 /* Do a staging transfer within the threaded context. The driver should
1461 * only get resource_copy_region.
1463 if (usage
& PIPE_TRANSFER_DISCARD_RANGE
) {
1464 struct threaded_transfer
*ttrans
= slab_alloc(&tc
->pool_transfers
);
1467 ttrans
->staging
= NULL
;
1469 u_upload_alloc(tc
->base
.stream_uploader
, 0,
1470 box
->width
+ (box
->x
% tc
->map_buffer_alignment
),
1471 64, &ttrans
->offset
, &ttrans
->staging
, (void**)&map
);
1473 slab_free(&tc
->pool_transfers
, ttrans
);
1477 tc_set_resource_reference(&ttrans
->b
.resource
, resource
);
1478 ttrans
->b
.level
= 0;
1479 ttrans
->b
.usage
= usage
;
1480 ttrans
->b
.box
= *box
;
1481 ttrans
->b
.stride
= 0;
1482 ttrans
->b
.layer_stride
= 0;
1483 *transfer
= &ttrans
->b
;
1484 return map
+ (box
->x
% tc
->map_buffer_alignment
);
1488 /* Unsychronized buffer mappings don't have to synchronize the thread. */
1489 if (!(usage
& TC_TRANSFER_MAP_THREADED_UNSYNC
))
1490 tc_sync_msg(tc
, resource
->target
!= PIPE_BUFFER
? " texture" :
1491 usage
& PIPE_TRANSFER_DISCARD_RANGE
? " discard_range" :
1492 usage
& PIPE_TRANSFER_READ
? " read" : " ??");
1494 tc
->bytes_mapped_estimate
+= box
->width
;
1496 return pipe
->transfer_map(pipe
, tres
->latest
? tres
->latest
: resource
,
1497 level
, usage
, box
, transfer
);
1500 struct tc_transfer_flush_region
{
1501 struct pipe_transfer
*transfer
;
1502 struct pipe_box box
;
1506 tc_call_transfer_flush_region(struct pipe_context
*pipe
,
1507 union tc_payload
*payload
)
1509 struct tc_transfer_flush_region
*p
=
1510 (struct tc_transfer_flush_region
*)payload
;
1512 pipe
->transfer_flush_region(pipe
, p
->transfer
, &p
->box
);
1515 struct tc_resource_copy_region
{
1516 struct pipe_resource
*dst
;
1518 unsigned dstx
, dsty
, dstz
;
1519 struct pipe_resource
*src
;
1521 struct pipe_box src_box
;
1525 tc_resource_copy_region(struct pipe_context
*_pipe
,
1526 struct pipe_resource
*dst
, unsigned dst_level
,
1527 unsigned dstx
, unsigned dsty
, unsigned dstz
,
1528 struct pipe_resource
*src
, unsigned src_level
,
1529 const struct pipe_box
*src_box
);
1532 tc_buffer_do_flush_region(struct threaded_context
*tc
,
1533 struct threaded_transfer
*ttrans
,
1534 const struct pipe_box
*box
)
1536 struct threaded_resource
*tres
= threaded_resource(ttrans
->b
.resource
);
1538 if (ttrans
->staging
) {
1539 struct pipe_box src_box
;
1541 u_box_1d(ttrans
->offset
+ ttrans
->b
.box
.x
% tc
->map_buffer_alignment
+
1542 (box
->x
- ttrans
->b
.box
.x
),
1543 box
->width
, &src_box
);
1545 /* Copy the staging buffer into the original one. */
1546 tc_resource_copy_region(&tc
->base
, ttrans
->b
.resource
, 0, box
->x
, 0, 0,
1547 ttrans
->staging
, 0, &src_box
);
1550 util_range_add(&tres
->b
, tres
->base_valid_buffer_range
,
1551 box
->x
, box
->x
+ box
->width
);
1555 tc_transfer_flush_region(struct pipe_context
*_pipe
,
1556 struct pipe_transfer
*transfer
,
1557 const struct pipe_box
*rel_box
)
1559 struct threaded_context
*tc
= threaded_context(_pipe
);
1560 struct threaded_transfer
*ttrans
= threaded_transfer(transfer
);
1561 struct threaded_resource
*tres
= threaded_resource(transfer
->resource
);
1562 unsigned required_usage
= PIPE_TRANSFER_WRITE
|
1563 PIPE_TRANSFER_FLUSH_EXPLICIT
;
1565 if (tres
->b
.target
== PIPE_BUFFER
) {
1566 if ((transfer
->usage
& required_usage
) == required_usage
) {
1567 struct pipe_box box
;
1569 u_box_1d(transfer
->box
.x
+ rel_box
->x
, rel_box
->width
, &box
);
1570 tc_buffer_do_flush_region(tc
, ttrans
, &box
);
1573 /* Staging transfers don't send the call to the driver. */
1574 if (ttrans
->staging
)
1578 struct tc_transfer_flush_region
*p
=
1579 tc_add_struct_typed_call(tc
, TC_CALL_transfer_flush_region
,
1580 tc_transfer_flush_region
);
1581 p
->transfer
= transfer
;
1586 tc_call_transfer_unmap(struct pipe_context
*pipe
, union tc_payload
*payload
)
1588 pipe
->transfer_unmap(pipe
, payload
->transfer
);
1592 tc_flush(struct pipe_context
*_pipe
, struct pipe_fence_handle
**fence
,
1596 tc_transfer_unmap(struct pipe_context
*_pipe
, struct pipe_transfer
*transfer
)
1598 struct threaded_context
*tc
= threaded_context(_pipe
);
1599 struct threaded_transfer
*ttrans
= threaded_transfer(transfer
);
1600 struct threaded_resource
*tres
= threaded_resource(transfer
->resource
);
1602 if (tres
->b
.target
== PIPE_BUFFER
) {
1603 if (transfer
->usage
& PIPE_TRANSFER_WRITE
&&
1604 !(transfer
->usage
& PIPE_TRANSFER_FLUSH_EXPLICIT
))
1605 tc_buffer_do_flush_region(tc
, ttrans
, &transfer
->box
);
1607 /* Staging transfers don't send the call to the driver. */
1608 if (ttrans
->staging
) {
1609 pipe_resource_reference(&ttrans
->staging
, NULL
);
1610 pipe_resource_reference(&ttrans
->b
.resource
, NULL
);
1611 slab_free(&tc
->pool_transfers
, ttrans
);
1616 tc_add_small_call(tc
, TC_CALL_transfer_unmap
)->transfer
= transfer
;
1618 /* tc_transfer_map directly maps the buffers, but tc_transfer_unmap
1619 * defers the unmap operation to the batch execution.
1620 * bytes_mapped_estimate is an estimation of the map/unmap bytes delta
1621 * and if it goes over an optional limit the current batch is flushed,
1622 * to reclaim some RAM. */
1623 if (!ttrans
->staging
&& tc
->bytes_mapped_limit
&&
1624 tc
->bytes_mapped_estimate
> tc
->bytes_mapped_limit
) {
1625 tc_flush(_pipe
, NULL
, PIPE_FLUSH_ASYNC
);
1629 struct tc_buffer_subdata
{
1630 struct pipe_resource
*resource
;
1631 unsigned usage
, offset
, size
;
1632 char slot
[0]; /* more will be allocated if needed */
1636 tc_call_buffer_subdata(struct pipe_context
*pipe
, union tc_payload
*payload
)
1638 struct tc_buffer_subdata
*p
= (struct tc_buffer_subdata
*)payload
;
1640 pipe
->buffer_subdata(pipe
, p
->resource
, p
->usage
, p
->offset
, p
->size
,
1642 pipe_resource_reference(&p
->resource
, NULL
);
1646 tc_buffer_subdata(struct pipe_context
*_pipe
,
1647 struct pipe_resource
*resource
,
1648 unsigned usage
, unsigned offset
,
1649 unsigned size
, const void *data
)
1651 struct threaded_context
*tc
= threaded_context(_pipe
);
1652 struct threaded_resource
*tres
= threaded_resource(resource
);
1657 usage
|= PIPE_TRANSFER_WRITE
;
1659 /* PIPE_TRANSFER_MAP_DIRECTLY supresses implicit DISCARD_RANGE. */
1660 if (!(usage
& PIPE_TRANSFER_MAP_DIRECTLY
))
1661 usage
|= PIPE_TRANSFER_DISCARD_RANGE
;
1663 usage
= tc_improve_map_buffer_flags(tc
, tres
, usage
, offset
, size
);
1665 /* Unsychronized and big transfers should use transfer_map. Also handle
1666 * full invalidations, because drivers aren't allowed to do them.
1668 if (usage
& (PIPE_TRANSFER_UNSYNCHRONIZED
|
1669 PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) ||
1670 size
> TC_MAX_SUBDATA_BYTES
) {
1671 struct pipe_transfer
*transfer
;
1672 struct pipe_box box
;
1673 uint8_t *map
= NULL
;
1675 u_box_1d(offset
, size
, &box
);
1677 map
= tc_transfer_map(_pipe
, resource
, 0, usage
, &box
, &transfer
);
1679 memcpy(map
, data
, size
);
1680 tc_transfer_unmap(_pipe
, transfer
);
1685 util_range_add(&tres
->b
, &tres
->valid_buffer_range
, offset
, offset
+ size
);
1687 /* The upload is small. Enqueue it. */
1688 struct tc_buffer_subdata
*p
=
1689 tc_add_slot_based_call(tc
, TC_CALL_buffer_subdata
, tc_buffer_subdata
, size
);
1691 tc_set_resource_reference(&p
->resource
, resource
);
1695 memcpy(p
->slot
, data
, size
);
1698 struct tc_texture_subdata
{
1699 struct pipe_resource
*resource
;
1700 unsigned level
, usage
, stride
, layer_stride
;
1701 struct pipe_box box
;
1702 char slot
[0]; /* more will be allocated if needed */
1706 tc_call_texture_subdata(struct pipe_context
*pipe
, union tc_payload
*payload
)
1708 struct tc_texture_subdata
*p
= (struct tc_texture_subdata
*)payload
;
1710 pipe
->texture_subdata(pipe
, p
->resource
, p
->level
, p
->usage
, &p
->box
,
1711 p
->slot
, p
->stride
, p
->layer_stride
);
1712 pipe_resource_reference(&p
->resource
, NULL
);
1716 tc_texture_subdata(struct pipe_context
*_pipe
,
1717 struct pipe_resource
*resource
,
1718 unsigned level
, unsigned usage
,
1719 const struct pipe_box
*box
,
1720 const void *data
, unsigned stride
,
1721 unsigned layer_stride
)
1723 struct threaded_context
*tc
= threaded_context(_pipe
);
1726 assert(box
->height
>= 1);
1727 assert(box
->depth
>= 1);
1729 size
= (box
->depth
- 1) * layer_stride
+
1730 (box
->height
- 1) * stride
+
1731 box
->width
* util_format_get_blocksize(resource
->format
);
1735 /* Small uploads can be enqueued, big uploads must sync. */
1736 if (size
<= TC_MAX_SUBDATA_BYTES
) {
1737 struct tc_texture_subdata
*p
=
1738 tc_add_slot_based_call(tc
, TC_CALL_texture_subdata
, tc_texture_subdata
, size
);
1740 tc_set_resource_reference(&p
->resource
, resource
);
1745 p
->layer_stride
= layer_stride
;
1746 memcpy(p
->slot
, data
, size
);
1748 struct pipe_context
*pipe
= tc
->pipe
;
1751 pipe
->texture_subdata(pipe
, resource
, level
, usage
, box
, data
,
1752 stride
, layer_stride
);
1757 /********************************************************************
1761 #define TC_FUNC_SYNC_RET0(ret_type, func) \
1763 tc_##func(struct pipe_context *_pipe) \
1765 struct threaded_context *tc = threaded_context(_pipe); \
1766 struct pipe_context *pipe = tc->pipe; \
1768 return pipe->func(pipe); \
1771 TC_FUNC_SYNC_RET0(enum pipe_reset_status
, get_device_reset_status
)
1772 TC_FUNC_SYNC_RET0(uint64_t, get_timestamp
)
1775 tc_get_sample_position(struct pipe_context
*_pipe
,
1776 unsigned sample_count
, unsigned sample_index
,
1779 struct threaded_context
*tc
= threaded_context(_pipe
);
1780 struct pipe_context
*pipe
= tc
->pipe
;
1783 pipe
->get_sample_position(pipe
, sample_count
, sample_index
,
1788 tc_set_device_reset_callback(struct pipe_context
*_pipe
,
1789 const struct pipe_device_reset_callback
*cb
)
1791 struct threaded_context
*tc
= threaded_context(_pipe
);
1792 struct pipe_context
*pipe
= tc
->pipe
;
1795 pipe
->set_device_reset_callback(pipe
, cb
);
1798 struct tc_string_marker
{
1800 char slot
[0]; /* more will be allocated if needed */
1804 tc_call_emit_string_marker(struct pipe_context
*pipe
, union tc_payload
*payload
)
1806 struct tc_string_marker
*p
= (struct tc_string_marker
*)payload
;
1807 pipe
->emit_string_marker(pipe
, p
->slot
, p
->len
);
1811 tc_emit_string_marker(struct pipe_context
*_pipe
,
1812 const char *string
, int len
)
1814 struct threaded_context
*tc
= threaded_context(_pipe
);
1816 if (len
<= TC_MAX_STRING_MARKER_BYTES
) {
1817 struct tc_string_marker
*p
=
1818 tc_add_slot_based_call(tc
, TC_CALL_emit_string_marker
, tc_string_marker
, len
);
1820 memcpy(p
->slot
, string
, len
);
1823 struct pipe_context
*pipe
= tc
->pipe
;
1826 pipe
->emit_string_marker(pipe
, string
, len
);
1831 tc_dump_debug_state(struct pipe_context
*_pipe
, FILE *stream
,
1834 struct threaded_context
*tc
= threaded_context(_pipe
);
1835 struct pipe_context
*pipe
= tc
->pipe
;
1838 pipe
->dump_debug_state(pipe
, stream
, flags
);
1842 tc_set_debug_callback(struct pipe_context
*_pipe
,
1843 const struct pipe_debug_callback
*cb
)
1845 struct threaded_context
*tc
= threaded_context(_pipe
);
1846 struct pipe_context
*pipe
= tc
->pipe
;
1848 /* Drop all synchronous debug callbacks. Drivers are expected to be OK
1849 * with this. shader-db will use an environment variable to disable
1850 * the threaded context.
1852 if (cb
&& cb
->debug_message
&& !cb
->async
)
1856 pipe
->set_debug_callback(pipe
, cb
);
1860 tc_set_log_context(struct pipe_context
*_pipe
, struct u_log_context
*log
)
1862 struct threaded_context
*tc
= threaded_context(_pipe
);
1863 struct pipe_context
*pipe
= tc
->pipe
;
1866 pipe
->set_log_context(pipe
, log
);
1870 tc_create_fence_fd(struct pipe_context
*_pipe
,
1871 struct pipe_fence_handle
**fence
, int fd
,
1872 enum pipe_fd_type type
)
1874 struct threaded_context
*tc
= threaded_context(_pipe
);
1875 struct pipe_context
*pipe
= tc
->pipe
;
1878 pipe
->create_fence_fd(pipe
, fence
, fd
, type
);
1882 tc_call_fence_server_sync(struct pipe_context
*pipe
, union tc_payload
*payload
)
1884 pipe
->fence_server_sync(pipe
, payload
->fence
);
1885 pipe
->screen
->fence_reference(pipe
->screen
, &payload
->fence
, NULL
);
1889 tc_fence_server_sync(struct pipe_context
*_pipe
,
1890 struct pipe_fence_handle
*fence
)
1892 struct threaded_context
*tc
= threaded_context(_pipe
);
1893 struct pipe_screen
*screen
= tc
->pipe
->screen
;
1894 union tc_payload
*payload
= tc_add_small_call(tc
, TC_CALL_fence_server_sync
);
1896 payload
->fence
= NULL
;
1897 screen
->fence_reference(screen
, &payload
->fence
, fence
);
1901 tc_call_fence_server_signal(struct pipe_context
*pipe
, union tc_payload
*payload
)
1903 pipe
->fence_server_signal(pipe
, payload
->fence
);
1904 pipe
->screen
->fence_reference(pipe
->screen
, &payload
->fence
, NULL
);
1908 tc_fence_server_signal(struct pipe_context
*_pipe
,
1909 struct pipe_fence_handle
*fence
)
1911 struct threaded_context
*tc
= threaded_context(_pipe
);
1912 struct pipe_screen
*screen
= tc
->pipe
->screen
;
1913 union tc_payload
*payload
= tc_add_small_call(tc
, TC_CALL_fence_server_signal
);
1915 payload
->fence
= NULL
;
1916 screen
->fence_reference(screen
, &payload
->fence
, fence
);
1919 static struct pipe_video_codec
*
1920 tc_create_video_codec(UNUSED
struct pipe_context
*_pipe
,
1921 UNUSED
const struct pipe_video_codec
*templ
)
1923 unreachable("Threaded context should not be enabled for video APIs");
1927 static struct pipe_video_buffer
*
1928 tc_create_video_buffer(UNUSED
struct pipe_context
*_pipe
,
1929 UNUSED
const struct pipe_video_buffer
*templ
)
1931 unreachable("Threaded context should not be enabled for video APIs");
1935 struct tc_context_param
{
1936 enum pipe_context_param param
;
1941 tc_call_set_context_param(struct pipe_context
*pipe
,
1942 union tc_payload
*payload
)
1944 struct tc_context_param
*p
= (struct tc_context_param
*)payload
;
1946 if (pipe
->set_context_param
)
1947 pipe
->set_context_param(pipe
, p
->param
, p
->value
);
1951 tc_set_context_param(struct pipe_context
*_pipe
,
1952 enum pipe_context_param param
,
1955 struct threaded_context
*tc
= threaded_context(_pipe
);
1957 if (param
== PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE
) {
1958 /* Pin the gallium thread as requested. */
1959 util_pin_thread_to_L3(tc
->queue
.threads
[0], value
,
1960 util_cpu_caps
.cores_per_L3
);
1962 /* Execute this immediately (without enqueuing).
1963 * It's required to be thread-safe.
1965 struct pipe_context
*pipe
= tc
->pipe
;
1966 if (pipe
->set_context_param
)
1967 pipe
->set_context_param(pipe
, param
, value
);
1971 if (tc
->pipe
->set_context_param
) {
1972 struct tc_context_param
*payload
=
1973 tc_add_struct_typed_call(tc
, TC_CALL_set_context_param
,
1976 payload
->param
= param
;
1977 payload
->value
= value
;
1982 /********************************************************************
1983 * draw, launch, clear, blit, copy, flush
1986 struct tc_flush_payload
{
1987 struct threaded_context
*tc
;
1988 struct pipe_fence_handle
*fence
;
1993 tc_flush_queries(struct threaded_context
*tc
)
1995 struct threaded_query
*tq
, *tmp
;
1996 LIST_FOR_EACH_ENTRY_SAFE(tq
, tmp
, &tc
->unflushed_queries
, head_unflushed
) {
1997 list_del(&tq
->head_unflushed
);
1999 /* Memory release semantics: due to a possible race with
2000 * tc_get_query_result, we must ensure that the linked list changes
2001 * are visible before setting tq->flushed.
2003 p_atomic_set(&tq
->flushed
, true);
2008 tc_call_flush(struct pipe_context
*pipe
, union tc_payload
*payload
)
2010 struct tc_flush_payload
*p
= (struct tc_flush_payload
*)payload
;
2011 struct pipe_screen
*screen
= pipe
->screen
;
2013 pipe
->flush(pipe
, p
->fence
? &p
->fence
: NULL
, p
->flags
);
2014 screen
->fence_reference(screen
, &p
->fence
, NULL
);
2016 if (!(p
->flags
& PIPE_FLUSH_DEFERRED
))
2017 tc_flush_queries(p
->tc
);
2021 tc_flush(struct pipe_context
*_pipe
, struct pipe_fence_handle
**fence
,
2024 struct threaded_context
*tc
= threaded_context(_pipe
);
2025 struct pipe_context
*pipe
= tc
->pipe
;
2026 struct pipe_screen
*screen
= pipe
->screen
;
2027 bool async
= flags
& PIPE_FLUSH_DEFERRED
;
2029 if (flags
& PIPE_FLUSH_ASYNC
) {
2030 struct tc_batch
*last
= &tc
->batch_slots
[tc
->last
];
2032 /* Prefer to do the flush in the driver thread, but avoid the inter-thread
2033 * communication overhead if the driver thread is currently idle and the
2034 * caller is going to wait for the fence immediately anyway.
2036 if (!(util_queue_fence_is_signalled(&last
->fence
) &&
2037 (flags
& PIPE_FLUSH_HINT_FINISH
)))
2041 if (async
&& tc
->create_fence
) {
2043 struct tc_batch
*next
= &tc
->batch_slots
[tc
->next
];
2046 next
->token
= malloc(sizeof(*next
->token
));
2050 pipe_reference_init(&next
->token
->ref
, 1);
2051 next
->token
->tc
= tc
;
2054 screen
->fence_reference(screen
, fence
, tc
->create_fence(pipe
, next
->token
));
2059 struct tc_flush_payload
*p
=
2060 tc_add_struct_typed_call(tc
, TC_CALL_flush
, tc_flush_payload
);
2062 p
->fence
= fence
? *fence
: NULL
;
2063 p
->flags
= flags
| TC_FLUSH_ASYNC
;
2065 if (!(flags
& PIPE_FLUSH_DEFERRED
))
2071 tc_sync_msg(tc
, flags
& PIPE_FLUSH_END_OF_FRAME
? "end of frame" :
2072 flags
& PIPE_FLUSH_DEFERRED
? "deferred fence" : "normal");
2074 if (!(flags
& PIPE_FLUSH_DEFERRED
))
2075 tc_flush_queries(tc
);
2076 pipe
->flush(pipe
, fence
, flags
);
2079 /* This is actually variable-sized, because indirect isn't allocated if it's
2081 struct tc_full_draw_info
{
2082 struct pipe_draw_info draw
;
2083 struct pipe_draw_indirect_info indirect
;
2087 tc_call_draw_vbo(struct pipe_context
*pipe
, union tc_payload
*payload
)
2089 struct tc_full_draw_info
*info
= (struct tc_full_draw_info
*)payload
;
2091 pipe
->draw_vbo(pipe
, &info
->draw
);
2092 pipe_so_target_reference(&info
->draw
.count_from_stream_output
, NULL
);
2093 if (info
->draw
.index_size
)
2094 pipe_resource_reference(&info
->draw
.index
.resource
, NULL
);
2095 if (info
->draw
.indirect
) {
2096 pipe_resource_reference(&info
->indirect
.buffer
, NULL
);
2097 pipe_resource_reference(&info
->indirect
.indirect_draw_count
, NULL
);
2101 static struct tc_full_draw_info
*
2102 tc_add_draw_vbo(struct pipe_context
*_pipe
, bool indirect
)
2104 return (struct tc_full_draw_info
*)
2105 tc_add_sized_call(threaded_context(_pipe
), TC_CALL_draw_vbo
,
2106 indirect
? sizeof(struct tc_full_draw_info
) :
2107 sizeof(struct pipe_draw_info
));
2111 tc_draw_vbo(struct pipe_context
*_pipe
, const struct pipe_draw_info
*info
)
2113 struct threaded_context
*tc
= threaded_context(_pipe
);
2114 struct pipe_draw_indirect_info
*indirect
= info
->indirect
;
2115 unsigned index_size
= info
->index_size
;
2116 bool has_user_indices
= info
->has_user_indices
;
2118 if (index_size
&& has_user_indices
) {
2119 unsigned size
= info
->count
* index_size
;
2120 struct pipe_resource
*buffer
= NULL
;
2123 tc_assert(!indirect
);
2125 /* This must be done before adding draw_vbo, because it could generate
2126 * e.g. transfer_unmap and flush partially-uninitialized draw_vbo
2127 * to the driver if it was done afterwards.
2129 u_upload_data(tc
->base
.stream_uploader
, 0, size
, 4,
2130 (uint8_t*)info
->index
.user
+ info
->start
* index_size
,
2132 if (unlikely(!buffer
))
2135 struct tc_full_draw_info
*p
= tc_add_draw_vbo(_pipe
, false);
2136 p
->draw
.count_from_stream_output
= NULL
;
2137 pipe_so_target_reference(&p
->draw
.count_from_stream_output
,
2138 info
->count_from_stream_output
);
2139 memcpy(&p
->draw
, info
, sizeof(*info
));
2140 p
->draw
.has_user_indices
= false;
2141 p
->draw
.index
.resource
= buffer
;
2142 p
->draw
.start
= offset
>> util_logbase2(index_size
);
2144 /* Non-indexed call or indexed with a real index buffer. */
2145 struct tc_full_draw_info
*p
= tc_add_draw_vbo(_pipe
, indirect
!= NULL
);
2146 p
->draw
.count_from_stream_output
= NULL
;
2147 pipe_so_target_reference(&p
->draw
.count_from_stream_output
,
2148 info
->count_from_stream_output
);
2150 tc_set_resource_reference(&p
->draw
.index
.resource
,
2151 info
->index
.resource
);
2153 memcpy(&p
->draw
, info
, sizeof(*info
));
2156 tc_set_resource_reference(&p
->draw
.indirect
->buffer
, indirect
->buffer
);
2157 tc_set_resource_reference(&p
->indirect
.indirect_draw_count
,
2158 indirect
->indirect_draw_count
);
2159 memcpy(&p
->indirect
, indirect
, sizeof(*indirect
));
2160 p
->draw
.indirect
= &p
->indirect
;
2166 tc_call_launch_grid(struct pipe_context
*pipe
, union tc_payload
*payload
)
2168 struct pipe_grid_info
*p
= (struct pipe_grid_info
*)payload
;
2170 pipe
->launch_grid(pipe
, p
);
2171 pipe_resource_reference(&p
->indirect
, NULL
);
2175 tc_launch_grid(struct pipe_context
*_pipe
,
2176 const struct pipe_grid_info
*info
)
2178 struct threaded_context
*tc
= threaded_context(_pipe
);
2179 struct pipe_grid_info
*p
= tc_add_struct_typed_call(tc
, TC_CALL_launch_grid
,
2181 assert(info
->input
== NULL
);
2183 tc_set_resource_reference(&p
->indirect
, info
->indirect
);
2184 memcpy(p
, info
, sizeof(*info
));
2188 tc_call_resource_copy_region(struct pipe_context
*pipe
, union tc_payload
*payload
)
2190 struct tc_resource_copy_region
*p
= (struct tc_resource_copy_region
*)payload
;
2192 pipe
->resource_copy_region(pipe
, p
->dst
, p
->dst_level
, p
->dstx
, p
->dsty
,
2193 p
->dstz
, p
->src
, p
->src_level
, &p
->src_box
);
2194 pipe_resource_reference(&p
->dst
, NULL
);
2195 pipe_resource_reference(&p
->src
, NULL
);
2199 tc_resource_copy_region(struct pipe_context
*_pipe
,
2200 struct pipe_resource
*dst
, unsigned dst_level
,
2201 unsigned dstx
, unsigned dsty
, unsigned dstz
,
2202 struct pipe_resource
*src
, unsigned src_level
,
2203 const struct pipe_box
*src_box
)
2205 struct threaded_context
*tc
= threaded_context(_pipe
);
2206 struct threaded_resource
*tdst
= threaded_resource(dst
);
2207 struct tc_resource_copy_region
*p
=
2208 tc_add_struct_typed_call(tc
, TC_CALL_resource_copy_region
,
2209 tc_resource_copy_region
);
2211 tc_set_resource_reference(&p
->dst
, dst
);
2212 p
->dst_level
= dst_level
;
2216 tc_set_resource_reference(&p
->src
, src
);
2217 p
->src_level
= src_level
;
2218 p
->src_box
= *src_box
;
2220 if (dst
->target
== PIPE_BUFFER
)
2221 util_range_add(&tdst
->b
, &tdst
->valid_buffer_range
,
2222 dstx
, dstx
+ src_box
->width
);
2226 tc_call_blit(struct pipe_context
*pipe
, union tc_payload
*payload
)
2228 struct pipe_blit_info
*blit
= (struct pipe_blit_info
*)payload
;
2230 pipe
->blit(pipe
, blit
);
2231 pipe_resource_reference(&blit
->dst
.resource
, NULL
);
2232 pipe_resource_reference(&blit
->src
.resource
, NULL
);
2236 tc_blit(struct pipe_context
*_pipe
, const struct pipe_blit_info
*info
)
2238 struct threaded_context
*tc
= threaded_context(_pipe
);
2239 struct pipe_blit_info
*blit
=
2240 tc_add_struct_typed_call(tc
, TC_CALL_blit
, pipe_blit_info
);
2242 tc_set_resource_reference(&blit
->dst
.resource
, info
->dst
.resource
);
2243 tc_set_resource_reference(&blit
->src
.resource
, info
->src
.resource
);
2244 memcpy(blit
, info
, sizeof(*info
));
2247 struct tc_generate_mipmap
{
2248 struct pipe_resource
*res
;
2249 enum pipe_format format
;
2250 unsigned base_level
;
2251 unsigned last_level
;
2252 unsigned first_layer
;
2253 unsigned last_layer
;
2257 tc_call_generate_mipmap(struct pipe_context
*pipe
, union tc_payload
*payload
)
2259 struct tc_generate_mipmap
*p
= (struct tc_generate_mipmap
*)payload
;
2260 ASSERTED
bool result
= pipe
->generate_mipmap(pipe
, p
->res
, p
->format
,
2266 pipe_resource_reference(&p
->res
, NULL
);
2270 tc_generate_mipmap(struct pipe_context
*_pipe
,
2271 struct pipe_resource
*res
,
2272 enum pipe_format format
,
2273 unsigned base_level
,
2274 unsigned last_level
,
2275 unsigned first_layer
,
2276 unsigned last_layer
)
2278 struct threaded_context
*tc
= threaded_context(_pipe
);
2279 struct pipe_context
*pipe
= tc
->pipe
;
2280 struct pipe_screen
*screen
= pipe
->screen
;
2281 unsigned bind
= PIPE_BIND_SAMPLER_VIEW
;
2283 if (util_format_is_depth_or_stencil(format
))
2284 bind
= PIPE_BIND_DEPTH_STENCIL
;
2286 bind
= PIPE_BIND_RENDER_TARGET
;
2288 if (!screen
->is_format_supported(screen
, format
, res
->target
,
2289 res
->nr_samples
, res
->nr_storage_samples
,
2293 struct tc_generate_mipmap
*p
=
2294 tc_add_struct_typed_call(tc
, TC_CALL_generate_mipmap
, tc_generate_mipmap
);
2296 tc_set_resource_reference(&p
->res
, res
);
2298 p
->base_level
= base_level
;
2299 p
->last_level
= last_level
;
2300 p
->first_layer
= first_layer
;
2301 p
->last_layer
= last_layer
;
2306 tc_call_flush_resource(struct pipe_context
*pipe
, union tc_payload
*payload
)
2308 pipe
->flush_resource(pipe
, payload
->resource
);
2309 pipe_resource_reference(&payload
->resource
, NULL
);
2313 tc_flush_resource(struct pipe_context
*_pipe
,
2314 struct pipe_resource
*resource
)
2316 struct threaded_context
*tc
= threaded_context(_pipe
);
2317 union tc_payload
*payload
= tc_add_small_call(tc
, TC_CALL_flush_resource
);
2319 tc_set_resource_reference(&payload
->resource
, resource
);
2323 tc_call_invalidate_resource(struct pipe_context
*pipe
, union tc_payload
*payload
)
2325 pipe
->invalidate_resource(pipe
, payload
->resource
);
2326 pipe_resource_reference(&payload
->resource
, NULL
);
2330 tc_invalidate_resource(struct pipe_context
*_pipe
,
2331 struct pipe_resource
*resource
)
2333 struct threaded_context
*tc
= threaded_context(_pipe
);
2335 if (resource
->target
== PIPE_BUFFER
) {
2336 tc_invalidate_buffer(tc
, threaded_resource(resource
));
2340 union tc_payload
*payload
= tc_add_small_call(tc
, TC_CALL_invalidate_resource
);
2341 tc_set_resource_reference(&payload
->resource
, resource
);
2346 union pipe_color_union color
;
2352 tc_call_clear(struct pipe_context
*pipe
, union tc_payload
*payload
)
2354 struct tc_clear
*p
= (struct tc_clear
*)payload
;
2355 pipe
->clear(pipe
, p
->buffers
, &p
->color
, p
->depth
, p
->stencil
);
2359 tc_clear(struct pipe_context
*_pipe
, unsigned buffers
,
2360 const union pipe_color_union
*color
, double depth
,
2363 struct threaded_context
*tc
= threaded_context(_pipe
);
2364 struct tc_clear
*p
= tc_add_struct_typed_call(tc
, TC_CALL_clear
, tc_clear
);
2366 p
->buffers
= buffers
;
2369 p
->stencil
= stencil
;
2373 tc_clear_render_target(struct pipe_context
*_pipe
,
2374 struct pipe_surface
*dst
,
2375 const union pipe_color_union
*color
,
2376 unsigned dstx
, unsigned dsty
,
2377 unsigned width
, unsigned height
,
2378 bool render_condition_enabled
)
2380 struct threaded_context
*tc
= threaded_context(_pipe
);
2381 struct pipe_context
*pipe
= tc
->pipe
;
2384 pipe
->clear_render_target(pipe
, dst
, color
, dstx
, dsty
, width
, height
,
2385 render_condition_enabled
);
2389 tc_clear_depth_stencil(struct pipe_context
*_pipe
,
2390 struct pipe_surface
*dst
, unsigned clear_flags
,
2391 double depth
, unsigned stencil
, unsigned dstx
,
2392 unsigned dsty
, unsigned width
, unsigned height
,
2393 bool render_condition_enabled
)
2395 struct threaded_context
*tc
= threaded_context(_pipe
);
2396 struct pipe_context
*pipe
= tc
->pipe
;
2399 pipe
->clear_depth_stencil(pipe
, dst
, clear_flags
, depth
, stencil
,
2400 dstx
, dsty
, width
, height
,
2401 render_condition_enabled
);
2404 struct tc_clear_buffer
{
2405 struct pipe_resource
*res
;
2408 char clear_value
[16];
2409 int clear_value_size
;
2413 tc_call_clear_buffer(struct pipe_context
*pipe
, union tc_payload
*payload
)
2415 struct tc_clear_buffer
*p
= (struct tc_clear_buffer
*)payload
;
2417 pipe
->clear_buffer(pipe
, p
->res
, p
->offset
, p
->size
, p
->clear_value
,
2418 p
->clear_value_size
);
2419 pipe_resource_reference(&p
->res
, NULL
);
2423 tc_clear_buffer(struct pipe_context
*_pipe
, struct pipe_resource
*res
,
2424 unsigned offset
, unsigned size
,
2425 const void *clear_value
, int clear_value_size
)
2427 struct threaded_context
*tc
= threaded_context(_pipe
);
2428 struct threaded_resource
*tres
= threaded_resource(res
);
2429 struct tc_clear_buffer
*p
=
2430 tc_add_struct_typed_call(tc
, TC_CALL_clear_buffer
, tc_clear_buffer
);
2432 tc_set_resource_reference(&p
->res
, res
);
2435 memcpy(p
->clear_value
, clear_value
, clear_value_size
);
2436 p
->clear_value_size
= clear_value_size
;
2438 util_range_add(&tres
->b
, &tres
->valid_buffer_range
, offset
, offset
+ size
);
2441 struct tc_clear_texture
{
2442 struct pipe_resource
*res
;
2444 struct pipe_box box
;
2449 tc_call_clear_texture(struct pipe_context
*pipe
, union tc_payload
*payload
)
2451 struct tc_clear_texture
*p
= (struct tc_clear_texture
*)payload
;
2453 pipe
->clear_texture(pipe
, p
->res
, p
->level
, &p
->box
, p
->data
);
2454 pipe_resource_reference(&p
->res
, NULL
);
2458 tc_clear_texture(struct pipe_context
*_pipe
, struct pipe_resource
*res
,
2459 unsigned level
, const struct pipe_box
*box
, const void *data
)
2461 struct threaded_context
*tc
= threaded_context(_pipe
);
2462 struct tc_clear_texture
*p
=
2463 tc_add_struct_typed_call(tc
, TC_CALL_clear_texture
, tc_clear_texture
);
2465 tc_set_resource_reference(&p
->res
, res
);
2468 memcpy(p
->data
, data
,
2469 util_format_get_blocksize(res
->format
));
2472 struct tc_resource_commit
{
2473 struct pipe_resource
*res
;
2475 struct pipe_box box
;
2480 tc_call_resource_commit(struct pipe_context
*pipe
, union tc_payload
*payload
)
2482 struct tc_resource_commit
*p
= (struct tc_resource_commit
*)payload
;
2484 pipe
->resource_commit(pipe
, p
->res
, p
->level
, &p
->box
, p
->commit
);
2485 pipe_resource_reference(&p
->res
, NULL
);
2489 tc_resource_commit(struct pipe_context
*_pipe
, struct pipe_resource
*res
,
2490 unsigned level
, struct pipe_box
*box
, bool commit
)
2492 struct threaded_context
*tc
= threaded_context(_pipe
);
2493 struct tc_resource_commit
*p
=
2494 tc_add_struct_typed_call(tc
, TC_CALL_resource_commit
, tc_resource_commit
);
2496 tc_set_resource_reference(&p
->res
, res
);
2500 return true; /* we don't care about the return value for this call */
2504 /********************************************************************
2508 struct tc_callback_payload
{
2509 void (*fn
)(void *data
);
2514 tc_call_callback(UNUSED
struct pipe_context
*pipe
, union tc_payload
*payload
)
2516 struct tc_callback_payload
*p
= (struct tc_callback_payload
*)payload
;
2522 tc_callback(struct pipe_context
*_pipe
, void (*fn
)(void *), void *data
,
2525 struct threaded_context
*tc
= threaded_context(_pipe
);
2527 if (asap
&& tc_is_sync(tc
)) {
2532 struct tc_callback_payload
*p
=
2533 tc_add_struct_typed_call(tc
, TC_CALL_callback
, tc_callback_payload
);
2539 /********************************************************************
2544 tc_destroy(struct pipe_context
*_pipe
)
2546 struct threaded_context
*tc
= threaded_context(_pipe
);
2547 struct pipe_context
*pipe
= tc
->pipe
;
2549 if (tc
->base
.const_uploader
&&
2550 tc
->base
.stream_uploader
!= tc
->base
.const_uploader
)
2551 u_upload_destroy(tc
->base
.const_uploader
);
2553 if (tc
->base
.stream_uploader
)
2554 u_upload_destroy(tc
->base
.stream_uploader
);
2558 if (util_queue_is_initialized(&tc
->queue
)) {
2559 util_queue_destroy(&tc
->queue
);
2561 for (unsigned i
= 0; i
< TC_MAX_BATCHES
; i
++) {
2562 util_queue_fence_destroy(&tc
->batch_slots
[i
].fence
);
2563 assert(!tc
->batch_slots
[i
].token
);
2567 slab_destroy_child(&tc
->pool_transfers
);
2568 assert(tc
->batch_slots
[tc
->next
].num_total_call_slots
== 0);
2569 pipe
->destroy(pipe
);
2570 os_free_aligned(tc
);
2573 static const tc_execute execute_func
[TC_NUM_CALLS
] = {
2574 #define CALL(name) tc_call_##name,
2575 #include "u_threaded_context_calls.h"
2580 * Wrap an existing pipe_context into a threaded_context.
2582 * \param pipe pipe_context to wrap
2583 * \param parent_transfer_pool parent slab pool set up for creating pipe_-
2584 * transfer objects; the driver should have one
2586 * \param replace_buffer callback for replacing a pipe_resource's storage
2587 * with another pipe_resource's storage.
2588 * \param out if successful, the threaded_context will be returned here in
2589 * addition to the return value if "out" != NULL
2591 struct pipe_context
*
2592 threaded_context_create(struct pipe_context
*pipe
,
2593 struct slab_parent_pool
*parent_transfer_pool
,
2594 tc_replace_buffer_storage_func replace_buffer
,
2595 tc_create_fence_func create_fence
,
2596 struct threaded_context
**out
)
2598 struct threaded_context
*tc
;
2600 STATIC_ASSERT(sizeof(union tc_payload
) <= 8);
2601 STATIC_ASSERT(sizeof(struct tc_call
) <= 16);
2608 if (!debug_get_bool_option("GALLIUM_THREAD", util_cpu_caps
.nr_cpus
> 1))
2611 tc
= os_malloc_aligned(sizeof(struct threaded_context
), 16);
2613 pipe
->destroy(pipe
);
2616 memset(tc
, 0, sizeof(*tc
));
2618 assert((uintptr_t)tc
% 16 == 0);
2619 /* These should be static asserts, but they don't work with MSVC */
2620 assert(offsetof(struct threaded_context
, batch_slots
) % 16 == 0);
2621 assert(offsetof(struct threaded_context
, batch_slots
[0].call
) % 16 == 0);
2622 assert(offsetof(struct threaded_context
, batch_slots
[0].call
[1]) % 16 == 0);
2623 assert(offsetof(struct threaded_context
, batch_slots
[1].call
) % 16 == 0);
2625 /* The driver context isn't wrapped, so set its "priv" to NULL. */
2629 tc
->replace_buffer_storage
= replace_buffer
;
2630 tc
->create_fence
= create_fence
;
2631 tc
->map_buffer_alignment
=
2632 pipe
->screen
->get_param(pipe
->screen
, PIPE_CAP_MIN_MAP_BUFFER_ALIGNMENT
);
2633 tc
->base
.priv
= pipe
; /* priv points to the wrapped driver context */
2634 tc
->base
.screen
= pipe
->screen
;
2635 tc
->base
.destroy
= tc_destroy
;
2636 tc
->base
.callback
= tc_callback
;
2638 tc
->base
.stream_uploader
= u_upload_clone(&tc
->base
, pipe
->stream_uploader
);
2639 if (pipe
->stream_uploader
== pipe
->const_uploader
)
2640 tc
->base
.const_uploader
= tc
->base
.stream_uploader
;
2642 tc
->base
.const_uploader
= u_upload_clone(&tc
->base
, pipe
->const_uploader
);
2644 if (!tc
->base
.stream_uploader
|| !tc
->base
.const_uploader
)
2647 /* The queue size is the number of batches "waiting". Batches are removed
2648 * from the queue before being executed, so keep one tc_batch slot for that
2649 * execution. Also, keep one unused slot for an unflushed batch.
2651 if (!util_queue_init(&tc
->queue
, "gdrv", TC_MAX_BATCHES
- 2, 1, 0))
2654 for (unsigned i
= 0; i
< TC_MAX_BATCHES
; i
++) {
2655 tc
->batch_slots
[i
].sentinel
= TC_SENTINEL
;
2656 tc
->batch_slots
[i
].pipe
= pipe
;
2657 util_queue_fence_init(&tc
->batch_slots
[i
].fence
);
2660 list_inithead(&tc
->unflushed_queries
);
2662 slab_create_child(&tc
->pool_transfers
, parent_transfer_pool
);
2664 tc
->base
.set_context_param
= tc_set_context_param
; /* always set this */
2666 #define CTX_INIT(_member) \
2667 tc->base._member = tc->pipe->_member ? tc_##_member : NULL
2671 CTX_INIT(launch_grid
);
2672 CTX_INIT(resource_copy_region
);
2675 CTX_INIT(clear_render_target
);
2676 CTX_INIT(clear_depth_stencil
);
2677 CTX_INIT(clear_buffer
);
2678 CTX_INIT(clear_texture
);
2679 CTX_INIT(flush_resource
);
2680 CTX_INIT(generate_mipmap
);
2681 CTX_INIT(render_condition
);
2682 CTX_INIT(create_query
);
2683 CTX_INIT(create_batch_query
);
2684 CTX_INIT(destroy_query
);
2685 CTX_INIT(begin_query
);
2686 CTX_INIT(end_query
);
2687 CTX_INIT(get_query_result
);
2688 CTX_INIT(get_query_result_resource
);
2689 CTX_INIT(set_active_query_state
);
2690 CTX_INIT(create_blend_state
);
2691 CTX_INIT(bind_blend_state
);
2692 CTX_INIT(delete_blend_state
);
2693 CTX_INIT(create_sampler_state
);
2694 CTX_INIT(bind_sampler_states
);
2695 CTX_INIT(delete_sampler_state
);
2696 CTX_INIT(create_rasterizer_state
);
2697 CTX_INIT(bind_rasterizer_state
);
2698 CTX_INIT(delete_rasterizer_state
);
2699 CTX_INIT(create_depth_stencil_alpha_state
);
2700 CTX_INIT(bind_depth_stencil_alpha_state
);
2701 CTX_INIT(delete_depth_stencil_alpha_state
);
2702 CTX_INIT(create_fs_state
);
2703 CTX_INIT(bind_fs_state
);
2704 CTX_INIT(delete_fs_state
);
2705 CTX_INIT(create_vs_state
);
2706 CTX_INIT(bind_vs_state
);
2707 CTX_INIT(delete_vs_state
);
2708 CTX_INIT(create_gs_state
);
2709 CTX_INIT(bind_gs_state
);
2710 CTX_INIT(delete_gs_state
);
2711 CTX_INIT(create_tcs_state
);
2712 CTX_INIT(bind_tcs_state
);
2713 CTX_INIT(delete_tcs_state
);
2714 CTX_INIT(create_tes_state
);
2715 CTX_INIT(bind_tes_state
);
2716 CTX_INIT(delete_tes_state
);
2717 CTX_INIT(create_compute_state
);
2718 CTX_INIT(bind_compute_state
);
2719 CTX_INIT(delete_compute_state
);
2720 CTX_INIT(create_vertex_elements_state
);
2721 CTX_INIT(bind_vertex_elements_state
);
2722 CTX_INIT(delete_vertex_elements_state
);
2723 CTX_INIT(set_blend_color
);
2724 CTX_INIT(set_stencil_ref
);
2725 CTX_INIT(set_sample_mask
);
2726 CTX_INIT(set_min_samples
);
2727 CTX_INIT(set_clip_state
);
2728 CTX_INIT(set_constant_buffer
);
2729 CTX_INIT(set_framebuffer_state
);
2730 CTX_INIT(set_polygon_stipple
);
2731 CTX_INIT(set_scissor_states
);
2732 CTX_INIT(set_viewport_states
);
2733 CTX_INIT(set_window_rectangles
);
2734 CTX_INIT(set_sampler_views
);
2735 CTX_INIT(set_tess_state
);
2736 CTX_INIT(set_shader_buffers
);
2737 CTX_INIT(set_shader_images
);
2738 CTX_INIT(set_vertex_buffers
);
2739 CTX_INIT(create_stream_output_target
);
2740 CTX_INIT(stream_output_target_destroy
);
2741 CTX_INIT(set_stream_output_targets
);
2742 CTX_INIT(create_sampler_view
);
2743 CTX_INIT(sampler_view_destroy
);
2744 CTX_INIT(create_surface
);
2745 CTX_INIT(surface_destroy
);
2746 CTX_INIT(transfer_map
);
2747 CTX_INIT(transfer_flush_region
);
2748 CTX_INIT(transfer_unmap
);
2749 CTX_INIT(buffer_subdata
);
2750 CTX_INIT(texture_subdata
);
2751 CTX_INIT(texture_barrier
);
2752 CTX_INIT(memory_barrier
);
2753 CTX_INIT(resource_commit
);
2754 CTX_INIT(create_video_codec
);
2755 CTX_INIT(create_video_buffer
);
2756 CTX_INIT(set_compute_resources
);
2757 CTX_INIT(set_global_binding
);
2758 CTX_INIT(get_sample_position
);
2759 CTX_INIT(invalidate_resource
);
2760 CTX_INIT(get_device_reset_status
);
2761 CTX_INIT(set_device_reset_callback
);
2762 CTX_INIT(dump_debug_state
);
2763 CTX_INIT(set_log_context
);
2764 CTX_INIT(emit_string_marker
);
2765 CTX_INIT(set_debug_callback
);
2766 CTX_INIT(create_fence_fd
);
2767 CTX_INIT(fence_server_sync
);
2768 CTX_INIT(fence_server_signal
);
2769 CTX_INIT(get_timestamp
);
2770 CTX_INIT(create_texture_handle
);
2771 CTX_INIT(delete_texture_handle
);
2772 CTX_INIT(make_texture_handle_resident
);
2773 CTX_INIT(create_image_handle
);
2774 CTX_INIT(delete_image_handle
);
2775 CTX_INIT(make_image_handle_resident
);
2784 tc_destroy(&tc
->base
);