2 * Copyright 2013 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
28 /* Recommended maximum sizes for optimal performance.
29 * Fall back to compute or SDMA if the size is greater.
31 #define CP_DMA_COPY_PERF_THRESHOLD (64 * 1024) /* copied from Vulkan */
32 #define CP_DMA_CLEAR_PERF_THRESHOLD (32 * 1024) /* guess (clear is much slower) */
34 /* Set this if you want the ME to wait until CP DMA is done.
35 * It should be set on the last CP DMA packet. */
36 #define CP_DMA_SYNC (1 << 0)
38 /* Set this if the source data was used as a destination in a previous CP DMA
39 * packet. It's for preventing a read-after-write (RAW) hazard between two
41 #define CP_DMA_RAW_WAIT (1 << 1)
42 #define CP_DMA_USE_L2 (1 << 2) /* CIK+ */
43 #define CP_DMA_CLEAR (1 << 3)
45 /* The max number of bytes that can be copied per packet. */
46 static inline unsigned cp_dma_max_byte_count(struct si_context
*sctx
)
48 unsigned max
= sctx
->chip_class
>= GFX9
?
49 S_414_BYTE_COUNT_GFX9(~0u) :
50 S_414_BYTE_COUNT_GFX6(~0u);
52 /* make it aligned for optimal performance */
53 return max
& ~(SI_CPDMA_ALIGNMENT
- 1);
57 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
58 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
61 static void si_emit_cp_dma(struct si_context
*sctx
, uint64_t dst_va
,
62 uint64_t src_va
, unsigned size
, unsigned flags
,
63 enum si_coherency coher
)
65 struct radeon_cmdbuf
*cs
= sctx
->gfx_cs
;
66 uint32_t header
= 0, command
= 0;
68 assert(size
<= cp_dma_max_byte_count(sctx
));
70 if (sctx
->chip_class
>= GFX9
)
71 command
|= S_414_BYTE_COUNT_GFX9(size
);
73 command
|= S_414_BYTE_COUNT_GFX6(size
);
76 if (flags
& CP_DMA_SYNC
)
77 header
|= S_411_CP_SYNC(1);
79 if (sctx
->chip_class
>= GFX9
)
80 command
|= S_414_DISABLE_WR_CONFIRM_GFX9(1);
82 command
|= S_414_DISABLE_WR_CONFIRM_GFX6(1);
85 if (flags
& CP_DMA_RAW_WAIT
)
86 command
|= S_414_RAW_WAIT(1);
88 /* Src and dst flags. */
89 if (sctx
->chip_class
>= GFX9
&& !(flags
& CP_DMA_CLEAR
) &&
91 header
|= S_411_DST_SEL(V_411_NOWHERE
); /* prefetch only */
92 else if (flags
& CP_DMA_USE_L2
)
93 header
|= S_411_DST_SEL(V_411_DST_ADDR_TC_L2
);
95 if (flags
& CP_DMA_CLEAR
)
96 header
|= S_411_SRC_SEL(V_411_DATA
);
97 else if (flags
& CP_DMA_USE_L2
)
98 header
|= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2
);
100 if (sctx
->chip_class
>= CIK
) {
101 radeon_emit(cs
, PKT3(PKT3_DMA_DATA
, 5, 0));
102 radeon_emit(cs
, header
);
103 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
104 radeon_emit(cs
, src_va
>> 32); /* SRC_ADDR_HI [31:0] */
105 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
106 radeon_emit(cs
, dst_va
>> 32); /* DST_ADDR_HI [31:0] */
107 radeon_emit(cs
, command
);
109 header
|= S_411_SRC_ADDR_HI(src_va
>> 32);
111 radeon_emit(cs
, PKT3(PKT3_CP_DMA
, 4, 0));
112 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
113 radeon_emit(cs
, header
); /* SRC_ADDR_HI [15:0] + flags. */
114 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
115 radeon_emit(cs
, (dst_va
>> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
116 radeon_emit(cs
, command
);
119 /* CP DMA is executed in ME, but index buffers are read by PFP.
120 * This ensures that ME (CP DMA) is idle before PFP starts fetching
121 * indices. If we wanted to execute CP DMA in PFP, this packet
124 if (coher
== SI_COHERENCY_SHADER
&& flags
& CP_DMA_SYNC
) {
125 radeon_emit(cs
, PKT3(PKT3_PFP_SYNC_ME
, 0, 0));
130 void si_cp_dma_wait_for_idle(struct si_context
*sctx
)
132 /* Issue a dummy DMA that copies zero bytes.
134 * The DMA engine will see that there's no work to do and skip this
135 * DMA request, however, the CP will see the sync flag and still wait
136 * for all DMAs to complete.
138 si_emit_cp_dma(sctx
, 0, 0, 0, CP_DMA_SYNC
, SI_COHERENCY_NONE
);
141 static unsigned get_flush_flags(struct si_context
*sctx
, enum si_coherency coher
)
145 case SI_COHERENCY_NONE
:
147 case SI_COHERENCY_SHADER
:
148 return SI_CONTEXT_INV_SMEM_L1
|
149 SI_CONTEXT_INV_VMEM_L1
|
150 (sctx
->chip_class
== SI
? SI_CONTEXT_INV_GLOBAL_L2
: 0);
151 case SI_COHERENCY_CB_META
:
152 return SI_CONTEXT_FLUSH_AND_INV_CB
;
156 static unsigned get_tc_l2_flag(struct si_context
*sctx
, enum si_coherency coher
)
158 if ((sctx
->chip_class
>= GFX9
&& coher
== SI_COHERENCY_CB_META
) ||
159 (sctx
->chip_class
>= CIK
&& coher
== SI_COHERENCY_SHADER
))
160 return CP_DMA_USE_L2
;
165 static void si_cp_dma_prepare(struct si_context
*sctx
, struct pipe_resource
*dst
,
166 struct pipe_resource
*src
, unsigned byte_count
,
167 uint64_t remaining_size
, unsigned user_flags
,
168 bool *is_first
, unsigned *packet_flags
)
170 /* Fast exit for a CPDMA prefetch. */
171 if ((user_flags
& SI_CPDMA_SKIP_ALL
) == SI_CPDMA_SKIP_ALL
) {
176 if (!(user_flags
& SI_CPDMA_SKIP_BO_LIST_UPDATE
)) {
177 /* Count memory usage in so that need_cs_space can take it into account. */
178 si_context_add_resource_size(sctx
, dst
);
180 si_context_add_resource_size(sctx
, src
);
183 if (!(user_flags
& SI_CPDMA_SKIP_CHECK_CS_SPACE
))
184 si_need_gfx_cs_space(sctx
);
186 /* This must be done after need_cs_space. */
187 if (!(user_flags
& SI_CPDMA_SKIP_BO_LIST_UPDATE
)) {
188 radeon_add_to_buffer_list(sctx
, sctx
->gfx_cs
,
190 RADEON_USAGE_WRITE
, RADEON_PRIO_CP_DMA
);
192 radeon_add_to_buffer_list(sctx
, sctx
->gfx_cs
,
194 RADEON_USAGE_READ
, RADEON_PRIO_CP_DMA
);
197 /* Flush the caches for the first copy only.
198 * Also wait for the previous CP DMA operations.
200 if (!(user_flags
& SI_CPDMA_SKIP_GFX_SYNC
) && sctx
->flags
)
201 si_emit_cache_flush(sctx
);
203 if (!(user_flags
& SI_CPDMA_SKIP_SYNC_BEFORE
) && *is_first
)
204 *packet_flags
|= CP_DMA_RAW_WAIT
;
208 /* Do the synchronization after the last dma, so that all data
209 * is written to memory.
211 if (!(user_flags
& SI_CPDMA_SKIP_SYNC_AFTER
) &&
212 byte_count
== remaining_size
)
213 *packet_flags
|= CP_DMA_SYNC
;
216 void si_clear_buffer(struct si_context
*sctx
, struct pipe_resource
*dst
,
217 uint64_t offset
, uint64_t size
, unsigned value
,
218 enum si_coherency coher
, enum si_method xfer
)
220 struct radeon_winsys
*ws
= sctx
->ws
;
221 struct r600_resource
*rdst
= r600_resource(dst
);
222 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, coher
);
223 unsigned flush_flags
= get_flush_flags(sctx
, coher
);
224 uint64_t dma_clear_size
;
225 bool is_first
= true;
230 dma_clear_size
= size
& ~3ull;
232 /* Mark the buffer range of destination as valid (initialized),
233 * so that transfer_map knows it should wait for the GPU when mapping
235 util_range_add(&rdst
->valid_buffer_range
, offset
,
236 offset
+ dma_clear_size
);
238 /* dma_clear_buffer can use clear_buffer on failure. Make sure that
239 * doesn't happen. We don't want an infinite recursion: */
241 !(dst
->flags
& PIPE_RESOURCE_FLAG_SPARSE
) &&
243 /* CP DMA is very slow. Always use SDMA for big clears. This
244 * alone improves DeusEx:MD performance by 70%. */
245 (size
> CP_DMA_CLEAR_PERF_THRESHOLD
||
246 /* Buffers not used by the GFX IB yet will be cleared by SDMA.
247 * This happens to move most buffer clears to SDMA, including
248 * DCC and CMASK clears, because pipe->clear clears them before
249 * si_emit_framebuffer_state (in a draw call) adds them.
250 * For example, DeusEx:MD has 21 buffer clears per frame and all
251 * of them are moved to SDMA thanks to this. */
252 !ws
->cs_is_buffer_referenced(sctx
->gfx_cs
, rdst
->buf
,
253 RADEON_USAGE_READWRITE
)) &&
254 /* bypass sdma transfer with param xfer */
255 (xfer
!= SI_METHOD_CP_DMA
)) {
256 sctx
->dma_clear_buffer(sctx
, dst
, offset
, dma_clear_size
, value
);
258 offset
+= dma_clear_size
;
259 size
-= dma_clear_size
;
260 } else if (dma_clear_size
>= 4) {
261 uint64_t va
= rdst
->gpu_address
+ offset
;
263 offset
+= dma_clear_size
;
264 size
-= dma_clear_size
;
266 /* Flush the caches. */
267 sctx
->flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
268 SI_CONTEXT_CS_PARTIAL_FLUSH
| flush_flags
;
270 while (dma_clear_size
) {
271 unsigned byte_count
= MIN2(dma_clear_size
, cp_dma_max_byte_count(sctx
));
272 unsigned dma_flags
= tc_l2_flag
| CP_DMA_CLEAR
;
274 si_cp_dma_prepare(sctx
, dst
, NULL
, byte_count
, dma_clear_size
, 0,
275 &is_first
, &dma_flags
);
277 /* Emit the clear packet. */
278 si_emit_cp_dma(sctx
, va
, value
, byte_count
, dma_flags
, coher
);
280 dma_clear_size
-= byte_count
;
285 rdst
->TC_L2_dirty
= true;
287 /* If it's not a framebuffer fast clear... */
288 if (coher
== SI_COHERENCY_SHADER
)
289 sctx
->num_cp_dma_calls
++;
293 /* Handle non-dword alignment.
295 * This function is called for embedded texture metadata clears,
296 * but those should always be properly aligned. */
297 assert(dst
->target
== PIPE_BUFFER
);
300 pipe_buffer_write(&sctx
->b
, dst
, offset
, size
, &value
);
304 static void si_pipe_clear_buffer(struct pipe_context
*ctx
,
305 struct pipe_resource
*dst
,
306 unsigned offset
, unsigned size
,
307 const void *clear_value_ptr
,
308 int clear_value_size
)
310 struct si_context
*sctx
= (struct si_context
*)ctx
;
311 uint32_t dword_value
;
314 assert(offset
% clear_value_size
== 0);
315 assert(size
% clear_value_size
== 0);
317 if (clear_value_size
> 4) {
318 const uint32_t *u32
= clear_value_ptr
;
319 bool clear_dword_duplicated
= true;
321 /* See if we can lower large fills to dword fills. */
322 for (i
= 1; i
< clear_value_size
/ 4; i
++)
323 if (u32
[0] != u32
[i
]) {
324 clear_dword_duplicated
= false;
328 if (!clear_dword_duplicated
) {
329 /* Use transform feedback for 64-bit, 96-bit, and
332 union pipe_color_union clear_value
;
334 memcpy(&clear_value
, clear_value_ptr
, clear_value_size
);
335 si_blitter_begin(sctx
, SI_DISABLE_RENDER_COND
);
336 util_blitter_clear_buffer(sctx
->blitter
, dst
, offset
,
337 size
, clear_value_size
/ 4,
339 si_blitter_end(sctx
);
344 /* Expand the clear value to a dword. */
345 switch (clear_value_size
) {
347 dword_value
= *(uint8_t*)clear_value_ptr
;
348 dword_value
|= (dword_value
<< 8) |
349 (dword_value
<< 16) |
353 dword_value
= *(uint16_t*)clear_value_ptr
;
354 dword_value
|= dword_value
<< 16;
357 dword_value
= *(uint32_t*)clear_value_ptr
;
360 si_clear_buffer(sctx
, dst
, offset
, size
, dword_value
,
361 SI_COHERENCY_SHADER
, SI_METHOD_BEST
);
365 * Realign the CP DMA engine. This must be done after a copy with an unaligned
368 * \param size Remaining size to the CP DMA alignment.
370 static void si_cp_dma_realign_engine(struct si_context
*sctx
, unsigned size
,
371 unsigned user_flags
, bool *is_first
)
374 unsigned dma_flags
= 0;
375 unsigned scratch_size
= SI_CPDMA_ALIGNMENT
* 2;
377 assert(size
< SI_CPDMA_ALIGNMENT
);
379 /* Use the scratch buffer as the dummy buffer. The 3D engine should be
380 * idle at this point.
382 if (!sctx
->scratch_buffer
||
383 sctx
->scratch_buffer
->b
.b
.width0
< scratch_size
) {
384 r600_resource_reference(&sctx
->scratch_buffer
, NULL
);
385 sctx
->scratch_buffer
=
386 si_aligned_buffer_create(&sctx
->screen
->b
,
387 SI_RESOURCE_FLAG_UNMAPPABLE
,
390 if (!sctx
->scratch_buffer
)
393 si_mark_atom_dirty(sctx
, &sctx
->atoms
.s
.scratch_state
);
396 si_cp_dma_prepare(sctx
, &sctx
->scratch_buffer
->b
.b
,
397 &sctx
->scratch_buffer
->b
.b
, size
, size
, user_flags
,
398 is_first
, &dma_flags
);
400 va
= sctx
->scratch_buffer
->gpu_address
;
401 si_emit_cp_dma(sctx
, va
, va
+ SI_CPDMA_ALIGNMENT
, size
, dma_flags
,
402 SI_COHERENCY_SHADER
);
406 * Do memcpy between buffers using CP DMA.
408 * \param user_flags bitmask of SI_CPDMA_*
410 void si_copy_buffer(struct si_context
*sctx
,
411 struct pipe_resource
*dst
, struct pipe_resource
*src
,
412 uint64_t dst_offset
, uint64_t src_offset
, unsigned size
,
415 uint64_t main_dst_offset
, main_src_offset
;
416 unsigned skipped_size
= 0;
417 unsigned realign_size
= 0;
418 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, SI_COHERENCY_SHADER
);
419 unsigned flush_flags
= get_flush_flags(sctx
, SI_COHERENCY_SHADER
);
420 bool is_first
= true;
425 if (dst
!= src
|| dst_offset
!= src_offset
) {
426 /* Mark the buffer range of destination as valid (initialized),
427 * so that transfer_map knows it should wait for the GPU when mapping
429 util_range_add(&r600_resource(dst
)->valid_buffer_range
, dst_offset
,
433 dst_offset
+= r600_resource(dst
)->gpu_address
;
434 src_offset
+= r600_resource(src
)->gpu_address
;
436 /* The workarounds aren't needed on Fiji and beyond. */
437 if (sctx
->family
<= CHIP_CARRIZO
||
438 sctx
->family
== CHIP_STONEY
) {
439 /* If the size is not aligned, we must add a dummy copy at the end
440 * just to align the internal counter. Otherwise, the DMA engine
441 * would slow down by an order of magnitude for following copies.
443 if (size
% SI_CPDMA_ALIGNMENT
)
444 realign_size
= SI_CPDMA_ALIGNMENT
- (size
% SI_CPDMA_ALIGNMENT
);
446 /* If the copy begins unaligned, we must start copying from the next
447 * aligned block and the skipped part should be copied after everything
448 * else has been copied. Only the src alignment matters, not dst.
450 if (src_offset
% SI_CPDMA_ALIGNMENT
) {
451 skipped_size
= SI_CPDMA_ALIGNMENT
- (src_offset
% SI_CPDMA_ALIGNMENT
);
452 /* The main part will be skipped if the size is too small. */
453 skipped_size
= MIN2(skipped_size
, size
);
454 size
-= skipped_size
;
458 /* Flush the caches. */
459 if (!(user_flags
& SI_CPDMA_SKIP_GFX_SYNC
))
460 sctx
->flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
461 SI_CONTEXT_CS_PARTIAL_FLUSH
| flush_flags
;
463 /* This is the main part doing the copying. Src is always aligned. */
464 main_dst_offset
= dst_offset
+ skipped_size
;
465 main_src_offset
= src_offset
+ skipped_size
;
468 unsigned dma_flags
= tc_l2_flag
;
469 unsigned byte_count
= MIN2(size
, cp_dma_max_byte_count(sctx
));
471 si_cp_dma_prepare(sctx
, dst
, src
, byte_count
,
472 size
+ skipped_size
+ realign_size
,
473 user_flags
, &is_first
, &dma_flags
);
475 si_emit_cp_dma(sctx
, main_dst_offset
, main_src_offset
,
476 byte_count
, dma_flags
, SI_COHERENCY_SHADER
);
479 main_src_offset
+= byte_count
;
480 main_dst_offset
+= byte_count
;
483 /* Copy the part we skipped because src wasn't aligned. */
485 unsigned dma_flags
= tc_l2_flag
;
487 si_cp_dma_prepare(sctx
, dst
, src
, skipped_size
,
488 skipped_size
+ realign_size
, user_flags
,
489 &is_first
, &dma_flags
);
491 si_emit_cp_dma(sctx
, dst_offset
, src_offset
, skipped_size
,
492 dma_flags
, SI_COHERENCY_SHADER
);
495 /* Finally, realign the engine if the size wasn't aligned. */
497 si_cp_dma_realign_engine(sctx
, realign_size
, user_flags
,
501 r600_resource(dst
)->TC_L2_dirty
= true;
503 /* If it's not a prefetch... */
504 if (dst_offset
!= src_offset
)
505 sctx
->num_cp_dma_calls
++;
508 void cik_prefetch_TC_L2_async(struct si_context
*sctx
, struct pipe_resource
*buf
,
509 uint64_t offset
, unsigned size
)
511 assert(sctx
->chip_class
>= CIK
);
513 si_copy_buffer(sctx
, buf
, buf
, offset
, offset
, size
, SI_CPDMA_SKIP_ALL
);
516 static void cik_prefetch_shader_async(struct si_context
*sctx
,
517 struct si_pm4_state
*state
)
519 struct pipe_resource
*bo
= &state
->bo
[0]->b
.b
;
520 assert(state
->nbo
== 1);
522 cik_prefetch_TC_L2_async(sctx
, bo
, 0, bo
->width0
);
525 static void cik_prefetch_VBO_descriptors(struct si_context
*sctx
)
527 if (!sctx
->vertex_elements
)
530 cik_prefetch_TC_L2_async(sctx
, &sctx
->vb_descriptors_buffer
->b
.b
,
531 sctx
->vb_descriptors_offset
,
532 sctx
->vertex_elements
->desc_list_byte_size
);
536 * Prefetch shaders and VBO descriptors.
538 * \param vertex_stage_only Whether only the the API VS and VBO descriptors
539 * should be prefetched.
541 void cik_emit_prefetch_L2(struct si_context
*sctx
, bool vertex_stage_only
)
543 unsigned mask
= sctx
->prefetch_L2_mask
;
546 /* Prefetch shaders and VBO descriptors to TC L2. */
547 if (sctx
->chip_class
>= GFX9
) {
548 /* Choose the right spot for the VBO prefetch. */
549 if (sctx
->tes_shader
.cso
) {
550 if (mask
& SI_PREFETCH_HS
)
551 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.hs
);
552 if (mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
553 cik_prefetch_VBO_descriptors(sctx
);
554 if (vertex_stage_only
) {
555 sctx
->prefetch_L2_mask
&= ~(SI_PREFETCH_HS
|
556 SI_PREFETCH_VBO_DESCRIPTORS
);
560 if (mask
& SI_PREFETCH_GS
)
561 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
562 if (mask
& SI_PREFETCH_VS
)
563 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
564 } else if (sctx
->gs_shader
.cso
) {
565 if (mask
& SI_PREFETCH_GS
)
566 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
567 if (mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
568 cik_prefetch_VBO_descriptors(sctx
);
569 if (vertex_stage_only
) {
570 sctx
->prefetch_L2_mask
&= ~(SI_PREFETCH_GS
|
571 SI_PREFETCH_VBO_DESCRIPTORS
);
575 if (mask
& SI_PREFETCH_VS
)
576 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
578 if (mask
& SI_PREFETCH_VS
)
579 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
580 if (mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
581 cik_prefetch_VBO_descriptors(sctx
);
582 if (vertex_stage_only
) {
583 sctx
->prefetch_L2_mask
&= ~(SI_PREFETCH_VS
|
584 SI_PREFETCH_VBO_DESCRIPTORS
);
590 /* Choose the right spot for the VBO prefetch. */
591 if (sctx
->tes_shader
.cso
) {
592 if (mask
& SI_PREFETCH_LS
)
593 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.ls
);
594 if (mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
595 cik_prefetch_VBO_descriptors(sctx
);
596 if (vertex_stage_only
) {
597 sctx
->prefetch_L2_mask
&= ~(SI_PREFETCH_LS
|
598 SI_PREFETCH_VBO_DESCRIPTORS
);
602 if (mask
& SI_PREFETCH_HS
)
603 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.hs
);
604 if (mask
& SI_PREFETCH_ES
)
605 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.es
);
606 if (mask
& SI_PREFETCH_GS
)
607 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
608 if (mask
& SI_PREFETCH_VS
)
609 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
610 } else if (sctx
->gs_shader
.cso
) {
611 if (mask
& SI_PREFETCH_ES
)
612 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.es
);
613 if (mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
614 cik_prefetch_VBO_descriptors(sctx
);
615 if (vertex_stage_only
) {
616 sctx
->prefetch_L2_mask
&= ~(SI_PREFETCH_ES
|
617 SI_PREFETCH_VBO_DESCRIPTORS
);
621 if (mask
& SI_PREFETCH_GS
)
622 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
623 if (mask
& SI_PREFETCH_VS
)
624 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
626 if (mask
& SI_PREFETCH_VS
)
627 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
628 if (mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
629 cik_prefetch_VBO_descriptors(sctx
);
630 if (vertex_stage_only
) {
631 sctx
->prefetch_L2_mask
&= ~(SI_PREFETCH_VS
|
632 SI_PREFETCH_VBO_DESCRIPTORS
);
638 if (mask
& SI_PREFETCH_PS
)
639 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.ps
);
641 sctx
->prefetch_L2_mask
= 0;
644 void si_init_cp_dma_functions(struct si_context
*sctx
)
646 sctx
->b
.clear_buffer
= si_pipe_clear_buffer
;