2 * Copyright 2013 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * Marek Olšák <maraeo@gmail.com>
29 #include "radeon/r600_cs.h"
31 /* Recommended maximum sizes for optimal performance.
32 * Fall back to compute or SDMA if the size is greater.
34 #define CP_DMA_COPY_PERF_THRESHOLD (64 * 1024) /* copied from Vulkan */
35 #define CP_DMA_CLEAR_PERF_THRESHOLD (32 * 1024) /* guess (clear is much slower) */
37 /* Set this if you want the ME to wait until CP DMA is done.
38 * It should be set on the last CP DMA packet. */
39 #define CP_DMA_SYNC (1 << 0)
41 /* Set this if the source data was used as a destination in a previous CP DMA
42 * packet. It's for preventing a read-after-write (RAW) hazard between two
44 #define CP_DMA_RAW_WAIT (1 << 1)
45 #define CP_DMA_USE_L2 (1 << 2) /* CIK+ */
46 #define CP_DMA_CLEAR (1 << 3)
48 /* The max number of bytes that can be copied per packet. */
49 static inline unsigned cp_dma_max_byte_count(struct si_context
*sctx
)
51 unsigned max
= sctx
->b
.chip_class
>= GFX9
?
52 S_414_BYTE_COUNT_GFX9(~0u) :
53 S_414_BYTE_COUNT_GFX6(~0u);
55 /* make it aligned for optimal performance */
56 return max
& ~(SI_CPDMA_ALIGNMENT
- 1);
60 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
61 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
64 static void si_emit_cp_dma(struct si_context
*sctx
, uint64_t dst_va
,
65 uint64_t src_va
, unsigned size
, unsigned flags
,
66 enum r600_coherency coher
)
68 struct radeon_winsys_cs
*cs
= sctx
->b
.gfx
.cs
;
69 uint32_t header
= 0, command
= 0;
72 assert(size
<= cp_dma_max_byte_count(sctx
));
74 if (sctx
->b
.chip_class
>= GFX9
)
75 command
|= S_414_BYTE_COUNT_GFX9(size
);
77 command
|= S_414_BYTE_COUNT_GFX6(size
);
80 if (flags
& CP_DMA_SYNC
)
81 header
|= S_411_CP_SYNC(1);
83 if (sctx
->b
.chip_class
>= GFX9
)
84 command
|= S_414_DISABLE_WR_CONFIRM_GFX9(1);
86 command
|= S_414_DISABLE_WR_CONFIRM_GFX6(1);
89 if (flags
& CP_DMA_RAW_WAIT
)
90 command
|= S_414_RAW_WAIT(1);
92 /* Src and dst flags. */
93 if (sctx
->b
.chip_class
>= GFX9
&& !(flags
& CP_DMA_CLEAR
) &&
95 header
|= S_411_DSL_SEL(V_411_NOWHERE
); /* prefetch only */
96 else if (flags
& CP_DMA_USE_L2
)
97 header
|= S_411_DSL_SEL(V_411_DST_ADDR_TC_L2
);
99 if (flags
& CP_DMA_CLEAR
)
100 header
|= S_411_SRC_SEL(V_411_DATA
);
101 else if (flags
& CP_DMA_USE_L2
)
102 header
|= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2
);
104 if (sctx
->b
.chip_class
>= CIK
) {
105 radeon_emit(cs
, PKT3(PKT3_DMA_DATA
, 5, 0));
106 radeon_emit(cs
, header
);
107 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
108 radeon_emit(cs
, src_va
>> 32); /* SRC_ADDR_HI [31:0] */
109 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
110 radeon_emit(cs
, dst_va
>> 32); /* DST_ADDR_HI [31:0] */
111 radeon_emit(cs
, command
);
113 header
|= S_411_SRC_ADDR_HI(src_va
>> 32);
115 radeon_emit(cs
, PKT3(PKT3_CP_DMA
, 4, 0));
116 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
117 radeon_emit(cs
, header
); /* SRC_ADDR_HI [15:0] + flags. */
118 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
119 radeon_emit(cs
, (dst_va
>> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
120 radeon_emit(cs
, command
);
123 /* CP DMA is executed in ME, but index buffers are read by PFP.
124 * This ensures that ME (CP DMA) is idle before PFP starts fetching
125 * indices. If we wanted to execute CP DMA in PFP, this packet
128 if (coher
== R600_COHERENCY_SHADER
&& flags
& CP_DMA_SYNC
) {
129 radeon_emit(cs
, PKT3(PKT3_PFP_SYNC_ME
, 0, 0));
134 static unsigned get_flush_flags(struct si_context
*sctx
, enum r600_coherency coher
)
138 case R600_COHERENCY_NONE
:
140 case R600_COHERENCY_SHADER
:
141 return SI_CONTEXT_INV_SMEM_L1
|
142 SI_CONTEXT_INV_VMEM_L1
|
143 (sctx
->b
.chip_class
== SI
? SI_CONTEXT_INV_GLOBAL_L2
: 0);
144 case R600_COHERENCY_CB_META
:
145 return SI_CONTEXT_FLUSH_AND_INV_CB
;
149 static unsigned get_tc_l2_flag(struct si_context
*sctx
, enum r600_coherency coher
)
151 if ((sctx
->b
.chip_class
>= GFX9
&& coher
== R600_COHERENCY_CB_META
) ||
152 (sctx
->b
.chip_class
>= CIK
&& coher
== R600_COHERENCY_SHADER
))
153 return CP_DMA_USE_L2
;
158 static void si_cp_dma_prepare(struct si_context
*sctx
, struct pipe_resource
*dst
,
159 struct pipe_resource
*src
, unsigned byte_count
,
160 uint64_t remaining_size
, unsigned user_flags
,
161 bool *is_first
, unsigned *packet_flags
)
163 /* Fast exit for a CPDMA prefetch. */
164 if ((user_flags
& SI_CPDMA_SKIP_ALL
) == SI_CPDMA_SKIP_ALL
) {
169 if (!(user_flags
& SI_CPDMA_SKIP_BO_LIST_UPDATE
)) {
170 /* Count memory usage in so that need_cs_space can take it into account. */
171 r600_context_add_resource_size(&sctx
->b
.b
, dst
);
173 r600_context_add_resource_size(&sctx
->b
.b
, src
);
176 if (!(user_flags
& SI_CPDMA_SKIP_CHECK_CS_SPACE
))
177 si_need_cs_space(sctx
);
179 /* This must be done after need_cs_space. */
180 if (!(user_flags
& SI_CPDMA_SKIP_BO_LIST_UPDATE
)) {
181 radeon_add_to_buffer_list(&sctx
->b
, &sctx
->b
.gfx
,
182 (struct r600_resource
*)dst
,
183 RADEON_USAGE_WRITE
, RADEON_PRIO_CP_DMA
);
185 radeon_add_to_buffer_list(&sctx
->b
, &sctx
->b
.gfx
,
186 (struct r600_resource
*)src
,
187 RADEON_USAGE_READ
, RADEON_PRIO_CP_DMA
);
190 /* Flush the caches for the first copy only.
191 * Also wait for the previous CP DMA operations.
193 if (!(user_flags
& SI_CPDMA_SKIP_GFX_SYNC
) && sctx
->b
.flags
)
194 si_emit_cache_flush(sctx
);
196 if (!(user_flags
& SI_CPDMA_SKIP_SYNC_BEFORE
) && *is_first
)
197 *packet_flags
|= CP_DMA_RAW_WAIT
;
201 /* Do the synchronization after the last dma, so that all data
202 * is written to memory.
204 if (!(user_flags
& SI_CPDMA_SKIP_SYNC_AFTER
) &&
205 byte_count
== remaining_size
)
206 *packet_flags
|= CP_DMA_SYNC
;
209 static void si_clear_buffer(struct pipe_context
*ctx
, struct pipe_resource
*dst
,
210 uint64_t offset
, uint64_t size
, unsigned value
,
211 enum r600_coherency coher
)
213 struct si_context
*sctx
= (struct si_context
*)ctx
;
214 struct radeon_winsys
*ws
= sctx
->b
.ws
;
215 struct r600_resource
*rdst
= r600_resource(dst
);
216 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, coher
);
217 unsigned flush_flags
= get_flush_flags(sctx
, coher
);
218 uint64_t dma_clear_size
;
219 bool is_first
= true;
224 dma_clear_size
= size
& ~3ull;
226 /* Mark the buffer range of destination as valid (initialized),
227 * so that transfer_map knows it should wait for the GPU when mapping
229 util_range_add(&rdst
->valid_buffer_range
, offset
,
230 offset
+ dma_clear_size
);
232 /* dma_clear_buffer can use clear_buffer on failure. Make sure that
233 * doesn't happen. We don't want an infinite recursion: */
234 if (sctx
->b
.dma
.cs
&&
235 !(dst
->flags
& PIPE_RESOURCE_FLAG_SPARSE
) &&
237 /* CP DMA is very slow. Always use SDMA for big clears. This
238 * alone improves DeusEx:MD performance by 70%. */
239 (size
> CP_DMA_CLEAR_PERF_THRESHOLD
||
240 /* Buffers not used by the GFX IB yet will be cleared by SDMA.
241 * This happens to move most buffer clears to SDMA, including
242 * DCC and CMASK clears, because pipe->clear clears them before
243 * si_emit_framebuffer_state (in a draw call) adds them.
244 * For example, DeusEx:MD has 21 buffer clears per frame and all
245 * of them are moved to SDMA thanks to this. */
246 !ws
->cs_is_buffer_referenced(sctx
->b
.gfx
.cs
, rdst
->buf
,
247 RADEON_USAGE_READWRITE
))) {
248 sctx
->b
.dma_clear_buffer(ctx
, dst
, offset
, dma_clear_size
, value
);
250 offset
+= dma_clear_size
;
251 size
-= dma_clear_size
;
252 } else if (dma_clear_size
>= 4) {
253 uint64_t va
= rdst
->gpu_address
+ offset
;
255 offset
+= dma_clear_size
;
256 size
-= dma_clear_size
;
258 /* Flush the caches. */
259 sctx
->b
.flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
260 SI_CONTEXT_CS_PARTIAL_FLUSH
| flush_flags
;
262 while (dma_clear_size
) {
263 unsigned byte_count
= MIN2(dma_clear_size
, cp_dma_max_byte_count(sctx
));
264 unsigned dma_flags
= tc_l2_flag
| CP_DMA_CLEAR
;
266 si_cp_dma_prepare(sctx
, dst
, NULL
, byte_count
, dma_clear_size
, 0,
267 &is_first
, &dma_flags
);
269 /* Emit the clear packet. */
270 si_emit_cp_dma(sctx
, va
, value
, byte_count
, dma_flags
, coher
);
272 dma_clear_size
-= byte_count
;
277 rdst
->TC_L2_dirty
= true;
279 /* If it's not a framebuffer fast clear... */
280 if (coher
== R600_COHERENCY_SHADER
)
281 sctx
->b
.num_cp_dma_calls
++;
285 /* Handle non-dword alignment.
287 * This function is called for embedded texture metadata clears,
288 * but those should always be properly aligned. */
289 assert(dst
->target
== PIPE_BUFFER
);
292 pipe_buffer_write(ctx
, dst
, offset
, size
, &value
);
297 * Realign the CP DMA engine. This must be done after a copy with an unaligned
300 * \param size Remaining size to the CP DMA alignment.
302 static void si_cp_dma_realign_engine(struct si_context
*sctx
, unsigned size
,
303 unsigned user_flags
, bool *is_first
)
306 unsigned dma_flags
= 0;
307 unsigned scratch_size
= SI_CPDMA_ALIGNMENT
* 2;
309 assert(size
< SI_CPDMA_ALIGNMENT
);
311 /* Use the scratch buffer as the dummy buffer. The 3D engine should be
312 * idle at this point.
314 if (!sctx
->scratch_buffer
||
315 sctx
->scratch_buffer
->b
.b
.width0
< scratch_size
) {
316 r600_resource_reference(&sctx
->scratch_buffer
, NULL
);
317 sctx
->scratch_buffer
= (struct r600_resource
*)
318 si_aligned_buffer_create(&sctx
->screen
->b
.b
,
319 R600_RESOURCE_FLAG_UNMAPPABLE
,
322 if (!sctx
->scratch_buffer
)
325 si_mark_atom_dirty(sctx
, &sctx
->scratch_state
);
328 si_cp_dma_prepare(sctx
, &sctx
->scratch_buffer
->b
.b
,
329 &sctx
->scratch_buffer
->b
.b
, size
, size
, user_flags
,
330 is_first
, &dma_flags
);
332 va
= sctx
->scratch_buffer
->gpu_address
;
333 si_emit_cp_dma(sctx
, va
, va
+ SI_CPDMA_ALIGNMENT
, size
, dma_flags
,
334 R600_COHERENCY_SHADER
);
338 * Do memcpy between buffers using CP DMA.
340 * \param user_flags bitmask of SI_CPDMA_*
342 void si_copy_buffer(struct si_context
*sctx
,
343 struct pipe_resource
*dst
, struct pipe_resource
*src
,
344 uint64_t dst_offset
, uint64_t src_offset
, unsigned size
,
347 uint64_t main_dst_offset
, main_src_offset
;
348 unsigned skipped_size
= 0;
349 unsigned realign_size
= 0;
350 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, R600_COHERENCY_SHADER
);
351 unsigned flush_flags
= get_flush_flags(sctx
, R600_COHERENCY_SHADER
);
352 bool is_first
= true;
357 if (dst
!= src
|| dst_offset
!= src_offset
) {
358 /* Mark the buffer range of destination as valid (initialized),
359 * so that transfer_map knows it should wait for the GPU when mapping
361 util_range_add(&r600_resource(dst
)->valid_buffer_range
, dst_offset
,
365 dst_offset
+= r600_resource(dst
)->gpu_address
;
366 src_offset
+= r600_resource(src
)->gpu_address
;
368 /* The workarounds aren't needed on Fiji and beyond. */
369 if (sctx
->b
.family
<= CHIP_CARRIZO
||
370 sctx
->b
.family
== CHIP_STONEY
) {
371 /* If the size is not aligned, we must add a dummy copy at the end
372 * just to align the internal counter. Otherwise, the DMA engine
373 * would slow down by an order of magnitude for following copies.
375 if (size
% SI_CPDMA_ALIGNMENT
)
376 realign_size
= SI_CPDMA_ALIGNMENT
- (size
% SI_CPDMA_ALIGNMENT
);
378 /* If the copy begins unaligned, we must start copying from the next
379 * aligned block and the skipped part should be copied after everything
380 * else has been copied. Only the src alignment matters, not dst.
382 if (src_offset
% SI_CPDMA_ALIGNMENT
) {
383 skipped_size
= SI_CPDMA_ALIGNMENT
- (src_offset
% SI_CPDMA_ALIGNMENT
);
384 /* The main part will be skipped if the size is too small. */
385 skipped_size
= MIN2(skipped_size
, size
);
386 size
-= skipped_size
;
390 /* Flush the caches. */
391 if (!(user_flags
& SI_CPDMA_SKIP_GFX_SYNC
))
392 sctx
->b
.flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
393 SI_CONTEXT_CS_PARTIAL_FLUSH
| flush_flags
;
395 /* This is the main part doing the copying. Src is always aligned. */
396 main_dst_offset
= dst_offset
+ skipped_size
;
397 main_src_offset
= src_offset
+ skipped_size
;
400 unsigned dma_flags
= tc_l2_flag
;
401 unsigned byte_count
= MIN2(size
, cp_dma_max_byte_count(sctx
));
403 si_cp_dma_prepare(sctx
, dst
, src
, byte_count
,
404 size
+ skipped_size
+ realign_size
,
405 user_flags
, &is_first
, &dma_flags
);
407 si_emit_cp_dma(sctx
, main_dst_offset
, main_src_offset
,
408 byte_count
, dma_flags
, R600_COHERENCY_SHADER
);
411 main_src_offset
+= byte_count
;
412 main_dst_offset
+= byte_count
;
415 /* Copy the part we skipped because src wasn't aligned. */
417 unsigned dma_flags
= tc_l2_flag
;
419 si_cp_dma_prepare(sctx
, dst
, src
, skipped_size
,
420 skipped_size
+ realign_size
, user_flags
,
421 &is_first
, &dma_flags
);
423 si_emit_cp_dma(sctx
, dst_offset
, src_offset
, skipped_size
,
424 dma_flags
, R600_COHERENCY_SHADER
);
427 /* Finally, realign the engine if the size wasn't aligned. */
429 si_cp_dma_realign_engine(sctx
, realign_size
, user_flags
,
433 r600_resource(dst
)->TC_L2_dirty
= true;
435 /* If it's not a prefetch... */
436 if (dst_offset
!= src_offset
)
437 sctx
->b
.num_cp_dma_calls
++;
440 void cik_prefetch_TC_L2_async(struct si_context
*sctx
, struct pipe_resource
*buf
,
441 uint64_t offset
, unsigned size
)
443 assert(sctx
->b
.chip_class
>= CIK
);
445 si_copy_buffer(sctx
, buf
, buf
, offset
, offset
, size
, SI_CPDMA_SKIP_ALL
);
448 static void cik_prefetch_shader_async(struct si_context
*sctx
,
449 struct si_pm4_state
*state
)
451 struct pipe_resource
*bo
= &state
->bo
[0]->b
.b
;
452 assert(state
->nbo
== 1);
454 cik_prefetch_TC_L2_async(sctx
, bo
, 0, bo
->width0
);
457 static void cik_prefetch_VBO_descriptors(struct si_context
*sctx
)
459 if (!sctx
->vertex_elements
)
462 cik_prefetch_TC_L2_async(sctx
, &sctx
->vertex_buffers
.buffer
->b
.b
,
463 sctx
->vertex_buffers
.buffer_offset
,
464 sctx
->vertex_elements
->desc_list_byte_size
);
467 void cik_emit_prefetch_L2(struct si_context
*sctx
)
469 /* Prefetch shaders and VBO descriptors to TC L2. */
470 if (sctx
->b
.chip_class
>= GFX9
) {
471 /* Choose the right spot for the VBO prefetch. */
472 if (sctx
->tes_shader
.cso
) {
473 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_HS
)
474 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.hs
);
475 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
476 cik_prefetch_VBO_descriptors(sctx
);
477 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_GS
)
478 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
479 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VS
)
480 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
481 } else if (sctx
->gs_shader
.cso
) {
482 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_GS
)
483 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
484 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
485 cik_prefetch_VBO_descriptors(sctx
);
486 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VS
)
487 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
489 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VS
)
490 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
491 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
492 cik_prefetch_VBO_descriptors(sctx
);
496 /* Choose the right spot for the VBO prefetch. */
497 if (sctx
->tes_shader
.cso
) {
498 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_LS
)
499 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.ls
);
500 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
501 cik_prefetch_VBO_descriptors(sctx
);
502 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_HS
)
503 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.hs
);
504 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_ES
)
505 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.es
);
506 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_GS
)
507 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
508 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VS
)
509 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
510 } else if (sctx
->gs_shader
.cso
) {
511 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_ES
)
512 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.es
);
513 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
514 cik_prefetch_VBO_descriptors(sctx
);
515 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_GS
)
516 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.gs
);
517 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VS
)
518 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
520 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VS
)
521 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.vs
);
522 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_VBO_DESCRIPTORS
)
523 cik_prefetch_VBO_descriptors(sctx
);
527 if (sctx
->prefetch_L2_mask
& SI_PREFETCH_PS
)
528 cik_prefetch_shader_async(sctx
, sctx
->queued
.named
.ps
);
530 sctx
->prefetch_L2_mask
= 0;
533 void si_init_cp_dma_functions(struct si_context
*sctx
)
535 sctx
->b
.clear_buffer
= si_clear_buffer
;