2 * Copyright 2013 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * Marek Olšák <maraeo@gmail.com>
29 #include "radeon/r600_cs.h"
31 /* Alignment for optimal performance. */
32 #define CP_DMA_ALIGNMENT 32
33 /* The max number of bytes to copy per packet. */
34 #define CP_DMA_MAX_BYTE_COUNT ((1 << 21) - CP_DMA_ALIGNMENT)
36 /* Set this if you want the ME to wait until CP DMA is done.
37 * It should be set on the last CP DMA packet. */
38 #define CP_DMA_SYNC (1 << 0)
40 /* Set this if the source data was used as a destination in a previous CP DMA
41 * packet. It's for preventing a read-after-write (RAW) hazard between two
43 #define CP_DMA_RAW_WAIT (1 << 1)
44 #define CP_DMA_USE_L2 (1 << 2) /* CIK+ */
45 #define CP_DMA_CLEAR (1 << 3)
47 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
48 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
51 static void si_emit_cp_dma(struct si_context
*sctx
, uint64_t dst_va
,
52 uint64_t src_va
, unsigned size
, unsigned flags
,
53 enum r600_coherency coher
)
55 struct radeon_winsys_cs
*cs
= sctx
->b
.gfx
.cs
;
56 uint32_t header
= 0, command
= S_414_BYTE_COUNT(size
);
59 assert(size
<= CP_DMA_MAX_BYTE_COUNT
);
62 if (flags
& CP_DMA_SYNC
)
63 header
|= S_411_CP_SYNC(1);
65 command
|= S_414_DISABLE_WR_CONFIRM(1);
67 if (flags
& CP_DMA_RAW_WAIT
)
68 command
|= S_414_RAW_WAIT(1);
70 /* Src and dst flags. */
71 if (flags
& CP_DMA_USE_L2
)
72 header
|= S_411_DSL_SEL(V_411_DST_ADDR_TC_L2
);
74 if (flags
& CP_DMA_CLEAR
)
75 header
|= S_411_SRC_SEL(V_411_DATA
);
76 else if (flags
& CP_DMA_USE_L2
)
77 header
|= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2
);
79 if (sctx
->b
.chip_class
>= CIK
) {
80 radeon_emit(cs
, PKT3(PKT3_DMA_DATA
, 5, 0));
81 radeon_emit(cs
, header
);
82 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
83 radeon_emit(cs
, src_va
>> 32); /* SRC_ADDR_HI [31:0] */
84 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
85 radeon_emit(cs
, dst_va
>> 32); /* DST_ADDR_HI [31:0] */
86 radeon_emit(cs
, command
);
88 header
|= S_411_SRC_ADDR_HI(src_va
>> 32);
90 radeon_emit(cs
, PKT3(PKT3_CP_DMA
, 4, 0));
91 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
92 radeon_emit(cs
, header
); /* SRC_ADDR_HI [15:0] + flags. */
93 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
94 radeon_emit(cs
, (dst_va
>> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
95 radeon_emit(cs
, command
);
98 /* CP DMA is executed in ME, but index buffers are read by PFP.
99 * This ensures that ME (CP DMA) is idle before PFP starts fetching
100 * indices. If we wanted to execute CP DMA in PFP, this packet
103 if (coher
== R600_COHERENCY_SHADER
&& flags
& CP_DMA_SYNC
) {
104 radeon_emit(cs
, PKT3(PKT3_PFP_SYNC_ME
, 0, 0));
109 static unsigned get_flush_flags(struct si_context
*sctx
, enum r600_coherency coher
)
113 case R600_COHERENCY_NONE
:
115 case R600_COHERENCY_SHADER
:
116 return SI_CONTEXT_INV_SMEM_L1
|
117 SI_CONTEXT_INV_VMEM_L1
|
118 (sctx
->b
.chip_class
== SI
? SI_CONTEXT_INV_GLOBAL_L2
: 0);
119 case R600_COHERENCY_CB_META
:
120 return SI_CONTEXT_FLUSH_AND_INV_CB
|
121 SI_CONTEXT_FLUSH_AND_INV_CB_META
;
125 static unsigned get_tc_l2_flag(struct si_context
*sctx
, enum r600_coherency coher
)
127 return coher
== R600_COHERENCY_SHADER
&&
128 sctx
->b
.chip_class
>= CIK
? CP_DMA_USE_L2
: 0;
131 static void si_cp_dma_prepare(struct si_context
*sctx
, struct pipe_resource
*dst
,
132 struct pipe_resource
*src
, unsigned byte_count
,
133 uint64_t remaining_size
, unsigned *flags
)
135 /* Count memory usage in so that need_cs_space can take it into account. */
136 r600_context_add_resource_size(&sctx
->b
.b
, dst
);
138 r600_context_add_resource_size(&sctx
->b
.b
, src
);
140 si_need_cs_space(sctx
);
142 /* This must be done after need_cs_space. */
143 radeon_add_to_buffer_list(&sctx
->b
, &sctx
->b
.gfx
,
144 (struct r600_resource
*)dst
,
145 RADEON_USAGE_WRITE
, RADEON_PRIO_CP_DMA
);
147 radeon_add_to_buffer_list(&sctx
->b
, &sctx
->b
.gfx
,
148 (struct r600_resource
*)src
,
149 RADEON_USAGE_READ
, RADEON_PRIO_CP_DMA
);
151 /* Flush the caches for the first copy only.
152 * Also wait for the previous CP DMA operations.
155 si_emit_cache_flush(sctx
);
156 *flags
|= CP_DMA_RAW_WAIT
;
159 /* Do the synchronization after the last dma, so that all data
160 * is written to memory.
162 if (byte_count
== remaining_size
)
163 *flags
|= CP_DMA_SYNC
;
166 static void si_clear_buffer(struct pipe_context
*ctx
, struct pipe_resource
*dst
,
167 uint64_t offset
, uint64_t size
, unsigned value
,
168 enum r600_coherency coher
)
170 struct si_context
*sctx
= (struct si_context
*)ctx
;
171 struct radeon_winsys
*ws
= sctx
->b
.ws
;
172 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, coher
);
173 unsigned flush_flags
= get_flush_flags(sctx
, coher
);
178 /* Mark the buffer range of destination as valid (initialized),
179 * so that transfer_map knows it should wait for the GPU when mapping
181 util_range_add(&r600_resource(dst
)->valid_buffer_range
, offset
,
184 /* Fallback for unaligned clears. */
185 if (offset
% 4 != 0 || size
% 4 != 0) {
186 uint8_t *map
= sctx
->b
.ws
->buffer_map(r600_resource(dst
)->buf
,
188 PIPE_TRANSFER_WRITE
);
190 for (uint64_t i
= 0; i
< size
; i
++) {
191 unsigned byte_within_dword
= (offset
+ i
) % 4;
192 *map
++ = (value
>> (byte_within_dword
* 8)) & 0xff;
197 /* dma_clear_buffer can use clear_buffer on failure. Make sure that
198 * doesn't happen. We don't want an infinite recursion: */
199 if (sctx
->b
.chip_class
>= CIK
&& sctx
->b
.dma
.cs
&&
200 /* CP DMA is very slow. Always use SDMA for big clears. This
201 * alone improves DeusEx:MD performance by 70%. */
202 (size
> 128 * 1024 ||
203 /* Buffers not used by the GFX IB yet will be cleared by SDMA.
204 * This happens to move most buffer clears to SDMA, including
205 * DCC and CMASK clears, because pipe->clear clears them before
206 * si_emit_framebuffer_state (in a draw call) adds them.
207 * For example, DeusEx:MD has 21 buffer clears per frame and all
208 * of them are moved to SDMA thanks to this. */
209 !ws
->cs_is_buffer_referenced(sctx
->b
.gfx
.cs
,
210 r600_resource(dst
)->buf
,
211 RADEON_USAGE_READWRITE
))) {
212 sctx
->b
.dma_clear_buffer(ctx
, dst
, offset
, size
, value
);
216 uint64_t va
= r600_resource(dst
)->gpu_address
+ offset
;
218 /* Flush the caches. */
219 sctx
->b
.flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
220 SI_CONTEXT_CS_PARTIAL_FLUSH
| flush_flags
;
223 unsigned byte_count
= MIN2(size
, CP_DMA_MAX_BYTE_COUNT
);
224 unsigned dma_flags
= tc_l2_flag
| CP_DMA_CLEAR
;
226 si_cp_dma_prepare(sctx
, dst
, NULL
, byte_count
, size
, &dma_flags
);
228 /* Emit the clear packet. */
229 si_emit_cp_dma(sctx
, va
, value
, byte_count
, dma_flags
, coher
);
236 r600_resource(dst
)->TC_L2_dirty
= true;
238 sctx
->b
.num_cp_dma_calls
++;
242 * Realign the CP DMA engine. This must be done after a copy with an unaligned
245 * \param size Remaining size to the CP DMA alignment.
247 static void si_cp_dma_realign_engine(struct si_context
*sctx
, unsigned size
)
250 unsigned dma_flags
= 0;
251 unsigned scratch_size
= CP_DMA_ALIGNMENT
* 2;
253 assert(size
< CP_DMA_ALIGNMENT
);
255 /* Use the scratch buffer as the dummy buffer. The 3D engine should be
256 * idle at this point.
258 if (!sctx
->scratch_buffer
||
259 sctx
->scratch_buffer
->b
.b
.width0
< scratch_size
) {
260 r600_resource_reference(&sctx
->scratch_buffer
, NULL
);
261 sctx
->scratch_buffer
= (struct r600_resource
*)
262 pipe_buffer_create(&sctx
->screen
->b
.b
, 0,
263 PIPE_USAGE_DEFAULT
, scratch_size
);
264 if (!sctx
->scratch_buffer
)
266 sctx
->emit_scratch_reloc
= true;
269 si_cp_dma_prepare(sctx
, &sctx
->scratch_buffer
->b
.b
,
270 &sctx
->scratch_buffer
->b
.b
, size
, size
, &dma_flags
);
272 va
= sctx
->scratch_buffer
->gpu_address
;
273 si_emit_cp_dma(sctx
, va
, va
+ CP_DMA_ALIGNMENT
, size
, dma_flags
,
274 R600_COHERENCY_SHADER
);
277 void si_copy_buffer(struct si_context
*sctx
,
278 struct pipe_resource
*dst
, struct pipe_resource
*src
,
279 uint64_t dst_offset
, uint64_t src_offset
, unsigned size
)
281 uint64_t main_dst_offset
, main_src_offset
;
282 unsigned skipped_size
= 0;
283 unsigned realign_size
= 0;
284 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, R600_COHERENCY_SHADER
);
285 unsigned flush_flags
= get_flush_flags(sctx
, R600_COHERENCY_SHADER
);
290 /* Mark the buffer range of destination as valid (initialized),
291 * so that transfer_map knows it should wait for the GPU when mapping
293 util_range_add(&r600_resource(dst
)->valid_buffer_range
, dst_offset
,
296 dst_offset
+= r600_resource(dst
)->gpu_address
;
297 src_offset
+= r600_resource(src
)->gpu_address
;
299 /* The workarounds aren't needed on Fiji and beyond. */
300 if (sctx
->b
.family
<= CHIP_CARRIZO
||
301 sctx
->b
.family
== CHIP_STONEY
) {
302 /* If the size is not aligned, we must add a dummy copy at the end
303 * just to align the internal counter. Otherwise, the DMA engine
304 * would slow down by an order of magnitude for following copies.
306 if (size
% CP_DMA_ALIGNMENT
)
307 realign_size
= CP_DMA_ALIGNMENT
- (size
% CP_DMA_ALIGNMENT
);
309 /* If the copy begins unaligned, we must start copying from the next
310 * aligned block and the skipped part should be copied after everything
311 * else has been copied. Only the src alignment matters, not dst.
313 if (src_offset
% CP_DMA_ALIGNMENT
) {
314 skipped_size
= CP_DMA_ALIGNMENT
- (src_offset
% CP_DMA_ALIGNMENT
);
315 /* The main part will be skipped if the size is too small. */
316 skipped_size
= MIN2(skipped_size
, size
);
317 size
-= skipped_size
;
321 /* Flush the caches. */
322 sctx
->b
.flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
323 SI_CONTEXT_CS_PARTIAL_FLUSH
| flush_flags
;
325 /* This is the main part doing the copying. Src is always aligned. */
326 main_dst_offset
= dst_offset
+ skipped_size
;
327 main_src_offset
= src_offset
+ skipped_size
;
330 unsigned dma_flags
= tc_l2_flag
;
331 unsigned byte_count
= MIN2(size
, CP_DMA_MAX_BYTE_COUNT
);
333 si_cp_dma_prepare(sctx
, dst
, src
, byte_count
,
334 size
+ skipped_size
+ realign_size
,
337 si_emit_cp_dma(sctx
, main_dst_offset
, main_src_offset
,
338 byte_count
, dma_flags
, R600_COHERENCY_SHADER
);
341 main_src_offset
+= byte_count
;
342 main_dst_offset
+= byte_count
;
345 /* Copy the part we skipped because src wasn't aligned. */
347 unsigned dma_flags
= tc_l2_flag
;
349 si_cp_dma_prepare(sctx
, dst
, src
, skipped_size
,
350 skipped_size
+ realign_size
,
353 si_emit_cp_dma(sctx
, dst_offset
, src_offset
, skipped_size
,
354 dma_flags
, R600_COHERENCY_SHADER
);
357 /* Finally, realign the engine if the size wasn't aligned. */
359 si_cp_dma_realign_engine(sctx
, realign_size
);
362 r600_resource(dst
)->TC_L2_dirty
= true;
364 sctx
->b
.num_cp_dma_calls
++;
367 void si_init_cp_dma_functions(struct si_context
*sctx
)
369 sctx
->b
.clear_buffer
= si_clear_buffer
;