2 * Copyright 2013 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * Marek Olšák <maraeo@gmail.com>
29 #include "radeon/r600_cs.h"
32 /* Set this if you want the 3D engine to wait until CP DMA is done.
33 * It should be set on the last CP DMA packet. */
34 #define R600_CP_DMA_SYNC (1 << 0) /* R600+ */
36 /* Set this if the source data was used as a destination in a previous CP DMA
37 * packet. It's for preventing a read-after-write (RAW) hazard between two
39 #define SI_CP_DMA_RAW_WAIT (1 << 1) /* SI+ */
40 #define CIK_CP_DMA_USE_L2 (1 << 2)
42 /* Emit a CP DMA packet to do a copy from one buffer to another.
43 * The size must fit in bits [20:0].
45 static void si_emit_cp_dma_copy_buffer(struct si_context
*sctx
,
46 uint64_t dst_va
, uint64_t src_va
,
47 unsigned size
, unsigned flags
)
49 struct radeon_winsys_cs
*cs
= sctx
->b
.gfx
.cs
;
50 uint32_t sync_flag
= flags
& R600_CP_DMA_SYNC
? S_411_CP_SYNC(1) : 0;
51 uint32_t wr_confirm
= !(flags
& R600_CP_DMA_SYNC
) ? S_414_DISABLE_WR_CONFIRM(1) : 0;
52 uint32_t raw_wait
= flags
& SI_CP_DMA_RAW_WAIT
? S_414_RAW_WAIT(1) : 0;
53 uint32_t sel
= flags
& CIK_CP_DMA_USE_L2
?
54 S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2
) |
55 S_411_DSL_SEL(V_411_DST_ADDR_TC_L2
) : 0;
58 assert((size
& ((1<<21)-1)) == size
);
60 if (sctx
->b
.chip_class
>= CIK
) {
61 radeon_emit(cs
, PKT3(PKT3_DMA_DATA
, 5, 0));
62 radeon_emit(cs
, sync_flag
| sel
); /* CP_SYNC [31] */
63 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
64 radeon_emit(cs
, src_va
>> 32); /* SRC_ADDR_HI [31:0] */
65 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
66 radeon_emit(cs
, dst_va
>> 32); /* DST_ADDR_HI [31:0] */
67 radeon_emit(cs
, size
| wr_confirm
| raw_wait
); /* COMMAND [29:22] | BYTE_COUNT [20:0] */
69 radeon_emit(cs
, PKT3(PKT3_CP_DMA
, 4, 0));
70 radeon_emit(cs
, src_va
); /* SRC_ADDR_LO [31:0] */
71 radeon_emit(cs
, sync_flag
| ((src_va
>> 32) & 0xffff)); /* CP_SYNC [31] | SRC_ADDR_HI [15:0] */
72 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
73 radeon_emit(cs
, (dst_va
>> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
74 radeon_emit(cs
, size
| wr_confirm
| raw_wait
); /* COMMAND [29:22] | BYTE_COUNT [20:0] */
78 /* Emit a CP DMA packet to clear a buffer. The size must fit in bits [20:0]. */
79 static void si_emit_cp_dma_clear_buffer(struct si_context
*sctx
,
80 uint64_t dst_va
, unsigned size
,
81 uint32_t clear_value
, unsigned flags
)
83 struct radeon_winsys_cs
*cs
= sctx
->b
.gfx
.cs
;
84 uint32_t sync_flag
= flags
& R600_CP_DMA_SYNC
? S_411_CP_SYNC(1) : 0;
85 uint32_t wr_confirm
= !(flags
& R600_CP_DMA_SYNC
) ? S_414_DISABLE_WR_CONFIRM(1) : 0;
86 uint32_t raw_wait
= flags
& SI_CP_DMA_RAW_WAIT
? S_414_RAW_WAIT(1) : 0;
87 uint32_t dst_sel
= flags
& CIK_CP_DMA_USE_L2
? S_411_DSL_SEL(V_411_DST_ADDR_TC_L2
) : 0;
90 assert((size
& ((1<<21)-1)) == size
);
92 if (sctx
->b
.chip_class
>= CIK
) {
93 radeon_emit(cs
, PKT3(PKT3_DMA_DATA
, 5, 0));
94 radeon_emit(cs
, sync_flag
| dst_sel
| S_411_SRC_SEL(V_411_DATA
)); /* CP_SYNC [31] | SRC_SEL[30:29] */
95 radeon_emit(cs
, clear_value
); /* DATA [31:0] */
97 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
98 radeon_emit(cs
, dst_va
>> 32); /* DST_ADDR_HI [15:0] */
99 radeon_emit(cs
, size
| wr_confirm
| raw_wait
); /* COMMAND [29:22] | BYTE_COUNT [20:0] */
101 radeon_emit(cs
, PKT3(PKT3_CP_DMA
, 4, 0));
102 radeon_emit(cs
, clear_value
); /* DATA [31:0] */
103 radeon_emit(cs
, sync_flag
| S_411_SRC_SEL(V_411_DATA
)); /* CP_SYNC [31] | SRC_SEL[30:29] */
104 radeon_emit(cs
, dst_va
); /* DST_ADDR_LO [31:0] */
105 radeon_emit(cs
, (dst_va
>> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
106 radeon_emit(cs
, size
| wr_confirm
| raw_wait
); /* COMMAND [29:22] | BYTE_COUNT [20:0] */
110 static unsigned get_flush_flags(struct si_context
*sctx
, bool is_framebuffer
)
113 return SI_CONTEXT_FLUSH_AND_INV_FRAMEBUFFER
;
115 return SI_CONTEXT_INV_SMEM_L1
|
116 SI_CONTEXT_INV_VMEM_L1
|
117 (sctx
->b
.chip_class
== SI
? SI_CONTEXT_INV_GLOBAL_L2
: 0);
120 static unsigned get_tc_l2_flag(struct si_context
*sctx
, bool is_framebuffer
)
122 return is_framebuffer
|| sctx
->b
.chip_class
== SI
? 0 : CIK_CP_DMA_USE_L2
;
125 static void si_cp_dma_prepare(struct si_context
*sctx
, struct pipe_resource
*dst
,
126 struct pipe_resource
*src
, unsigned byte_count
,
127 unsigned remaining_size
, unsigned *flags
)
129 si_need_cs_space(sctx
);
131 /* This must be done after need_cs_space. */
132 radeon_add_to_buffer_list(&sctx
->b
, &sctx
->b
.gfx
,
133 (struct r600_resource
*)dst
,
134 RADEON_USAGE_WRITE
, RADEON_PRIO_CP_DMA
);
136 radeon_add_to_buffer_list(&sctx
->b
, &sctx
->b
.gfx
,
137 (struct r600_resource
*)src
,
138 RADEON_USAGE_READ
, RADEON_PRIO_CP_DMA
);
140 /* Flush the caches for the first copy only.
141 * Also wait for the previous CP DMA operations.
144 si_emit_cache_flush(sctx
, NULL
);
145 *flags
|= SI_CP_DMA_RAW_WAIT
;
148 /* Do the synchronization after the last dma, so that all data
149 * is written to memory.
151 if (byte_count
== remaining_size
)
152 *flags
|= R600_CP_DMA_SYNC
;
155 /* Alignment for optimal performance. */
156 #define CP_DMA_ALIGNMENT 32
157 /* The max number of bytes to copy per packet. */
158 #define CP_DMA_MAX_BYTE_COUNT ((1 << 21) - CP_DMA_ALIGNMENT)
160 static void si_clear_buffer(struct pipe_context
*ctx
, struct pipe_resource
*dst
,
161 unsigned offset
, unsigned size
, unsigned value
,
164 struct si_context
*sctx
= (struct si_context
*)ctx
;
165 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, is_framebuffer
);
166 unsigned flush_flags
= get_flush_flags(sctx
, is_framebuffer
);
171 /* Mark the buffer range of destination as valid (initialized),
172 * so that transfer_map knows it should wait for the GPU when mapping
174 util_range_add(&r600_resource(dst
)->valid_buffer_range
, offset
,
177 /* Fallback for unaligned clears. */
178 if (offset
% 4 != 0 || size
% 4 != 0) {
179 uint8_t *map
= sctx
->b
.ws
->buffer_map(r600_resource(dst
)->buf
,
181 PIPE_TRANSFER_WRITE
);
183 for (unsigned i
= 0; i
< size
; i
++) {
184 unsigned byte_within_dword
= (offset
+ i
) % 4;
185 *map
++ = (value
>> (byte_within_dword
* 8)) & 0xff;
190 uint64_t va
= r600_resource(dst
)->gpu_address
+ offset
;
192 /* Flush the caches. */
193 sctx
->b
.flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
| flush_flags
;
196 unsigned byte_count
= MIN2(size
, CP_DMA_MAX_BYTE_COUNT
);
197 unsigned dma_flags
= tc_l2_flag
;
199 si_cp_dma_prepare(sctx
, dst
, NULL
, byte_count
, size
, &dma_flags
);
201 /* Emit the clear packet. */
202 si_emit_cp_dma_clear_buffer(sctx
, va
, byte_count
, value
, dma_flags
);
208 /* Flush the caches again in case the 3D engine has been prefetching
210 sctx
->b
.flags
|= flush_flags
;
213 r600_resource(dst
)->TC_L2_dirty
= true;
217 * Realign the CP DMA engine. This must be done after a copy with an unaligned
220 * \param size Remaining size to the CP DMA alignment.
222 static void si_cp_dma_realign_engine(struct si_context
*sctx
, unsigned size
)
225 unsigned dma_flags
= 0;
226 unsigned scratch_size
= CP_DMA_ALIGNMENT
* 2;
228 assert(size
< CP_DMA_ALIGNMENT
);
230 /* Use the scratch buffer as the dummy buffer. The 3D engine should be
231 * idle at this point.
233 if (!sctx
->scratch_buffer
||
234 sctx
->scratch_buffer
->b
.b
.width0
< scratch_size
) {
235 r600_resource_reference(&sctx
->scratch_buffer
, NULL
);
236 sctx
->scratch_buffer
=
237 si_resource_create_custom(&sctx
->screen
->b
.b
,
240 if (!sctx
->scratch_buffer
)
242 sctx
->emit_scratch_reloc
= true;
245 si_cp_dma_prepare(sctx
, &sctx
->scratch_buffer
->b
.b
,
246 &sctx
->scratch_buffer
->b
.b
, size
, size
, &dma_flags
);
248 va
= sctx
->scratch_buffer
->gpu_address
;
249 si_emit_cp_dma_copy_buffer(sctx
, va
, va
+ CP_DMA_ALIGNMENT
, size
,
253 void si_copy_buffer(struct si_context
*sctx
,
254 struct pipe_resource
*dst
, struct pipe_resource
*src
,
255 uint64_t dst_offset
, uint64_t src_offset
, unsigned size
,
258 uint64_t main_dst_offset
, main_src_offset
;
259 unsigned skipped_size
= 0;
260 unsigned realign_size
= 0;
261 unsigned tc_l2_flag
= get_tc_l2_flag(sctx
, is_framebuffer
);
262 unsigned flush_flags
= get_flush_flags(sctx
, is_framebuffer
);
267 /* Mark the buffer range of destination as valid (initialized),
268 * so that transfer_map knows it should wait for the GPU when mapping
270 util_range_add(&r600_resource(dst
)->valid_buffer_range
, dst_offset
,
273 dst_offset
+= r600_resource(dst
)->gpu_address
;
274 src_offset
+= r600_resource(src
)->gpu_address
;
276 /* The workarounds aren't needed on Fiji and beyond. */
277 if (sctx
->b
.family
<= CHIP_CARRIZO
||
278 sctx
->b
.family
== CHIP_STONEY
) {
279 /* If the size is not aligned, we must add a dummy copy at the end
280 * just to align the internal counter. Otherwise, the DMA engine
281 * would slow down by an order of magnitude for following copies.
283 if (size
% CP_DMA_ALIGNMENT
)
284 realign_size
= CP_DMA_ALIGNMENT
- (size
% CP_DMA_ALIGNMENT
);
286 /* If the copy begins unaligned, we must start copying from the next
287 * aligned block and the skipped part should be copied after everything
288 * else has been copied. Only the src alignment matters, not dst.
290 if (src_offset
% CP_DMA_ALIGNMENT
) {
291 skipped_size
= CP_DMA_ALIGNMENT
- (src_offset
% CP_DMA_ALIGNMENT
);
292 /* The main part will be skipped if the size is too small. */
293 skipped_size
= MIN2(skipped_size
, size
);
294 size
-= skipped_size
;
298 /* Flush the caches. */
299 sctx
->b
.flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
| flush_flags
;
301 /* This is the main part doing the copying. Src is always aligned. */
302 main_dst_offset
= dst_offset
+ skipped_size
;
303 main_src_offset
= src_offset
+ skipped_size
;
306 unsigned dma_flags
= tc_l2_flag
;
307 unsigned byte_count
= MIN2(size
, CP_DMA_MAX_BYTE_COUNT
);
309 si_cp_dma_prepare(sctx
, dst
, src
, byte_count
,
310 size
+ skipped_size
+ realign_size
,
313 si_emit_cp_dma_copy_buffer(sctx
, main_dst_offset
, main_src_offset
,
314 byte_count
, dma_flags
);
317 main_src_offset
+= byte_count
;
318 main_dst_offset
+= byte_count
;
321 /* Copy the part we skipped because src wasn't aligned. */
323 unsigned dma_flags
= tc_l2_flag
;
325 si_cp_dma_prepare(sctx
, dst
, src
, skipped_size
,
326 skipped_size
+ realign_size
,
329 si_emit_cp_dma_copy_buffer(sctx
, dst_offset
, src_offset
,
330 skipped_size
, dma_flags
);
333 /* Finally, realign the engine if the size wasn't aligned. */
335 si_cp_dma_realign_engine(sctx
, realign_size
);
337 /* Flush the caches again in case the 3D engine has been prefetching
339 sctx
->b
.flags
|= flush_flags
;
342 r600_resource(dst
)->TC_L2_dirty
= true;
345 void si_init_cp_dma_functions(struct si_context
*sctx
)
347 sctx
->b
.clear_buffer
= si_clear_buffer
;