projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
util: remove LIST_IS_EMPTY macro
[mesa.git] / src / gallium / drivers / radeonsi / si_cp_dma.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "si_pipe.h"
26 #include "sid.h"
27
28 /* Set this if you want the ME to wait until CP DMA is done.
29 * It should be set on the last CP DMA packet. */
30 #define CP_DMA_SYNC (1 << 0)
31
32 /* Set this if the source data was used as a destination in a previous CP DMA
33 * packet. It's for preventing a read-after-write (RAW) hazard between two
34 * CP DMA packets. */
35 #define CP_DMA_RAW_WAIT (1 << 1)
36 #define CP_DMA_DST_IS_GDS (1 << 2)
37 #define CP_DMA_CLEAR (1 << 3)
38 #define CP_DMA_PFP_SYNC_ME (1 << 4)
39 #define CP_DMA_SRC_IS_GDS (1 << 5)
40
41 /* The max number of bytes that can be copied per packet. */
42 static inline unsigned cp_dma_max_byte_count(struct si_context *sctx)
43 {
44 unsigned max = sctx->chip_class >= GFX9 ?
45 S_414_BYTE_COUNT_GFX9(~0u) :
46 S_414_BYTE_COUNT_GFX6(~0u);
47
48 /* make it aligned for optimal performance */
49 return max & ~(SI_CPDMA_ALIGNMENT - 1);
50 }
51
52
53 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
54 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
55 * clear value.
56 */
57 static void si_emit_cp_dma(struct si_context *sctx, struct radeon_cmdbuf *cs,
58 uint64_t dst_va, uint64_t src_va, unsigned size,
59 unsigned flags, enum si_cache_policy cache_policy)
60 {
61 uint32_t header = 0, command = 0;
62
63 assert(size <= cp_dma_max_byte_count(sctx));
64 assert(sctx->chip_class != GFX6 || cache_policy == L2_BYPASS);
65
66 if (sctx->chip_class >= GFX9)
67 command |= S_414_BYTE_COUNT_GFX9(size);
68 else
69 command |= S_414_BYTE_COUNT_GFX6(size);
70
71 /* Sync flags. */
72 if (flags & CP_DMA_SYNC)
73 header |= S_411_CP_SYNC(1);
74 else {
75 if (sctx->chip_class >= GFX9)
76 command |= S_414_DISABLE_WR_CONFIRM_GFX9(1);
77 else
78 command |= S_414_DISABLE_WR_CONFIRM_GFX6(1);
79 }
80
81 if (flags & CP_DMA_RAW_WAIT)
82 command |= S_414_RAW_WAIT(1);
83
84 /* Src and dst flags. */
85 if (sctx->chip_class >= GFX9 && !(flags & CP_DMA_CLEAR) &&
86 src_va == dst_va) {
87 header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */
88 } else if (flags & CP_DMA_DST_IS_GDS) {
89 header |= S_411_DST_SEL(V_411_GDS);
90 /* GDS increments the address, not CP. */
91 command |= S_414_DAS(V_414_REGISTER) |
92 S_414_DAIC(V_414_NO_INCREMENT);
93 } else if (sctx->chip_class >= GFX7 && cache_policy != L2_BYPASS) {
94 header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2) |
95 S_500_DST_CACHE_POLICY(cache_policy == L2_STREAM);
96 }
97
98 if (flags & CP_DMA_CLEAR) {
99 header |= S_411_SRC_SEL(V_411_DATA);
100 } else if (flags & CP_DMA_SRC_IS_GDS) {
101 header |= S_411_SRC_SEL(V_411_GDS);
102 /* Both of these are required for GDS. It does increment the address. */
103 command |= S_414_SAS(V_414_REGISTER) |
104 S_414_SAIC(V_414_NO_INCREMENT);
105 } else if (sctx->chip_class >= GFX7 && cache_policy != L2_BYPASS) {
106 header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2) |
107 S_500_SRC_CACHE_POLICY(cache_policy == L2_STREAM);
108 }
109
110 if (sctx->chip_class >= GFX7) {
111 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
112 radeon_emit(cs, header);
113 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
114 radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */
115 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
116 radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */
117 radeon_emit(cs, command);
118 } else {
119 header |= S_411_SRC_ADDR_HI(src_va >> 32);
120
121 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
122 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
123 radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */
124 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
125 radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
126 radeon_emit(cs, command);
127 }
128
129 /* CP DMA is executed in ME, but index buffers are read by PFP.
130 * This ensures that ME (CP DMA) is idle before PFP starts fetching
131 * indices. If we wanted to execute CP DMA in PFP, this packet
132 * should precede it.
133 */
134 if (sctx->has_graphics && flags & CP_DMA_PFP_SYNC_ME) {
135 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
136 radeon_emit(cs, 0);
137 }
138 }
139
140 void si_cp_dma_wait_for_idle(struct si_context *sctx)
141 {
142 /* Issue a dummy DMA that copies zero bytes.
143 *
144 * The DMA engine will see that there's no work to do and skip this
145 * DMA request, however, the CP will see the sync flag and still wait
146 * for all DMAs to complete.
147 */
148 si_emit_cp_dma(sctx, sctx->gfx_cs, 0, 0, 0, CP_DMA_SYNC, L2_BYPASS);
149 }
150
151 static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
152 struct pipe_resource *src, unsigned byte_count,
153 uint64_t remaining_size, unsigned user_flags,
154 enum si_coherency coher, bool *is_first,
155 unsigned *packet_flags)
156 {
157 /* Fast exit for a CPDMA prefetch. */
158 if ((user_flags & SI_CPDMA_SKIP_ALL) == SI_CPDMA_SKIP_ALL) {
159 *is_first = false;
160 return;
161 }
162
163 if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
164 /* Count memory usage in so that need_cs_space can take it into account. */
165 if (dst)
166 si_context_add_resource_size(sctx, dst);
167 if (src)
168 si_context_add_resource_size(sctx, src);
169 }
170
171 if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
172 si_need_gfx_cs_space(sctx);
173
174 /* This must be done after need_cs_space. */
175 if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
176 if (dst)
177 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
178 si_resource(dst),
179 RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
180 if (src)
181 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
182 si_resource(src),
183 RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
184 }
185
186 /* Flush the caches for the first copy only.
187 * Also wait for the previous CP DMA operations.
188 */
189 if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->flags)
190 sctx->emit_cache_flush(sctx);
191
192 if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first &&
193 !(*packet_flags & CP_DMA_CLEAR))
194 *packet_flags |= CP_DMA_RAW_WAIT;
195
196 *is_first = false;
197
198 /* Do the synchronization after the last dma, so that all data
199 * is written to memory.
200 */
201 if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) &&
202 byte_count == remaining_size) {
203 *packet_flags |= CP_DMA_SYNC;
204
205 if (coher == SI_COHERENCY_SHADER)
206 *packet_flags |= CP_DMA_PFP_SYNC_ME;
207 }
208 }
209
210 void si_cp_dma_clear_buffer(struct si_context *sctx, struct radeon_cmdbuf *cs,
211 struct pipe_resource *dst, uint64_t offset,
212 uint64_t size, unsigned value, unsigned user_flags,
213 enum si_coherency coher, enum si_cache_policy cache_policy)
214 {
215 struct si_resource *sdst = si_resource(dst);
216 uint64_t va = (sdst ? sdst->gpu_address : 0) + offset;
217 bool is_first = true;
218
219 assert(size && size % 4 == 0);
220
221 /* Mark the buffer range of destination as valid (initialized),
222 * so that transfer_map knows it should wait for the GPU when mapping
223 * that range. */
224 if (sdst)
225 util_range_add(dst, &sdst->valid_buffer_range, offset, offset + size);
226
227 /* Flush the caches. */
228 if (sdst && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
229 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
230 SI_CONTEXT_CS_PARTIAL_FLUSH |
231 si_get_flush_flags(sctx, coher, cache_policy);
232 }
233
234 while (size) {
235 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
236 unsigned dma_flags = CP_DMA_CLEAR | (sdst ? 0 : CP_DMA_DST_IS_GDS);
237
238 si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, user_flags,
239 coher, &is_first, &dma_flags);
240
241 /* Emit the clear packet. */
242 si_emit_cp_dma(sctx, cs, va, value, byte_count, dma_flags, cache_policy);
243
244 size -= byte_count;
245 va += byte_count;
246 }
247
248 if (sdst && cache_policy != L2_BYPASS)
249 sdst->TC_L2_dirty = true;
250
251 /* If it's not a framebuffer fast clear... */
252 if (coher == SI_COHERENCY_SHADER) {
253 sctx->num_cp_dma_calls++;
254 si_prim_discard_signal_next_compute_ib_start(sctx);
255 }
256 }
257
258 /**
259 * Realign the CP DMA engine. This must be done after a copy with an unaligned
260 * size.
261 *
262 * \param size Remaining size to the CP DMA alignment.
263 */
264 static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
265 unsigned user_flags, enum si_coherency coher,
266 enum si_cache_policy cache_policy,
267 bool *is_first)
268 {
269 uint64_t va;
270 unsigned dma_flags = 0;
271 unsigned scratch_size = SI_CPDMA_ALIGNMENT * 2;
272
273 assert(size < SI_CPDMA_ALIGNMENT);
274
275 /* Use the scratch buffer as the dummy buffer. The 3D engine should be
276 * idle at this point.
277 */
278 if (!sctx->scratch_buffer ||
279 sctx->scratch_buffer->b.b.width0 < scratch_size) {
280 si_resource_reference(&sctx->scratch_buffer, NULL);
281 sctx->scratch_buffer =
282 si_aligned_buffer_create(&sctx->screen->b,
283 SI_RESOURCE_FLAG_UNMAPPABLE,
284 PIPE_USAGE_DEFAULT,
285 scratch_size, 256);
286 if (!sctx->scratch_buffer)
287 return;
288
289 si_mark_atom_dirty(sctx, &sctx->atoms.s.scratch_state);
290 }
291
292 si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b,
293 &sctx->scratch_buffer->b.b, size, size, user_flags,
294 coher, is_first, &dma_flags);
295
296 va = sctx->scratch_buffer->gpu_address;
297 si_emit_cp_dma(sctx, sctx->gfx_cs, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags,
298 cache_policy);
299 }
300
301 /**
302 * Do memcpy between buffers using CP DMA.
303 * If src or dst is NULL, it means read or write GDS, respectively.
304 *
305 * \param user_flags bitmask of SI_CPDMA_*
306 */
307 void si_cp_dma_copy_buffer(struct si_context *sctx,
308 struct pipe_resource *dst, struct pipe_resource *src,
309 uint64_t dst_offset, uint64_t src_offset, unsigned size,
310 unsigned user_flags, enum si_coherency coher,
311 enum si_cache_policy cache_policy)
312 {
313 uint64_t main_dst_offset, main_src_offset;
314 unsigned skipped_size = 0;
315 unsigned realign_size = 0;
316 unsigned gds_flags = (dst ? 0 : CP_DMA_DST_IS_GDS) |
317 (src ? 0 : CP_DMA_SRC_IS_GDS);
318 bool is_first = true;
319
320 assert(size);
321
322 if (dst) {
323 /* Skip this for the L2 prefetch. */
324 if (dst != src || dst_offset != src_offset) {
325 /* Mark the buffer range of destination as valid (initialized),
326 * so that transfer_map knows it should wait for the GPU when mapping
327 * that range. */
328 util_range_add(dst, &si_resource(dst)->valid_buffer_range, dst_offset,
329 dst_offset + size);
330 }
331
332 dst_offset += si_resource(dst)->gpu_address;
333 }
334 if (src)
335 src_offset += si_resource(src)->gpu_address;
336
337 /* The workarounds aren't needed on Fiji and beyond. */
338 if (sctx->family <= CHIP_CARRIZO ||
339 sctx->family == CHIP_STONEY) {
340 /* If the size is not aligned, we must add a dummy copy at the end
341 * just to align the internal counter. Otherwise, the DMA engine
342 * would slow down by an order of magnitude for following copies.
343 */
344 if (size % SI_CPDMA_ALIGNMENT)
345 realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);
346
347 /* If the copy begins unaligned, we must start copying from the next
348 * aligned block and the skipped part should be copied after everything
349 * else has been copied. Only the src alignment matters, not dst.
350 *
351 * GDS doesn't need the source address to be aligned.
352 */
353 if (src && src_offset % SI_CPDMA_ALIGNMENT) {
354 skipped_size = SI_CPDMA_ALIGNMENT - (src_offset % SI_CPDMA_ALIGNMENT);
355 /* The main part will be skipped if the size is too small. */
356 skipped_size = MIN2(skipped_size, size);
357 size -= skipped_size;
358 }
359 }
360
361 /* Flush the caches. */
362 if ((dst || src) && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
363 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
364 SI_CONTEXT_CS_PARTIAL_FLUSH |
365 si_get_flush_flags(sctx, coher, cache_policy);
366 }
367
368 /* This is the main part doing the copying. Src is always aligned. */
369 main_dst_offset = dst_offset + skipped_size;
370 main_src_offset = src_offset + skipped_size;
371
372 while (size) {
373 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
374 unsigned dma_flags = gds_flags;
375
376 si_cp_dma_prepare(sctx, dst, src, byte_count,
377 size + skipped_size + realign_size,
378 user_flags, coher, &is_first, &dma_flags);
379
380 si_emit_cp_dma(sctx, sctx->gfx_cs, main_dst_offset, main_src_offset,
381 byte_count, dma_flags, cache_policy);
382
383 size -= byte_count;
384 main_src_offset += byte_count;
385 main_dst_offset += byte_count;
386 }
387
388 /* Copy the part we skipped because src wasn't aligned. */
389 if (skipped_size) {
390 unsigned dma_flags = gds_flags;
391
392 si_cp_dma_prepare(sctx, dst, src, skipped_size,
393 skipped_size + realign_size, user_flags,
394 coher, &is_first, &dma_flags);
395
396 si_emit_cp_dma(sctx, sctx->gfx_cs, dst_offset, src_offset, skipped_size,
397 dma_flags, cache_policy);
398 }
399
400 /* Finally, realign the engine if the size wasn't aligned. */
401 if (realign_size) {
402 si_cp_dma_realign_engine(sctx, realign_size, user_flags, coher,
403 cache_policy, &is_first);
404 }
405
406 if (dst && cache_policy != L2_BYPASS)
407 si_resource(dst)->TC_L2_dirty = true;
408
409 /* If it's not a prefetch or GDS copy... */
410 if (dst && src && (dst != src || dst_offset != src_offset)) {
411 sctx->num_cp_dma_calls++;
412 si_prim_discard_signal_next_compute_ib_start(sctx);
413 }
414 }
415
416 void cik_prefetch_TC_L2_async(struct si_context *sctx, struct pipe_resource *buf,
417 uint64_t offset, unsigned size)
418 {
419 assert(sctx->chip_class >= GFX7);
420
421 si_cp_dma_copy_buffer(sctx, buf, buf, offset, offset, size,
422 SI_CPDMA_SKIP_ALL, SI_COHERENCY_SHADER, L2_LRU);
423 }
424
425 static void cik_prefetch_shader_async(struct si_context *sctx,
426 struct si_pm4_state *state)
427 {
428 struct pipe_resource *bo = &state->bo[0]->b.b;
429 assert(state->nbo == 1);
430
431 cik_prefetch_TC_L2_async(sctx, bo, 0, bo->width0);
432 }
433
434 static void cik_prefetch_VBO_descriptors(struct si_context *sctx)
435 {
436 if (!sctx->vertex_elements || !sctx->vertex_elements->desc_list_byte_size)
437 return;
438
439 cik_prefetch_TC_L2_async(sctx, &sctx->vb_descriptors_buffer->b.b,
440 sctx->vb_descriptors_offset,
441 sctx->vertex_elements->desc_list_byte_size);
442 }
443
444 /**
445 * Prefetch shaders and VBO descriptors.
446 *
447 * \param vertex_stage_only Whether only the the API VS and VBO descriptors
448 * should be prefetched.
449 */
450 void cik_emit_prefetch_L2(struct si_context *sctx, bool vertex_stage_only)
451 {
452 unsigned mask = sctx->prefetch_L2_mask;
453 assert(mask);
454
455 /* Prefetch shaders and VBO descriptors to TC L2. */
456 if (sctx->chip_class >= GFX9) {
457 /* Choose the right spot for the VBO prefetch. */
458 if (sctx->queued.named.hs) {
459 if (mask & SI_PREFETCH_HS)
460 cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
461 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
462 cik_prefetch_VBO_descriptors(sctx);
463 if (vertex_stage_only) {
464 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_HS |
465 SI_PREFETCH_VBO_DESCRIPTORS);
466 return;
467 }
468
469 if (mask & SI_PREFETCH_GS)
470 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
471 if (mask & SI_PREFETCH_VS)
472 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
473 } else if (sctx->queued.named.gs) {
474 if (mask & SI_PREFETCH_GS)
475 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
476 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
477 cik_prefetch_VBO_descriptors(sctx);
478 if (vertex_stage_only) {
479 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_GS |
480 SI_PREFETCH_VBO_DESCRIPTORS);
481 return;
482 }
483
484 if (mask & SI_PREFETCH_VS)
485 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
486 } else {
487 if (mask & SI_PREFETCH_VS)
488 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
489 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
490 cik_prefetch_VBO_descriptors(sctx);
491 if (vertex_stage_only) {
492 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS |
493 SI_PREFETCH_VBO_DESCRIPTORS);
494 return;
495 }
496 }
497 } else {
498 /* GFX6-GFX8 */
499 /* Choose the right spot for the VBO prefetch. */
500 if (sctx->tes_shader.cso) {
501 if (mask & SI_PREFETCH_LS)
502 cik_prefetch_shader_async(sctx, sctx->queued.named.ls);
503 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
504 cik_prefetch_VBO_descriptors(sctx);
505 if (vertex_stage_only) {
506 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_LS |
507 SI_PREFETCH_VBO_DESCRIPTORS);
508 return;
509 }
510
511 if (mask & SI_PREFETCH_HS)
512 cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
513 if (mask & SI_PREFETCH_ES)
514 cik_prefetch_shader_async(sctx, sctx->queued.named.es);
515 if (mask & SI_PREFETCH_GS)
516 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
517 if (mask & SI_PREFETCH_VS)
518 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
519 } else if (sctx->gs_shader.cso) {
520 if (mask & SI_PREFETCH_ES)
521 cik_prefetch_shader_async(sctx, sctx->queued.named.es);
522 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
523 cik_prefetch_VBO_descriptors(sctx);
524 if (vertex_stage_only) {
525 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_ES |
526 SI_PREFETCH_VBO_DESCRIPTORS);
527 return;
528 }
529
530 if (mask & SI_PREFETCH_GS)
531 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
532 if (mask & SI_PREFETCH_VS)
533 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
534 } else {
535 if (mask & SI_PREFETCH_VS)
536 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
537 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
538 cik_prefetch_VBO_descriptors(sctx);
539 if (vertex_stage_only) {
540 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS |
541 SI_PREFETCH_VBO_DESCRIPTORS);
542 return;
543 }
544 }
545 }
546
547 if (mask & SI_PREFETCH_PS)
548 cik_prefetch_shader_async(sctx, sctx->queued.named.ps);
549
550 sctx->prefetch_L2_mask = 0;
551 }
552
553 void si_test_gds(struct si_context *sctx)
554 {
555 struct pipe_context *ctx = &sctx->b;
556 struct pipe_resource *src, *dst;
557 unsigned r[4] = {};
558 unsigned offset = debug_get_num_option("OFFSET", 16);
559
560 src = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16);
561 dst = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16);
562 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 0, 4, 0xabcdef01, 0, SI_COHERENCY_SHADER, L2_BYPASS);
563 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 4, 4, 0x23456789, 0, SI_COHERENCY_SHADER, L2_BYPASS);
564 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 8, 4, 0x87654321, 0, SI_COHERENCY_SHADER, L2_BYPASS);
565 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 12, 4, 0xfedcba98, 0, SI_COHERENCY_SHADER, L2_BYPASS);
566 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, dst, 0, 16, 0xdeadbeef, 0, SI_COHERENCY_SHADER, L2_BYPASS);
567
568 si_cp_dma_copy_buffer(sctx, NULL, src, offset, 0, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
569 si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
570
571 pipe_buffer_read(ctx, dst, 0, sizeof(r), r);
572 printf("GDS copy = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3],
573 r[0] == 0xabcdef01 && r[1] == 0x23456789 &&
574 r[2] == 0x87654321 && r[3] == 0xfedcba98 ? "pass" : "fail");
575
576 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, NULL, offset, 16, 0xc1ea4146, 0, SI_COHERENCY_NONE, L2_BYPASS);
577 si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
578
579 pipe_buffer_read(ctx, dst, 0, sizeof(r), r);
580 printf("GDS clear = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3],
581 r[0] == 0xc1ea4146 && r[1] == 0xc1ea4146 &&
582 r[2] == 0xc1ea4146 && r[3] == 0xc1ea4146 ? "pass" : "fail");
583
584 pipe_resource_reference(&src, NULL);
585 pipe_resource_reference(&dst, NULL);
586 exit(0);
587 }
588
589 void si_cp_write_data(struct si_context *sctx, struct si_resource *buf,
590 unsigned offset, unsigned size, unsigned dst_sel,
591 unsigned engine, const void *data)
592 {
593 struct radeon_cmdbuf *cs = sctx->gfx_cs;
594
595 assert(offset % 4 == 0);
596 assert(size % 4 == 0);
597
598 if (sctx->chip_class == GFX6 && dst_sel == V_370_MEM)
599 dst_sel = V_370_MEM_GRBM;
600
601 radeon_add_to_buffer_list(sctx, cs, buf,
602 RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
603 uint64_t va = buf->gpu_address + offset;
604
605 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + size/4, 0));
606 radeon_emit(cs, S_370_DST_SEL(dst_sel) |
607 S_370_WR_CONFIRM(1) |
608 S_370_ENGINE_SEL(engine));
609 radeon_emit(cs, va);
610 radeon_emit(cs, va >> 32);
611 radeon_emit_array(cs, (const uint32_t*)data, size/4);
612 }
613
614 void si_cp_copy_data(struct si_context *sctx, struct radeon_cmdbuf *cs,
615 unsigned dst_sel, struct si_resource *dst, unsigned dst_offset,
616 unsigned src_sel, struct si_resource *src, unsigned src_offset)
617 {
618 /* cs can point to the compute IB, which has the buffer list in gfx_cs. */
619 if (dst) {
620 radeon_add_to_buffer_list(sctx, sctx->gfx_cs, dst,
621 RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
622 }
623 if (src) {
624 radeon_add_to_buffer_list(sctx, sctx->gfx_cs, src,
625 RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
626 }
627
628 uint64_t dst_va = (dst ? dst->gpu_address : 0ull) + dst_offset;
629 uint64_t src_va = (src ? src->gpu_address : 0ull) + src_offset;
630
631 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
632 radeon_emit(cs, COPY_DATA_SRC_SEL(src_sel) |
633 COPY_DATA_DST_SEL(dst_sel) |
634 COPY_DATA_WR_CONFIRM);
635 radeon_emit(cs, src_va);
636 radeon_emit(cs, src_va >> 32);
637 radeon_emit(cs, dst_va);
638 radeon_emit(cs, dst_va >> 32);
639 }