radeonsi: declare coher in si_copy_buffer
[mesa.git] / src / gallium / drivers / radeonsi / si_cp_dma.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "si_pipe.h"
26 #include "sid.h"
27
28 /* Recommended maximum sizes for optimal performance.
29 * Fall back to compute or SDMA if the size is greater.
30 */
31 #define CP_DMA_COPY_PERF_THRESHOLD (64 * 1024) /* copied from Vulkan */
32 #define CP_DMA_CLEAR_PERF_THRESHOLD (32 * 1024) /* guess (clear is much slower) */
33
34 /* Set this if you want the ME to wait until CP DMA is done.
35 * It should be set on the last CP DMA packet. */
36 #define CP_DMA_SYNC (1 << 0)
37
38 /* Set this if the source data was used as a destination in a previous CP DMA
39 * packet. It's for preventing a read-after-write (RAW) hazard between two
40 * CP DMA packets. */
41 #define CP_DMA_RAW_WAIT (1 << 1)
42 #define CP_DMA_USE_L2 (1 << 2) /* CIK+ */
43 #define CP_DMA_CLEAR (1 << 3)
44 #define CP_DMA_PFP_SYNC_ME (1 << 4)
45
46 /* The max number of bytes that can be copied per packet. */
47 static inline unsigned cp_dma_max_byte_count(struct si_context *sctx)
48 {
49 unsigned max = sctx->chip_class >= GFX9 ?
50 S_414_BYTE_COUNT_GFX9(~0u) :
51 S_414_BYTE_COUNT_GFX6(~0u);
52
53 /* make it aligned for optimal performance */
54 return max & ~(SI_CPDMA_ALIGNMENT - 1);
55 }
56
57
58 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
59 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
60 * clear value.
61 */
62 static void si_emit_cp_dma(struct si_context *sctx, uint64_t dst_va,
63 uint64_t src_va, unsigned size, unsigned flags)
64 {
65 struct radeon_cmdbuf *cs = sctx->gfx_cs;
66 uint32_t header = 0, command = 0;
67
68 assert(size <= cp_dma_max_byte_count(sctx));
69
70 if (sctx->chip_class >= GFX9)
71 command |= S_414_BYTE_COUNT_GFX9(size);
72 else
73 command |= S_414_BYTE_COUNT_GFX6(size);
74
75 /* Sync flags. */
76 if (flags & CP_DMA_SYNC)
77 header |= S_411_CP_SYNC(1);
78 else {
79 if (sctx->chip_class >= GFX9)
80 command |= S_414_DISABLE_WR_CONFIRM_GFX9(1);
81 else
82 command |= S_414_DISABLE_WR_CONFIRM_GFX6(1);
83 }
84
85 if (flags & CP_DMA_RAW_WAIT)
86 command |= S_414_RAW_WAIT(1);
87
88 /* Src and dst flags. */
89 if (sctx->chip_class >= GFX9 && !(flags & CP_DMA_CLEAR) &&
90 src_va == dst_va)
91 header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */
92 else if (flags & CP_DMA_USE_L2)
93 header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2);
94
95 if (flags & CP_DMA_CLEAR)
96 header |= S_411_SRC_SEL(V_411_DATA);
97 else if (flags & CP_DMA_USE_L2)
98 header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
99
100 if (sctx->chip_class >= CIK) {
101 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
102 radeon_emit(cs, header);
103 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
104 radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */
105 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
106 radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */
107 radeon_emit(cs, command);
108 } else {
109 header |= S_411_SRC_ADDR_HI(src_va >> 32);
110
111 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
112 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
113 radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */
114 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
115 radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
116 radeon_emit(cs, command);
117 }
118
119 /* CP DMA is executed in ME, but index buffers are read by PFP.
120 * This ensures that ME (CP DMA) is idle before PFP starts fetching
121 * indices. If we wanted to execute CP DMA in PFP, this packet
122 * should precede it.
123 */
124 if (flags & CP_DMA_PFP_SYNC_ME) {
125 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
126 radeon_emit(cs, 0);
127 }
128 }
129
130 void si_cp_dma_wait_for_idle(struct si_context *sctx)
131 {
132 /* Issue a dummy DMA that copies zero bytes.
133 *
134 * The DMA engine will see that there's no work to do and skip this
135 * DMA request, however, the CP will see the sync flag and still wait
136 * for all DMAs to complete.
137 */
138 si_emit_cp_dma(sctx, 0, 0, 0, CP_DMA_SYNC);
139 }
140
141 static unsigned get_flush_flags(struct si_context *sctx, enum si_coherency coher)
142 {
143 switch (coher) {
144 default:
145 case SI_COHERENCY_NONE:
146 return 0;
147 case SI_COHERENCY_SHADER:
148 return SI_CONTEXT_INV_SMEM_L1 |
149 SI_CONTEXT_INV_VMEM_L1 |
150 (sctx->chip_class == SI ? SI_CONTEXT_INV_GLOBAL_L2 : 0);
151 case SI_COHERENCY_CB_META:
152 return SI_CONTEXT_FLUSH_AND_INV_CB;
153 }
154 }
155
156 static unsigned get_tc_l2_flag(struct si_context *sctx, enum si_coherency coher)
157 {
158 if ((sctx->chip_class >= GFX9 && coher == SI_COHERENCY_CB_META) ||
159 (sctx->chip_class >= CIK && coher == SI_COHERENCY_SHADER))
160 return CP_DMA_USE_L2;
161
162 return 0;
163 }
164
165 static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
166 struct pipe_resource *src, unsigned byte_count,
167 uint64_t remaining_size, unsigned user_flags,
168 enum si_coherency coher, bool *is_first,
169 unsigned *packet_flags)
170 {
171 /* Fast exit for a CPDMA prefetch. */
172 if ((user_flags & SI_CPDMA_SKIP_ALL) == SI_CPDMA_SKIP_ALL) {
173 *is_first = false;
174 return;
175 }
176
177 if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
178 /* Count memory usage in so that need_cs_space can take it into account. */
179 si_context_add_resource_size(sctx, dst);
180 if (src)
181 si_context_add_resource_size(sctx, src);
182 }
183
184 if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
185 si_need_gfx_cs_space(sctx);
186
187 /* This must be done after need_cs_space. */
188 if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
189 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
190 r600_resource(dst),
191 RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
192 if (src)
193 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
194 r600_resource(src),
195 RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
196 }
197
198 /* Flush the caches for the first copy only.
199 * Also wait for the previous CP DMA operations.
200 */
201 if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->flags)
202 si_emit_cache_flush(sctx);
203
204 if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first)
205 *packet_flags |= CP_DMA_RAW_WAIT;
206
207 *is_first = false;
208
209 /* Do the synchronization after the last dma, so that all data
210 * is written to memory.
211 */
212 if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) &&
213 byte_count == remaining_size) {
214 *packet_flags |= CP_DMA_SYNC;
215
216 if (coher == SI_COHERENCY_SHADER)
217 *packet_flags |= CP_DMA_PFP_SYNC_ME;
218 }
219 }
220
221 void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
222 uint64_t offset, uint64_t size, unsigned value,
223 enum si_coherency coher, enum si_method xfer)
224 {
225 struct radeon_winsys *ws = sctx->ws;
226 struct r600_resource *rdst = r600_resource(dst);
227 unsigned tc_l2_flag = get_tc_l2_flag(sctx, coher);
228 unsigned flush_flags = get_flush_flags(sctx, coher);
229 uint64_t dma_clear_size;
230 bool is_first = true;
231
232 if (!size)
233 return;
234
235 dma_clear_size = size & ~3ull;
236
237 /* Mark the buffer range of destination as valid (initialized),
238 * so that transfer_map knows it should wait for the GPU when mapping
239 * that range. */
240 util_range_add(&rdst->valid_buffer_range, offset,
241 offset + dma_clear_size);
242
243 /* dma_clear_buffer can use clear_buffer on failure. Make sure that
244 * doesn't happen. We don't want an infinite recursion: */
245 if (sctx->dma_cs &&
246 !(dst->flags & PIPE_RESOURCE_FLAG_SPARSE) &&
247 (offset % 4 == 0) &&
248 /* CP DMA is very slow. Always use SDMA for big clears. This
249 * alone improves DeusEx:MD performance by 70%. */
250 (size > CP_DMA_CLEAR_PERF_THRESHOLD ||
251 /* Buffers not used by the GFX IB yet will be cleared by SDMA.
252 * This happens to move most buffer clears to SDMA, including
253 * DCC and CMASK clears, because pipe->clear clears them before
254 * si_emit_framebuffer_state (in a draw call) adds them.
255 * For example, DeusEx:MD has 21 buffer clears per frame and all
256 * of them are moved to SDMA thanks to this. */
257 !ws->cs_is_buffer_referenced(sctx->gfx_cs, rdst->buf,
258 RADEON_USAGE_READWRITE)) &&
259 /* bypass sdma transfer with param xfer */
260 (xfer != SI_METHOD_CP_DMA)) {
261 sctx->dma_clear_buffer(sctx, dst, offset, dma_clear_size, value);
262
263 offset += dma_clear_size;
264 size -= dma_clear_size;
265 } else if (dma_clear_size >= 4) {
266 uint64_t va = rdst->gpu_address + offset;
267
268 offset += dma_clear_size;
269 size -= dma_clear_size;
270
271 /* Flush the caches. */
272 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
273 SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
274
275 while (dma_clear_size) {
276 unsigned byte_count = MIN2(dma_clear_size, cp_dma_max_byte_count(sctx));
277 unsigned dma_flags = tc_l2_flag | CP_DMA_CLEAR;
278
279 si_cp_dma_prepare(sctx, dst, NULL, byte_count, dma_clear_size, 0,
280 coher, &is_first, &dma_flags);
281
282 /* Emit the clear packet. */
283 si_emit_cp_dma(sctx, va, value, byte_count, dma_flags);
284
285 dma_clear_size -= byte_count;
286 va += byte_count;
287 }
288
289 if (tc_l2_flag)
290 rdst->TC_L2_dirty = true;
291
292 /* If it's not a framebuffer fast clear... */
293 if (coher == SI_COHERENCY_SHADER)
294 sctx->num_cp_dma_calls++;
295 }
296
297 if (size) {
298 /* Handle non-dword alignment.
299 *
300 * This function is called for embedded texture metadata clears,
301 * but those should always be properly aligned. */
302 assert(dst->target == PIPE_BUFFER);
303 assert(size < 4);
304
305 pipe_buffer_write(&sctx->b, dst, offset, size, &value);
306 }
307 }
308
309 static void si_pipe_clear_buffer(struct pipe_context *ctx,
310 struct pipe_resource *dst,
311 unsigned offset, unsigned size,
312 const void *clear_value_ptr,
313 int clear_value_size)
314 {
315 struct si_context *sctx = (struct si_context*)ctx;
316 uint32_t dword_value;
317 unsigned i;
318
319 assert(offset % clear_value_size == 0);
320 assert(size % clear_value_size == 0);
321
322 if (clear_value_size > 4) {
323 const uint32_t *u32 = clear_value_ptr;
324 bool clear_dword_duplicated = true;
325
326 /* See if we can lower large fills to dword fills. */
327 for (i = 1; i < clear_value_size / 4; i++)
328 if (u32[0] != u32[i]) {
329 clear_dword_duplicated = false;
330 break;
331 }
332
333 if (!clear_dword_duplicated) {
334 /* Use transform feedback for 64-bit, 96-bit, and
335 * 128-bit fills.
336 */
337 union pipe_color_union clear_value;
338
339 memcpy(&clear_value, clear_value_ptr, clear_value_size);
340 si_blitter_begin(sctx, SI_DISABLE_RENDER_COND);
341 util_blitter_clear_buffer(sctx->blitter, dst, offset,
342 size, clear_value_size / 4,
343 &clear_value);
344 si_blitter_end(sctx);
345 return;
346 }
347 }
348
349 /* Expand the clear value to a dword. */
350 switch (clear_value_size) {
351 case 1:
352 dword_value = *(uint8_t*)clear_value_ptr;
353 dword_value |= (dword_value << 8) |
354 (dword_value << 16) |
355 (dword_value << 24);
356 break;
357 case 2:
358 dword_value = *(uint16_t*)clear_value_ptr;
359 dword_value |= dword_value << 16;
360 break;
361 default:
362 dword_value = *(uint32_t*)clear_value_ptr;
363 }
364
365 si_clear_buffer(sctx, dst, offset, size, dword_value,
366 SI_COHERENCY_SHADER, SI_METHOD_BEST);
367 }
368
369 /**
370 * Realign the CP DMA engine. This must be done after a copy with an unaligned
371 * size.
372 *
373 * \param size Remaining size to the CP DMA alignment.
374 */
375 static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
376 unsigned user_flags, enum si_coherency coher,
377 bool *is_first)
378 {
379 uint64_t va;
380 unsigned dma_flags = 0;
381 unsigned scratch_size = SI_CPDMA_ALIGNMENT * 2;
382
383 assert(size < SI_CPDMA_ALIGNMENT);
384
385 /* Use the scratch buffer as the dummy buffer. The 3D engine should be
386 * idle at this point.
387 */
388 if (!sctx->scratch_buffer ||
389 sctx->scratch_buffer->b.b.width0 < scratch_size) {
390 r600_resource_reference(&sctx->scratch_buffer, NULL);
391 sctx->scratch_buffer =
392 si_aligned_buffer_create(&sctx->screen->b,
393 SI_RESOURCE_FLAG_UNMAPPABLE,
394 PIPE_USAGE_DEFAULT,
395 scratch_size, 256);
396 if (!sctx->scratch_buffer)
397 return;
398
399 si_mark_atom_dirty(sctx, &sctx->atoms.s.scratch_state);
400 }
401
402 si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b,
403 &sctx->scratch_buffer->b.b, size, size, user_flags,
404 coher, is_first, &dma_flags);
405
406 va = sctx->scratch_buffer->gpu_address;
407 si_emit_cp_dma(sctx, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags);
408 }
409
410 /**
411 * Do memcpy between buffers using CP DMA.
412 *
413 * \param user_flags bitmask of SI_CPDMA_*
414 */
415 void si_copy_buffer(struct si_context *sctx,
416 struct pipe_resource *dst, struct pipe_resource *src,
417 uint64_t dst_offset, uint64_t src_offset, unsigned size,
418 unsigned user_flags)
419 {
420 uint64_t main_dst_offset, main_src_offset;
421 unsigned skipped_size = 0;
422 unsigned realign_size = 0;
423 enum si_coherency coher = SI_COHERENCY_SHADER;
424 unsigned tc_l2_flag = get_tc_l2_flag(sctx, coher);
425 unsigned flush_flags = get_flush_flags(sctx, coher);
426 bool is_first = true;
427
428 if (!size)
429 return;
430
431 if (dst != src || dst_offset != src_offset) {
432 /* Mark the buffer range of destination as valid (initialized),
433 * so that transfer_map knows it should wait for the GPU when mapping
434 * that range. */
435 util_range_add(&r600_resource(dst)->valid_buffer_range, dst_offset,
436 dst_offset + size);
437 }
438
439 dst_offset += r600_resource(dst)->gpu_address;
440 src_offset += r600_resource(src)->gpu_address;
441
442 /* The workarounds aren't needed on Fiji and beyond. */
443 if (sctx->family <= CHIP_CARRIZO ||
444 sctx->family == CHIP_STONEY) {
445 /* If the size is not aligned, we must add a dummy copy at the end
446 * just to align the internal counter. Otherwise, the DMA engine
447 * would slow down by an order of magnitude for following copies.
448 */
449 if (size % SI_CPDMA_ALIGNMENT)
450 realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);
451
452 /* If the copy begins unaligned, we must start copying from the next
453 * aligned block and the skipped part should be copied after everything
454 * else has been copied. Only the src alignment matters, not dst.
455 */
456 if (src_offset % SI_CPDMA_ALIGNMENT) {
457 skipped_size = SI_CPDMA_ALIGNMENT - (src_offset % SI_CPDMA_ALIGNMENT);
458 /* The main part will be skipped if the size is too small. */
459 skipped_size = MIN2(skipped_size, size);
460 size -= skipped_size;
461 }
462 }
463
464 /* Flush the caches. */
465 if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC))
466 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
467 SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
468
469 /* This is the main part doing the copying. Src is always aligned. */
470 main_dst_offset = dst_offset + skipped_size;
471 main_src_offset = src_offset + skipped_size;
472
473 while (size) {
474 unsigned dma_flags = tc_l2_flag;
475 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
476
477 si_cp_dma_prepare(sctx, dst, src, byte_count,
478 size + skipped_size + realign_size,
479 user_flags, coher, &is_first, &dma_flags);
480
481 si_emit_cp_dma(sctx, main_dst_offset, main_src_offset,
482 byte_count, dma_flags);
483
484 size -= byte_count;
485 main_src_offset += byte_count;
486 main_dst_offset += byte_count;
487 }
488
489 /* Copy the part we skipped because src wasn't aligned. */
490 if (skipped_size) {
491 unsigned dma_flags = tc_l2_flag;
492
493 si_cp_dma_prepare(sctx, dst, src, skipped_size,
494 skipped_size + realign_size, user_flags,
495 coher, &is_first, &dma_flags);
496
497 si_emit_cp_dma(sctx, dst_offset, src_offset, skipped_size,
498 dma_flags);
499 }
500
501 /* Finally, realign the engine if the size wasn't aligned. */
502 if (realign_size) {
503 si_cp_dma_realign_engine(sctx, realign_size, user_flags, coher,
504 &is_first);
505 }
506
507 if (tc_l2_flag)
508 r600_resource(dst)->TC_L2_dirty = true;
509
510 /* If it's not a prefetch... */
511 if (dst_offset != src_offset)
512 sctx->num_cp_dma_calls++;
513 }
514
515 void cik_prefetch_TC_L2_async(struct si_context *sctx, struct pipe_resource *buf,
516 uint64_t offset, unsigned size)
517 {
518 assert(sctx->chip_class >= CIK);
519
520 si_copy_buffer(sctx, buf, buf, offset, offset, size, SI_CPDMA_SKIP_ALL);
521 }
522
523 static void cik_prefetch_shader_async(struct si_context *sctx,
524 struct si_pm4_state *state)
525 {
526 struct pipe_resource *bo = &state->bo[0]->b.b;
527 assert(state->nbo == 1);
528
529 cik_prefetch_TC_L2_async(sctx, bo, 0, bo->width0);
530 }
531
532 static void cik_prefetch_VBO_descriptors(struct si_context *sctx)
533 {
534 if (!sctx->vertex_elements)
535 return;
536
537 cik_prefetch_TC_L2_async(sctx, &sctx->vb_descriptors_buffer->b.b,
538 sctx->vb_descriptors_offset,
539 sctx->vertex_elements->desc_list_byte_size);
540 }
541
542 /**
543 * Prefetch shaders and VBO descriptors.
544 *
545 * \param vertex_stage_only Whether only the the API VS and VBO descriptors
546 * should be prefetched.
547 */
548 void cik_emit_prefetch_L2(struct si_context *sctx, bool vertex_stage_only)
549 {
550 unsigned mask = sctx->prefetch_L2_mask;
551 assert(mask);
552
553 /* Prefetch shaders and VBO descriptors to TC L2. */
554 if (sctx->chip_class >= GFX9) {
555 /* Choose the right spot for the VBO prefetch. */
556 if (sctx->tes_shader.cso) {
557 if (mask & SI_PREFETCH_HS)
558 cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
559 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
560 cik_prefetch_VBO_descriptors(sctx);
561 if (vertex_stage_only) {
562 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_HS |
563 SI_PREFETCH_VBO_DESCRIPTORS);
564 return;
565 }
566
567 if (mask & SI_PREFETCH_GS)
568 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
569 if (mask & SI_PREFETCH_VS)
570 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
571 } else if (sctx->gs_shader.cso) {
572 if (mask & SI_PREFETCH_GS)
573 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
574 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
575 cik_prefetch_VBO_descriptors(sctx);
576 if (vertex_stage_only) {
577 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_GS |
578 SI_PREFETCH_VBO_DESCRIPTORS);
579 return;
580 }
581
582 if (mask & SI_PREFETCH_VS)
583 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
584 } else {
585 if (mask & SI_PREFETCH_VS)
586 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
587 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
588 cik_prefetch_VBO_descriptors(sctx);
589 if (vertex_stage_only) {
590 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS |
591 SI_PREFETCH_VBO_DESCRIPTORS);
592 return;
593 }
594 }
595 } else {
596 /* SI-CI-VI */
597 /* Choose the right spot for the VBO prefetch. */
598 if (sctx->tes_shader.cso) {
599 if (mask & SI_PREFETCH_LS)
600 cik_prefetch_shader_async(sctx, sctx->queued.named.ls);
601 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
602 cik_prefetch_VBO_descriptors(sctx);
603 if (vertex_stage_only) {
604 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_LS |
605 SI_PREFETCH_VBO_DESCRIPTORS);
606 return;
607 }
608
609 if (mask & SI_PREFETCH_HS)
610 cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
611 if (mask & SI_PREFETCH_ES)
612 cik_prefetch_shader_async(sctx, sctx->queued.named.es);
613 if (mask & SI_PREFETCH_GS)
614 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
615 if (mask & SI_PREFETCH_VS)
616 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
617 } else if (sctx->gs_shader.cso) {
618 if (mask & SI_PREFETCH_ES)
619 cik_prefetch_shader_async(sctx, sctx->queued.named.es);
620 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
621 cik_prefetch_VBO_descriptors(sctx);
622 if (vertex_stage_only) {
623 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_ES |
624 SI_PREFETCH_VBO_DESCRIPTORS);
625 return;
626 }
627
628 if (mask & SI_PREFETCH_GS)
629 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
630 if (mask & SI_PREFETCH_VS)
631 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
632 } else {
633 if (mask & SI_PREFETCH_VS)
634 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
635 if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
636 cik_prefetch_VBO_descriptors(sctx);
637 if (vertex_stage_only) {
638 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS |
639 SI_PREFETCH_VBO_DESCRIPTORS);
640 return;
641 }
642 }
643 }
644
645 if (mask & SI_PREFETCH_PS)
646 cik_prefetch_shader_async(sctx, sctx->queued.named.ps);
647
648 sctx->prefetch_L2_mask = 0;
649 }
650
651 void si_init_cp_dma_functions(struct si_context *sctx)
652 {
653 sctx->b.clear_buffer = si_pipe_clear_buffer;
654 }