src/gallium/drivers/radeonsi/si_cp_dma.c

   1 /*
   2  * Copyright 2013 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "si_pipe.h"
  26 #include "sid.h"
  27
  28 /* Set this if you want the ME to wait until CP DMA is done.
  29  * It should be set on the last CP DMA packet. */
  30 #define CP_DMA_SYNC             (1 << 0)
  31
  32 /* Set this if the source data was used as a destination in a previous CP DMA
  33  * packet. It's for preventing a read-after-write (RAW) hazard between two
  34  * CP DMA packets. */
  35 #define CP_DMA_RAW_WAIT         (1 << 1)
  36 #define CP_DMA_DST_IS_GDS       (1 << 2)
  37 #define CP_DMA_CLEAR            (1 << 3)
  38 #define CP_DMA_PFP_SYNC_ME      (1 << 4)
  39 #define CP_DMA_SRC_IS_GDS       (1 << 5)
  40
  41 /* The max number of bytes that can be copied per packet. */
  42 static inline unsigned cp_dma_max_byte_count(struct si_context *sctx)
  43 {
  44         unsigned max = sctx->chip_class >= GFX9 ?
  45                                S_414_BYTE_COUNT_GFX9(~0u) :
  46                                S_414_BYTE_COUNT_GFX6(~0u);
  47
  48         /* make it aligned for optimal performance */
  49         return max & ~(SI_CPDMA_ALIGNMENT - 1);
  50 }
  51
  52
  53 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
  54  * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
  55  * clear value.
  56  */
  57 static void si_emit_cp_dma(struct si_context *sctx, struct radeon_cmdbuf *cs,
  58                            uint64_t dst_va, uint64_t src_va, unsigned size,
  59                            unsigned flags, enum si_cache_policy cache_policy)
  60 {
  61         uint32_t header = 0, command = 0;
  62
  63         assert(size <= cp_dma_max_byte_count(sctx));
  64         assert(sctx->chip_class != SI || cache_policy == L2_BYPASS);
  65
  66         if (sctx->chip_class >= GFX9)
  67                 command |= S_414_BYTE_COUNT_GFX9(size);
  68         else
  69                 command |= S_414_BYTE_COUNT_GFX6(size);
  70
  71         /* Sync flags. */
  72         if (flags & CP_DMA_SYNC)
  73                 header |= S_411_CP_SYNC(1);
  74         else {
  75                 if (sctx->chip_class >= GFX9)
  76                         command |= S_414_DISABLE_WR_CONFIRM_GFX9(1);
  77                 else
  78                         command |= S_414_DISABLE_WR_CONFIRM_GFX6(1);
  79         }
  80
  81         if (flags & CP_DMA_RAW_WAIT)
  82                 command |= S_414_RAW_WAIT(1);
  83
  84         /* Src and dst flags. */
  85         if (sctx->chip_class >= GFX9 && !(flags & CP_DMA_CLEAR) &&
  86             src_va == dst_va) {
  87                 header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */
  88         } else if (flags & CP_DMA_DST_IS_GDS) {
  89                 header |= S_411_DST_SEL(V_411_GDS);
  90                 /* GDS increments the address, not CP. */
  91                 command |= S_414_DAS(V_414_REGISTER) |
  92                            S_414_DAIC(V_414_NO_INCREMENT);
  93         } else if (sctx->chip_class >= CIK && cache_policy != L2_BYPASS) {
  94                 header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2) |
  95                           S_500_DST_CACHE_POLICY(cache_policy == L2_STREAM);
  96         }
  97
  98         if (flags & CP_DMA_CLEAR) {
  99                 header |= S_411_SRC_SEL(V_411_DATA);
 100         } else if (flags & CP_DMA_SRC_IS_GDS) {
 101                 header |= S_411_SRC_SEL(V_411_GDS);
 102                 /* Both of these are required for GDS. It does increment the address. */
 103                 command |= S_414_SAS(V_414_REGISTER) |
 104                            S_414_SAIC(V_414_NO_INCREMENT);
 105         } else if (sctx->chip_class >= CIK && cache_policy != L2_BYPASS) {
 106                 header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2) |
 107                           S_500_SRC_CACHE_POLICY(cache_policy == L2_STREAM);
 108         }
 109
 110         if (sctx->chip_class >= CIK) {
 111                 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
 112                 radeon_emit(cs, header);
 113                 radeon_emit(cs, src_va);        /* SRC_ADDR_LO [31:0] */
 114                 radeon_emit(cs, src_va >> 32);  /* SRC_ADDR_HI [31:0] */
 115                 radeon_emit(cs, dst_va);        /* DST_ADDR_LO [31:0] */
 116                 radeon_emit(cs, dst_va >> 32);  /* DST_ADDR_HI [31:0] */
 117                 radeon_emit(cs, command);
 118         } else {
 119                 header |= S_411_SRC_ADDR_HI(src_va >> 32);
 120
 121                 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
 122                 radeon_emit(cs, src_va);        /* SRC_ADDR_LO [31:0] */
 123                 radeon_emit(cs, header);        /* SRC_ADDR_HI [15:0] + flags. */
 124                 radeon_emit(cs, dst_va);        /* DST_ADDR_LO [31:0] */
 125                 radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
 126                 radeon_emit(cs, command);
 127         }
 128
 129         /* CP DMA is executed in ME, but index buffers are read by PFP.
 130          * This ensures that ME (CP DMA) is idle before PFP starts fetching
 131          * indices. If we wanted to execute CP DMA in PFP, this packet
 132          * should precede it.
 133          */
 134         if (flags & CP_DMA_PFP_SYNC_ME) {
 135                 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
 136                 radeon_emit(cs, 0);
 137         }
 138 }
 139
 140 void si_cp_dma_wait_for_idle(struct si_context *sctx)
 141 {
 142         /* Issue a dummy DMA that copies zero bytes.
 143          *
 144          * The DMA engine will see that there's no work to do and skip this
 145          * DMA request, however, the CP will see the sync flag and still wait
 146          * for all DMAs to complete.
 147          */
 148         si_emit_cp_dma(sctx, sctx->gfx_cs, 0, 0, 0, CP_DMA_SYNC, L2_BYPASS);
 149 }
 150
 151 static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
 152                               struct pipe_resource *src, unsigned byte_count,
 153                               uint64_t remaining_size, unsigned user_flags,
 154                               enum si_coherency coher, bool *is_first,
 155                               unsigned *packet_flags)
 156 {
 157         /* Fast exit for a CPDMA prefetch. */
 158         if ((user_flags & SI_CPDMA_SKIP_ALL) == SI_CPDMA_SKIP_ALL) {
 159                 *is_first = false;
 160                 return;
 161         }
 162
 163         if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
 164                 /* Count memory usage in so that need_cs_space can take it into account. */
 165                 if (dst)
 166                         si_context_add_resource_size(sctx, dst);
 167                 if (src)
 168                         si_context_add_resource_size(sctx, src);
 169         }
 170
 171         if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
 172                 si_need_gfx_cs_space(sctx);
 173
 174         /* This must be done after need_cs_space. */
 175         if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
 176                 if (dst)
 177                         radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
 178                                                   r600_resource(dst),
 179                                                   RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
 180                 if (src)
 181                         radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
 182                                                   r600_resource(src),
 183                                                   RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
 184         }
 185
 186         /* Flush the caches for the first copy only.
 187          * Also wait for the previous CP DMA operations.
 188          */
 189         if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->flags)
 190                 si_emit_cache_flush(sctx);
 191
 192         if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first)
 193                 *packet_flags |= CP_DMA_RAW_WAIT;
 194
 195         *is_first = false;
 196
 197         /* Do the synchronization after the last dma, so that all data
 198          * is written to memory.
 199          */
 200         if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) &&
 201             byte_count == remaining_size) {
 202                 *packet_flags |= CP_DMA_SYNC;
 203
 204                 if (coher == SI_COHERENCY_SHADER)
 205                         *packet_flags |= CP_DMA_PFP_SYNC_ME;
 206         }
 207 }
 208
 209 void si_cp_dma_clear_buffer(struct si_context *sctx, struct radeon_cmdbuf *cs,
 210                             struct pipe_resource *dst, uint64_t offset,
 211                             uint64_t size, unsigned value, unsigned user_flags,
 212                             enum si_coherency coher, enum si_cache_policy cache_policy)
 213 {
 214         struct r600_resource *rdst = r600_resource(dst);
 215         uint64_t va = (rdst ? rdst->gpu_address : 0) + offset;
 216         bool is_first = true;
 217
 218         assert(size && size % 4 == 0);
 219
 220         /* Mark the buffer range of destination as valid (initialized),
 221          * so that transfer_map knows it should wait for the GPU when mapping
 222          * that range. */
 223         if (rdst)
 224                 util_range_add(&rdst->valid_buffer_range, offset, offset + size);
 225
 226         /* Flush the caches. */
 227         if (rdst && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
 228                 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
 229                                SI_CONTEXT_CS_PARTIAL_FLUSH |
 230                                si_get_flush_flags(sctx, coher, cache_policy);
 231         }
 232
 233         while (size) {
 234                 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
 235                 unsigned dma_flags = CP_DMA_CLEAR | (rdst ? 0 : CP_DMA_DST_IS_GDS);
 236
 237                 si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, user_flags,
 238                                   coher, &is_first, &dma_flags);
 239
 240                 /* Emit the clear packet. */
 241                 si_emit_cp_dma(sctx, cs, va, value, byte_count, dma_flags, cache_policy);
 242
 243                 size -= byte_count;
 244                 va += byte_count;
 245         }
 246
 247         if (rdst && cache_policy != L2_BYPASS)
 248                 rdst->TC_L2_dirty = true;
 249
 250         /* If it's not a framebuffer fast clear... */
 251         if (coher == SI_COHERENCY_SHADER)
 252                 sctx->num_cp_dma_calls++;
 253 }
 254
 255 /**
 256  * Realign the CP DMA engine. This must be done after a copy with an unaligned
 257  * size.
 258  *
 259  * \param size  Remaining size to the CP DMA alignment.
 260  */
 261 static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
 262                                      unsigned user_flags, enum si_coherency coher,
 263                                      enum si_cache_policy cache_policy,
 264                                      bool *is_first)
 265 {
 266         uint64_t va;
 267         unsigned dma_flags = 0;
 268         unsigned scratch_size = SI_CPDMA_ALIGNMENT * 2;
 269
 270         assert(size < SI_CPDMA_ALIGNMENT);
 271
 272         /* Use the scratch buffer as the dummy buffer. The 3D engine should be
 273          * idle at this point.
 274          */
 275         if (!sctx->scratch_buffer ||
 276             sctx->scratch_buffer->b.b.width0 < scratch_size) {
 277                 r600_resource_reference(&sctx->scratch_buffer, NULL);
 278                 sctx->scratch_buffer =
 279                         si_aligned_buffer_create(&sctx->screen->b,
 280                                                    SI_RESOURCE_FLAG_UNMAPPABLE,
 281                                                    PIPE_USAGE_DEFAULT,
 282                                                    scratch_size, 256);
 283                 if (!sctx->scratch_buffer)
 284                         return;
 285
 286                 si_mark_atom_dirty(sctx, &sctx->atoms.s.scratch_state);
 287         }
 288
 289         si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b,
 290                           &sctx->scratch_buffer->b.b, size, size, user_flags,
 291                           coher, is_first, &dma_flags);
 292
 293         va = sctx->scratch_buffer->gpu_address;
 294         si_emit_cp_dma(sctx, sctx->gfx_cs, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags,
 295                        cache_policy);
 296 }
 297
 298 /**
 299  * Do memcpy between buffers using CP DMA.
 300  * If src or dst is NULL, it means read or write GDS, respectively.
 301  *
 302  * \param user_flags    bitmask of SI_CPDMA_*
 303  */
 304 void si_cp_dma_copy_buffer(struct si_context *sctx,
 305                            struct pipe_resource *dst, struct pipe_resource *src,
 306                            uint64_t dst_offset, uint64_t src_offset, unsigned size,
 307                            unsigned user_flags, enum si_coherency coher,
 308                            enum si_cache_policy cache_policy)
 309 {
 310         uint64_t main_dst_offset, main_src_offset;
 311         unsigned skipped_size = 0;
 312         unsigned realign_size = 0;
 313         unsigned gds_flags = (dst ? 0 : CP_DMA_DST_IS_GDS) |
 314                              (src ? 0 : CP_DMA_SRC_IS_GDS);
 315         bool is_first = true;
 316
 317         assert(size);
 318
 319         if (dst) {
 320                 /* Skip this for the L2 prefetch. */
 321                 if (dst != src || dst_offset != src_offset) {
 322                         /* Mark the buffer range of destination as valid (initialized),
 323                          * so that transfer_map knows it should wait for the GPU when mapping
 324                          * that range. */
 325                         util_range_add(&r600_resource(dst)->valid_buffer_range, dst_offset,
 326                                        dst_offset + size);
 327                 }
 328
 329                 dst_offset += r600_resource(dst)->gpu_address;
 330         }
 331         if (src)
 332                 src_offset += r600_resource(src)->gpu_address;
 333
 334         /* The workarounds aren't needed on Fiji and beyond. */
 335         if (sctx->family <= CHIP_CARRIZO ||
 336             sctx->family == CHIP_STONEY) {
 337                 /* If the size is not aligned, we must add a dummy copy at the end
 338                  * just to align the internal counter. Otherwise, the DMA engine
 339                  * would slow down by an order of magnitude for following copies.
 340                  */
 341                 if (size % SI_CPDMA_ALIGNMENT)
 342                         realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);
 343
 344                 /* If the copy begins unaligned, we must start copying from the next
 345                  * aligned block and the skipped part should be copied after everything
 346                  * else has been copied. Only the src alignment matters, not dst.
 347                  *
 348                  * GDS doesn't need the source address to be aligned.
 349                  */
 350                 if (src && src_offset % SI_CPDMA_ALIGNMENT) {
 351                         skipped_size = SI_CPDMA_ALIGNMENT - (src_offset % SI_CPDMA_ALIGNMENT);
 352                         /* The main part will be skipped if the size is too small. */
 353                         skipped_size = MIN2(skipped_size, size);
 354                         size -= skipped_size;
 355                 }
 356         }
 357
 358         /* Flush the caches. */
 359         if ((dst || src) && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
 360                 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
 361                                SI_CONTEXT_CS_PARTIAL_FLUSH |
 362                                si_get_flush_flags(sctx, coher, cache_policy);
 363         }
 364
 365         /* This is the main part doing the copying. Src is always aligned. */
 366         main_dst_offset = dst_offset + skipped_size;
 367         main_src_offset = src_offset + skipped_size;
 368
 369         while (size) {
 370                 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
 371                 unsigned dma_flags = gds_flags;
 372
 373                 si_cp_dma_prepare(sctx, dst, src, byte_count,
 374                                   size + skipped_size + realign_size,
 375                                   user_flags, coher, &is_first, &dma_flags);
 376
 377                 si_emit_cp_dma(sctx, sctx->gfx_cs, main_dst_offset, main_src_offset,
 378                                byte_count, dma_flags, cache_policy);
 379
 380                 size -= byte_count;
 381                 main_src_offset += byte_count;
 382                 main_dst_offset += byte_count;
 383         }
 384
 385         /* Copy the part we skipped because src wasn't aligned. */
 386         if (skipped_size) {
 387                 unsigned dma_flags = gds_flags;
 388
 389                 si_cp_dma_prepare(sctx, dst, src, skipped_size,
 390                                   skipped_size + realign_size, user_flags,
 391                                   coher, &is_first, &dma_flags);
 392
 393                 si_emit_cp_dma(sctx, sctx->gfx_cs, dst_offset, src_offset, skipped_size,
 394                                dma_flags, cache_policy);
 395         }
 396
 397         /* Finally, realign the engine if the size wasn't aligned. */
 398         if (realign_size) {
 399                 si_cp_dma_realign_engine(sctx, realign_size, user_flags, coher,
 400                                          cache_policy, &is_first);
 401         }
 402
 403         if (dst && cache_policy != L2_BYPASS)
 404                 r600_resource(dst)->TC_L2_dirty = true;
 405
 406         /* If it's not a prefetch or GDS copy... */
 407         if (dst && src && (dst != src || dst_offset != src_offset))
 408                 sctx->num_cp_dma_calls++;
 409 }
 410
 411 void cik_prefetch_TC_L2_async(struct si_context *sctx, struct pipe_resource *buf,
 412                               uint64_t offset, unsigned size)
 413 {
 414         assert(sctx->chip_class >= CIK);
 415
 416         si_cp_dma_copy_buffer(sctx, buf, buf, offset, offset, size,
 417                               SI_CPDMA_SKIP_ALL, SI_COHERENCY_SHADER, L2_LRU);
 418 }
 419
 420 static void cik_prefetch_shader_async(struct si_context *sctx,
 421                                       struct si_pm4_state *state)
 422 {
 423         struct pipe_resource *bo = &state->bo[0]->b.b;
 424         assert(state->nbo == 1);
 425
 426         cik_prefetch_TC_L2_async(sctx, bo, 0, bo->width0);
 427 }
 428
 429 static void cik_prefetch_VBO_descriptors(struct si_context *sctx)
 430 {
 431         if (!sctx->vertex_elements || !sctx->vertex_elements->desc_list_byte_size)
 432                 return;
 433
 434         cik_prefetch_TC_L2_async(sctx, &sctx->vb_descriptors_buffer->b.b,
 435                                  sctx->vb_descriptors_offset,
 436                                  sctx->vertex_elements->desc_list_byte_size);
 437 }
 438
 439 /**
 440  * Prefetch shaders and VBO descriptors.
 441  *
 442  * \param vertex_stage_only  Whether only the the API VS and VBO descriptors
 443  *                           should be prefetched.
 444  */
 445 void cik_emit_prefetch_L2(struct si_context *sctx, bool vertex_stage_only)
 446 {
 447         unsigned mask = sctx->prefetch_L2_mask;
 448         assert(mask);
 449
 450         /* Prefetch shaders and VBO descriptors to TC L2. */
 451         if (sctx->chip_class >= GFX9) {
 452                 /* Choose the right spot for the VBO prefetch. */
 453                 if (sctx->tes_shader.cso) {
 454                         if (mask & SI_PREFETCH_HS)
 455                                 cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
 456                         if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
 457                                 cik_prefetch_VBO_descriptors(sctx);
 458                         if (vertex_stage_only) {
 459                                 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_HS |
 460                                                             SI_PREFETCH_VBO_DESCRIPTORS);
 461                                 return;
 462                         }
 463
 464                         if (mask & SI_PREFETCH_GS)
 465                                 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
 466                         if (mask & SI_PREFETCH_VS)
 467                                 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
 468                 } else if (sctx->gs_shader.cso) {
 469                         if (mask & SI_PREFETCH_GS)
 470                                 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
 471                         if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
 472                                 cik_prefetch_VBO_descriptors(sctx);
 473                         if (vertex_stage_only) {
 474                                 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_GS |
 475                                                             SI_PREFETCH_VBO_DESCRIPTORS);
 476                                 return;
 477                         }
 478
 479                         if (mask & SI_PREFETCH_VS)
 480                                 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
 481                 } else {
 482                         if (mask & SI_PREFETCH_VS)
 483                                 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
 484                         if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
 485                                 cik_prefetch_VBO_descriptors(sctx);
 486                         if (vertex_stage_only) {
 487                                 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS |
 488                                                             SI_PREFETCH_VBO_DESCRIPTORS);
 489                                 return;
 490                         }
 491                 }
 492         } else {
 493                 /* SI-CI-VI */
 494                 /* Choose the right spot for the VBO prefetch. */
 495                 if (sctx->tes_shader.cso) {
 496                         if (mask & SI_PREFETCH_LS)
 497                                 cik_prefetch_shader_async(sctx, sctx->queued.named.ls);
 498                         if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
 499                                 cik_prefetch_VBO_descriptors(sctx);
 500                         if (vertex_stage_only) {
 501                                 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_LS |
 502                                                             SI_PREFETCH_VBO_DESCRIPTORS);
 503                                 return;
 504                         }
 505
 506                         if (mask & SI_PREFETCH_HS)
 507                                 cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
 508                         if (mask & SI_PREFETCH_ES)
 509                                 cik_prefetch_shader_async(sctx, sctx->queued.named.es);
 510                         if (mask & SI_PREFETCH_GS)
 511                                 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
 512                         if (mask & SI_PREFETCH_VS)
 513                                 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
 514                 } else if (sctx->gs_shader.cso) {
 515                         if (mask & SI_PREFETCH_ES)
 516                                 cik_prefetch_shader_async(sctx, sctx->queued.named.es);
 517                         if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
 518                                 cik_prefetch_VBO_descriptors(sctx);
 519                         if (vertex_stage_only) {
 520                                 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_ES |
 521                                                             SI_PREFETCH_VBO_DESCRIPTORS);
 522                                 return;
 523                         }
 524
 525                         if (mask & SI_PREFETCH_GS)
 526                                 cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
 527                         if (mask & SI_PREFETCH_VS)
 528                                 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
 529                 } else {
 530                         if (mask & SI_PREFETCH_VS)
 531                                 cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
 532                         if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
 533                                 cik_prefetch_VBO_descriptors(sctx);
 534                         if (vertex_stage_only) {
 535                                 sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS |
 536                                                             SI_PREFETCH_VBO_DESCRIPTORS);
 537                                 return;
 538                         }
 539                 }
 540         }
 541
 542         if (mask & SI_PREFETCH_PS)
 543                 cik_prefetch_shader_async(sctx, sctx->queued.named.ps);
 544
 545         sctx->prefetch_L2_mask = 0;
 546 }
 547
 548 void si_test_gds(struct si_context *sctx)
 549 {
 550         struct pipe_context *ctx = &sctx->b;
 551         struct pipe_resource *src, *dst;
 552         unsigned r[4] = {};
 553         unsigned offset = debug_get_num_option("OFFSET", 16);
 554
 555         src = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16);
 556         dst = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16);
 557         si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 0, 4, 0xabcdef01, 0, SI_COHERENCY_SHADER, L2_BYPASS);
 558         si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 4, 4, 0x23456789, 0, SI_COHERENCY_SHADER, L2_BYPASS);
 559         si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 8, 4, 0x87654321, 0, SI_COHERENCY_SHADER, L2_BYPASS);
 560         si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 12, 4, 0xfedcba98, 0, SI_COHERENCY_SHADER, L2_BYPASS);
 561         si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, dst, 0, 16, 0xdeadbeef, 0, SI_COHERENCY_SHADER, L2_BYPASS);
 562
 563         si_cp_dma_copy_buffer(sctx, NULL, src, offset, 0, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
 564         si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
 565
 566         pipe_buffer_read(ctx, dst, 0, sizeof(r), r);
 567         printf("GDS copy  = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3],
 568                         r[0] == 0xabcdef01 && r[1] == 0x23456789 &&
 569                         r[2] == 0x87654321 && r[3] == 0xfedcba98 ? "pass" : "fail");
 570
 571         si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, NULL, offset, 16, 0xc1ea4146, 0, SI_COHERENCY_NONE, L2_BYPASS);
 572         si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
 573
 574         pipe_buffer_read(ctx, dst, 0, sizeof(r), r);
 575         printf("GDS clear = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3],
 576                         r[0] == 0xc1ea4146 && r[1] == 0xc1ea4146 &&
 577                         r[2] == 0xc1ea4146 && r[3] == 0xc1ea4146 ? "pass" : "fail");
 578
 579         pipe_resource_reference(&src, NULL);
 580         pipe_resource_reference(&dst, NULL);
 581         exit(0);
 582 }