src/gallium/drivers/radeonsi/si_dma_cs.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "si_pipe.h"
  26 #include "sid.h"
  27
  28 static void si_dma_emit_wait_idle(struct si_context *sctx)
  29 {
  30         struct radeon_cmdbuf *cs = sctx->sdma_cs;
  31
  32         /* NOP waits for idle. */
  33         if (sctx->chip_class >= GFX7)
  34                 radeon_emit(cs, 0x00000000); /* NOP */
  35         else
  36                 radeon_emit(cs, 0xf0000000); /* NOP */
  37 }
  38
  39 void si_dma_emit_timestamp(struct si_context *sctx, struct si_resource *dst,
  40                            uint64_t offset)
  41 {
  42         struct radeon_cmdbuf *cs = sctx->sdma_cs;
  43         uint64_t va = dst->gpu_address + offset;
  44
  45         if (sctx->chip_class == GFX6) {
  46                 unreachable("SI DMA doesn't support the timestamp packet.");
  47                 return;
  48         }
  49
  50         /* Mark the buffer range of destination as valid (initialized),
  51          * so that transfer_map knows it should wait for the GPU when mapping
  52          * that range. */
  53         util_range_add(&dst->b.b, &dst->valid_buffer_range, offset, offset + 8);
  54
  55         assert(va % 8 == 0);
  56
  57         si_need_dma_space(sctx, 4, dst, NULL);
  58         si_dma_emit_wait_idle(sctx);
  59
  60         radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_TIMESTAMP,
  61                                         SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP,
  62                                         0));
  63         radeon_emit(cs, va);
  64         radeon_emit(cs, va >> 32);
  65 }
  66
  67 void si_sdma_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
  68                           uint64_t offset, uint64_t size, unsigned clear_value)
  69 {
  70         struct radeon_cmdbuf *cs = sctx->sdma_cs;
  71         unsigned i, ncopy, csize;
  72         struct si_resource *sdst = si_resource(dst);
  73
  74         assert(offset % 4 == 0);
  75         assert(size);
  76         assert(size % 4 == 0);
  77
  78         if (!cs || dst->flags & PIPE_RESOURCE_FLAG_SPARSE ||
  79             sctx->screen->debug_flags & DBG(NO_SDMA_CLEARS)) {
  80                 sctx->b.clear_buffer(&sctx->b, dst, offset, size, &clear_value, 4);
  81                 return;
  82         }
  83
  84         /* Mark the buffer range of destination as valid (initialized),
  85          * so that transfer_map knows it should wait for the GPU when mapping
  86          * that range. */
  87         util_range_add(dst, &sdst->valid_buffer_range, offset, offset + size);
  88
  89         offset += sdst->gpu_address;
  90
  91         if (sctx->chip_class == GFX6) {
  92                 /* the same maximum size as for copying */
  93                 ncopy = DIV_ROUND_UP(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE);
  94                 si_need_dma_space(sctx, ncopy * 4, sdst, NULL);
  95
  96                 for (i = 0; i < ncopy; i++) {
  97                         csize = MIN2(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE);
  98                         radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_CONSTANT_FILL, 0,
  99                                                       csize / 4));
 100                         radeon_emit(cs, offset);
 101                         radeon_emit(cs, clear_value);
 102                         radeon_emit(cs, (offset >> 32) << 16);
 103                         offset += csize;
 104                         size -= csize;
 105                 }
 106                 return;
 107         }
 108
 109         /* The following code is for Sea Islands and later. */
 110         /* the same maximum size as for copying */
 111         ncopy = DIV_ROUND_UP(size, CIK_SDMA_COPY_MAX_SIZE);
 112         si_need_dma_space(sctx, ncopy * 5, sdst, NULL);
 113
 114         for (i = 0; i < ncopy; i++) {
 115                 csize = MIN2(size, CIK_SDMA_COPY_MAX_SIZE);
 116                 radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_PACKET_CONSTANT_FILL, 0,
 117                                                 0x8000 /* dword copy */));
 118                 radeon_emit(cs, offset);
 119                 radeon_emit(cs, offset >> 32);
 120                 radeon_emit(cs, clear_value);
 121                 /* dw count */
 122                 radeon_emit(cs, (sctx->chip_class >= GFX9 ? csize - 1 : csize) & 0xfffffffc);
 123                 offset += csize;
 124                 size -= csize;
 125         }
 126 }
 127
 128 void si_sdma_copy_buffer(struct si_context *sctx, struct pipe_resource *dst,
 129                          struct pipe_resource *src, uint64_t dst_offset,
 130                          uint64_t src_offset, uint64_t size)
 131 {
 132         struct radeon_cmdbuf *cs = sctx->sdma_cs;
 133         unsigned i, ncopy, csize;
 134         struct si_resource *sdst = si_resource(dst);
 135         struct si_resource *ssrc = si_resource(src);
 136
 137         if (!cs ||
 138             dst->flags & PIPE_RESOURCE_FLAG_SPARSE ||
 139             src->flags & PIPE_RESOURCE_FLAG_SPARSE) {
 140                 si_copy_buffer(sctx, dst, src, dst_offset, src_offset, size);
 141                 return;
 142         }
 143
 144         /* Mark the buffer range of destination as valid (initialized),
 145          * so that transfer_map knows it should wait for the GPU when mapping
 146          * that range. */
 147         util_range_add(dst, &sdst->valid_buffer_range, dst_offset,
 148                        dst_offset + size);
 149
 150         dst_offset += sdst->gpu_address;
 151         src_offset += ssrc->gpu_address;
 152
 153         if (sctx->chip_class == GFX6) {
 154                 unsigned max_size, sub_cmd, shift;
 155
 156                 /* see whether we should use the dword-aligned or byte-aligned copy */
 157                 if (!(dst_offset % 4) && !(src_offset % 4) && !(size % 4)) {
 158                         sub_cmd = SI_DMA_COPY_DWORD_ALIGNED;
 159                         shift = 2;
 160                         max_size = SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE;
 161                 } else {
 162                         sub_cmd = SI_DMA_COPY_BYTE_ALIGNED;
 163                         shift = 0;
 164                         max_size = SI_DMA_COPY_MAX_BYTE_ALIGNED_SIZE;
 165                 }
 166
 167                 ncopy = DIV_ROUND_UP(size, max_size);
 168                 si_need_dma_space(sctx, ncopy * 5, sdst, ssrc);
 169
 170                 for (i = 0; i < ncopy; i++) {
 171                         csize = MIN2(size, max_size);
 172                         radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd,
 173                                                       csize >> shift));
 174                         radeon_emit(cs, dst_offset);
 175                         radeon_emit(cs, src_offset);
 176                         radeon_emit(cs, (dst_offset >> 32UL) & 0xff);
 177                         radeon_emit(cs, (src_offset >> 32UL) & 0xff);
 178                         dst_offset += csize;
 179                         src_offset += csize;
 180                         size -= csize;
 181                 }
 182                 return;
 183         }
 184
 185         /* The following code is for CI and later. */
 186         unsigned align = ~0u;
 187         ncopy = DIV_ROUND_UP(size, CIK_SDMA_COPY_MAX_SIZE);
 188
 189         /* Align copy size to dw if src/dst address are dw aligned */
 190         if ((src_offset & 0x3) == 0 &&
 191             (dst_offset & 0x3) == 0 &&
 192             size > 4 &&
 193             (size & 3) != 0) {
 194                 align = ~0x3u;
 195                 ncopy++;
 196         }
 197
 198         si_need_dma_space(sctx, ncopy * 7, sdst, ssrc);
 199
 200         for (i = 0; i < ncopy; i++) {
 201                 csize = size >= 4 ? MIN2(size & align, CIK_SDMA_COPY_MAX_SIZE) : size;
 202                 radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY,
 203                                                 CIK_SDMA_COPY_SUB_OPCODE_LINEAR,
 204                                                 0));
 205                 radeon_emit(cs, sctx->chip_class >= GFX9 ? csize - 1 : csize);
 206                 radeon_emit(cs, 0); /* src/dst endian swap */
 207                 radeon_emit(cs, src_offset);
 208                 radeon_emit(cs, src_offset >> 32);
 209                 radeon_emit(cs, dst_offset);
 210                 radeon_emit(cs, dst_offset >> 32);
 211                 dst_offset += csize;
 212                 src_offset += csize;
 213                 size -= csize;
 214         }
 215 }
 216
 217 void si_need_dma_space(struct si_context *ctx, unsigned num_dw,
 218                        struct si_resource *dst, struct si_resource *src)
 219 {
 220         struct radeon_winsys *ws = ctx->ws;
 221         uint64_t vram = ctx->sdma_cs->used_vram;
 222         uint64_t gtt = ctx->sdma_cs->used_gart;
 223
 224         if (dst) {
 225                 vram += dst->vram_usage;
 226                 gtt += dst->gart_usage;
 227         }
 228         if (src) {
 229                 vram += src->vram_usage;
 230                 gtt += src->gart_usage;
 231         }
 232
 233         /* Flush the GFX IB if DMA depends on it. */
 234         if (!ctx->sdma_uploads_in_progress &&
 235             radeon_emitted(ctx->gfx_cs, ctx->initial_gfx_cs_size) &&
 236             ((dst &&
 237               ws->cs_is_buffer_referenced(ctx->gfx_cs, dst->buf,
 238                                           RADEON_USAGE_READWRITE)) ||
 239              (src &&
 240               ws->cs_is_buffer_referenced(ctx->gfx_cs, src->buf,
 241                                           RADEON_USAGE_WRITE))))
 242                 si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
 243
 244         /* Flush if there's not enough space, or if the memory usage per IB
 245          * is too large.
 246          *
 247          * IBs using too little memory are limited by the IB submission overhead.
 248          * IBs using too much memory are limited by the kernel/TTM overhead.
 249          * Too long IBs create CPU-GPU pipeline bubbles and add latency.
 250          *
 251          * This heuristic makes sure that DMA requests are executed
 252          * very soon after the call is made and lowers memory usage.
 253          * It improves texture upload performance by keeping the DMA
 254          * engine busy while uploads are being submitted.
 255          */
 256         num_dw++; /* for emit_wait_idle below */
 257         if (!ctx->sdma_uploads_in_progress &&
 258             (!ws->cs_check_space(ctx->sdma_cs, num_dw, false) ||
 259              ctx->sdma_cs->used_vram + ctx->sdma_cs->used_gart > 64 * 1024 * 1024 ||
 260              !radeon_cs_memory_below_limit(ctx->screen, ctx->sdma_cs, vram, gtt))) {
 261                 si_flush_dma_cs(ctx, PIPE_FLUSH_ASYNC, NULL);
 262                 assert((num_dw + ctx->sdma_cs->current.cdw) <= ctx->sdma_cs->current.max_dw);
 263         }
 264
 265         /* Wait for idle if either buffer has been used in the IB before to
 266          * prevent read-after-write hazards.
 267          */
 268         if ((dst &&
 269              ws->cs_is_buffer_referenced(ctx->sdma_cs, dst->buf,
 270                                          RADEON_USAGE_READWRITE)) ||
 271             (src &&
 272              ws->cs_is_buffer_referenced(ctx->sdma_cs, src->buf,
 273                                          RADEON_USAGE_WRITE)))
 274                 si_dma_emit_wait_idle(ctx);
 275
 276         unsigned sync = ctx->sdma_uploads_in_progress ? 0 : RADEON_USAGE_SYNCHRONIZED;
 277         if (dst) {
 278                 ws->cs_add_buffer(ctx->sdma_cs, dst->buf, RADEON_USAGE_WRITE | sync,
 279                                   dst->domains, 0);
 280         }
 281         if (src) {
 282                 ws->cs_add_buffer(ctx->sdma_cs, src->buf, RADEON_USAGE_READ | sync,
 283                                   src->domains, 0);
 284         }
 285
 286         /* this function is called before all DMA calls, so increment this. */
 287         ctx->num_dma_calls++;
 288 }
 289
 290 void si_flush_dma_cs(struct si_context *ctx, unsigned flags,
 291                      struct pipe_fence_handle **fence)
 292 {
 293         struct radeon_cmdbuf *cs = ctx->sdma_cs;
 294         struct radeon_saved_cs saved;
 295         bool check_vm = (ctx->screen->debug_flags & DBG(CHECK_VM)) != 0;
 296
 297         if (!radeon_emitted(cs, 0)) {
 298                 if (fence)
 299                         ctx->ws->fence_reference(fence, ctx->last_sdma_fence);
 300                 return;
 301         }
 302
 303         if (check_vm)
 304                 si_save_cs(ctx->ws, cs, &saved, true);
 305
 306         ctx->ws->cs_flush(cs, flags, &ctx->last_sdma_fence);
 307         if (fence)
 308                 ctx->ws->fence_reference(fence, ctx->last_sdma_fence);
 309
 310         if (check_vm) {
 311                 /* Use conservative timeout 800ms, after which we won't wait any
 312                  * longer and assume the GPU is hung.
 313                  */
 314                 ctx->ws->fence_wait(ctx->ws, ctx->last_sdma_fence, 800*1000*1000);
 315
 316                 si_check_vm_faults(ctx, &saved, RING_DMA);
 317                 si_clear_saved_cs(&saved);
 318         }
 319 }
 320
 321 void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst,
 322                             uint64_t offset, uint64_t size, unsigned value)
 323 {
 324         struct si_context *ctx = (struct si_context*)sscreen->aux_context;
 325
 326         simple_mtx_lock(&sscreen->aux_context_lock);
 327         si_sdma_clear_buffer(ctx, dst, offset, size, value);
 328         sscreen->aux_context->flush(sscreen->aux_context, NULL, 0);
 329         simple_mtx_unlock(&sscreen->aux_context_lock);
 330 }