src/gallium/drivers/radeonsi/si_dma_cs.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "si_pipe.h"
  26 #include "sid.h"
  27
  28 static void si_dma_emit_wait_idle(struct si_context *sctx)
  29 {
  30         struct radeon_cmdbuf *cs = sctx->sdma_cs;
  31
  32         /* NOP waits for idle. */
  33         if (sctx->chip_class >= GFX7)
  34                 radeon_emit(cs, 0x00000000); /* NOP */
  35         else
  36                 radeon_emit(cs, 0xf0000000); /* NOP */
  37 }
  38
  39 void si_dma_emit_timestamp(struct si_context *sctx, struct si_resource *dst,
  40                            uint64_t offset)
  41 {
  42         struct radeon_cmdbuf *cs = sctx->sdma_cs;
  43         uint64_t va = dst->gpu_address + offset;
  44
  45         if (sctx->chip_class == GFX6) {
  46                 unreachable("SI DMA doesn't support the timestamp packet.");
  47                 return;
  48         }
  49
  50         /* Mark the buffer range of destination as valid (initialized),
  51          * so that transfer_map knows it should wait for the GPU when mapping
  52          * that range. */
  53         util_range_add(&dst->b.b, &dst->valid_buffer_range, offset, offset + 8);
  54
  55         assert(va % 8 == 0);
  56
  57         si_need_dma_space(sctx, 4, dst, NULL);
  58         si_dma_emit_wait_idle(sctx);
  59
  60         radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_TIMESTAMP,
  61                                         SDMA_TS_SUB_OPCODE_GET_GLOBAL_TIMESTAMP,
  62                                         0));
  63         radeon_emit(cs, va);
  64         radeon_emit(cs, va >> 32);
  65 }
  66
  67 void si_sdma_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
  68                           uint64_t offset, uint64_t size, unsigned clear_value)
  69 {
  70         struct radeon_cmdbuf *cs = sctx->sdma_cs;
  71         unsigned i, ncopy, csize;
  72         struct si_resource *sdst = si_resource(dst);
  73
  74         assert(offset % 4 == 0);
  75         assert(size);
  76         assert(size % 4 == 0);
  77
  78         if (!cs || dst->flags & PIPE_RESOURCE_FLAG_SPARSE ||
  79             sctx->screen->debug_flags & DBG(NO_SDMA_CLEARS)) {
  80                 sctx->b.clear_buffer(&sctx->b, dst, offset, size, &clear_value, 4);
  81                 return;
  82         }
  83
  84         /* Mark the buffer range of destination as valid (initialized),
  85          * so that transfer_map knows it should wait for the GPU when mapping
  86          * that range. */
  87         util_range_add(dst, &sdst->valid_buffer_range, offset, offset + size);
  88
  89         offset += sdst->gpu_address;
  90
  91         if (sctx->chip_class == GFX6) {
  92                 /* the same maximum size as for copying */
  93                 ncopy = DIV_ROUND_UP(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE);
  94                 si_need_dma_space(sctx, ncopy * 4, sdst, NULL);
  95
  96                 for (i = 0; i < ncopy; i++) {
  97                         csize = MIN2(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE);
  98                         radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_CONSTANT_FILL, 0,
  99                                                       csize / 4));
 100                         radeon_emit(cs, offset);
 101                         radeon_emit(cs, clear_value);
 102                         radeon_emit(cs, (offset >> 32) << 16);
 103                         offset += csize;
 104                         size -= csize;
 105                 }
 106                 return;
 107         }
 108
 109         /* The following code is for Sea Islands and later. */
 110         /* the same maximum size as for copying */
 111         ncopy = DIV_ROUND_UP(size, CIK_SDMA_COPY_MAX_SIZE);
 112         si_need_dma_space(sctx, ncopy * 5, sdst, NULL);
 113
 114         for (i = 0; i < ncopy; i++) {
 115                 csize = MIN2(size, CIK_SDMA_COPY_MAX_SIZE);
 116                 radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_PACKET_CONSTANT_FILL, 0,
 117                                                 0x8000 /* dword copy */));
 118                 radeon_emit(cs, offset);
 119                 radeon_emit(cs, offset >> 32);
 120                 radeon_emit(cs, clear_value);
 121                 /* dw count */
 122                 radeon_emit(cs, (sctx->chip_class >= GFX9 ? csize - 1 : csize) & 0xfffffffc);
 123                 offset += csize;
 124                 size -= csize;
 125         }
 126 }
 127
 128 void si_need_dma_space(struct si_context *ctx, unsigned num_dw,
 129                        struct si_resource *dst, struct si_resource *src)
 130 {
 131         struct radeon_winsys *ws = ctx->ws;
 132         uint64_t vram = ctx->sdma_cs->used_vram;
 133         uint64_t gtt = ctx->sdma_cs->used_gart;
 134
 135         if (dst) {
 136                 vram += dst->vram_usage;
 137                 gtt += dst->gart_usage;
 138         }
 139         if (src) {
 140                 vram += src->vram_usage;
 141                 gtt += src->gart_usage;
 142         }
 143
 144         /* Flush the GFX IB if DMA depends on it. */
 145         if (!ctx->sdma_uploads_in_progress &&
 146             radeon_emitted(ctx->gfx_cs, ctx->initial_gfx_cs_size) &&
 147             ((dst &&
 148               ws->cs_is_buffer_referenced(ctx->gfx_cs, dst->buf,
 149                                           RADEON_USAGE_READWRITE)) ||
 150              (src &&
 151               ws->cs_is_buffer_referenced(ctx->gfx_cs, src->buf,
 152                                           RADEON_USAGE_WRITE))))
 153                 si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
 154
 155         /* Flush if there's not enough space, or if the memory usage per IB
 156          * is too large.
 157          *
 158          * IBs using too little memory are limited by the IB submission overhead.
 159          * IBs using too much memory are limited by the kernel/TTM overhead.
 160          * Too long IBs create CPU-GPU pipeline bubbles and add latency.
 161          *
 162          * This heuristic makes sure that DMA requests are executed
 163          * very soon after the call is made and lowers memory usage.
 164          * It improves texture upload performance by keeping the DMA
 165          * engine busy while uploads are being submitted.
 166          */
 167         num_dw++; /* for emit_wait_idle below */
 168         if (!ctx->sdma_uploads_in_progress &&
 169             (!ws->cs_check_space(ctx->sdma_cs, num_dw, false) ||
 170              ctx->sdma_cs->used_vram + ctx->sdma_cs->used_gart > 64 * 1024 * 1024 ||
 171              !radeon_cs_memory_below_limit(ctx->screen, ctx->sdma_cs, vram, gtt))) {
 172                 si_flush_dma_cs(ctx, PIPE_FLUSH_ASYNC, NULL);
 173                 assert((num_dw + ctx->sdma_cs->current.cdw) <= ctx->sdma_cs->current.max_dw);
 174         }
 175
 176         /* Wait for idle if either buffer has been used in the IB before to
 177          * prevent read-after-write hazards.
 178          */
 179         if ((dst &&
 180              ws->cs_is_buffer_referenced(ctx->sdma_cs, dst->buf,
 181                                          RADEON_USAGE_READWRITE)) ||
 182             (src &&
 183              ws->cs_is_buffer_referenced(ctx->sdma_cs, src->buf,
 184                                          RADEON_USAGE_WRITE)))
 185                 si_dma_emit_wait_idle(ctx);
 186
 187         unsigned sync = ctx->sdma_uploads_in_progress ? 0 : RADEON_USAGE_SYNCHRONIZED;
 188         if (dst) {
 189                 ws->cs_add_buffer(ctx->sdma_cs, dst->buf, RADEON_USAGE_WRITE | sync,
 190                                   dst->domains, 0);
 191         }
 192         if (src) {
 193                 ws->cs_add_buffer(ctx->sdma_cs, src->buf, RADEON_USAGE_READ | sync,
 194                                   src->domains, 0);
 195         }
 196
 197         /* this function is called before all DMA calls, so increment this. */
 198         ctx->num_dma_calls++;
 199 }
 200
 201 void si_flush_dma_cs(struct si_context *ctx, unsigned flags,
 202                      struct pipe_fence_handle **fence)
 203 {
 204         struct radeon_cmdbuf *cs = ctx->sdma_cs;
 205         struct radeon_saved_cs saved;
 206         bool check_vm = (ctx->screen->debug_flags & DBG(CHECK_VM)) != 0;
 207
 208         if (!radeon_emitted(cs, 0)) {
 209                 if (fence)
 210                         ctx->ws->fence_reference(fence, ctx->last_sdma_fence);
 211                 return;
 212         }
 213
 214         if (check_vm)
 215                 si_save_cs(ctx->ws, cs, &saved, true);
 216
 217         ctx->ws->cs_flush(cs, flags, &ctx->last_sdma_fence);
 218         if (fence)
 219                 ctx->ws->fence_reference(fence, ctx->last_sdma_fence);
 220
 221         if (check_vm) {
 222                 /* Use conservative timeout 800ms, after which we won't wait any
 223                  * longer and assume the GPU is hung.
 224                  */
 225                 ctx->ws->fence_wait(ctx->ws, ctx->last_sdma_fence, 800*1000*1000);
 226
 227                 si_check_vm_faults(ctx, &saved, RING_DMA);
 228                 si_clear_saved_cs(&saved);
 229         }
 230 }
 231
 232 void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst,
 233                             uint64_t offset, uint64_t size, unsigned value)
 234 {
 235         struct si_context *ctx = (struct si_context*)sscreen->aux_context;
 236
 237         simple_mtx_lock(&sscreen->aux_context_lock);
 238         si_sdma_clear_buffer(ctx, dst, offset, size, value);
 239         sscreen->aux_context->flush(sscreen->aux_context, NULL, 0);
 240         simple_mtx_unlock(&sscreen->aux_context_lock);
 241 }