src/gallium/drivers/radeonsi/si_dma_cs.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "si_pipe.h"
  26
  27 static void si_dma_emit_wait_idle(struct si_context *sctx)
  28 {
  29         struct radeon_winsys_cs *cs = sctx->dma_cs;
  30
  31         /* NOP waits for idle on Evergreen and later. */
  32         if (sctx->chip_class >= CIK)
  33                 radeon_emit(cs, 0x00000000); /* NOP */
  34         else
  35                 radeon_emit(cs, 0xf0000000); /* NOP */
  36 }
  37
  38 void si_need_dma_space(struct si_context *ctx, unsigned num_dw,
  39                        struct r600_resource *dst, struct r600_resource *src)
  40 {
  41         uint64_t vram = ctx->dma_cs->used_vram;
  42         uint64_t gtt = ctx->dma_cs->used_gart;
  43
  44         if (dst) {
  45                 vram += dst->vram_usage;
  46                 gtt += dst->gart_usage;
  47         }
  48         if (src) {
  49                 vram += src->vram_usage;
  50                 gtt += src->gart_usage;
  51         }
  52
  53         /* Flush the GFX IB if DMA depends on it. */
  54         if (radeon_emitted(ctx->gfx_cs, ctx->initial_gfx_cs_size) &&
  55             ((dst &&
  56               ctx->ws->cs_is_buffer_referenced(ctx->gfx_cs, dst->buf,
  57                                                  RADEON_USAGE_READWRITE)) ||
  58              (src &&
  59               ctx->ws->cs_is_buffer_referenced(ctx->gfx_cs, src->buf,
  60                                                  RADEON_USAGE_WRITE))))
  61                 si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  62
  63         /* Flush if there's not enough space, or if the memory usage per IB
  64          * is too large.
  65          *
  66          * IBs using too little memory are limited by the IB submission overhead.
  67          * IBs using too much memory are limited by the kernel/TTM overhead.
  68          * Too long IBs create CPU-GPU pipeline bubbles and add latency.
  69          *
  70          * This heuristic makes sure that DMA requests are executed
  71          * very soon after the call is made and lowers memory usage.
  72          * It improves texture upload performance by keeping the DMA
  73          * engine busy while uploads are being submitted.
  74          */
  75         num_dw++; /* for emit_wait_idle below */
  76         if (!ctx->ws->cs_check_space(ctx->dma_cs, num_dw) ||
  77             ctx->dma_cs->used_vram + ctx->dma_cs->used_gart > 64 * 1024 * 1024 ||
  78             !radeon_cs_memory_below_limit(ctx->screen, ctx->dma_cs, vram, gtt)) {
  79                 si_flush_dma_cs(ctx, PIPE_FLUSH_ASYNC, NULL);
  80                 assert((num_dw + ctx->dma_cs->current.cdw) <= ctx->dma_cs->current.max_dw);
  81         }
  82
  83         /* Wait for idle if either buffer has been used in the IB before to
  84          * prevent read-after-write hazards.
  85          */
  86         if ((dst &&
  87              ctx->ws->cs_is_buffer_referenced(ctx->dma_cs, dst->buf,
  88                                                 RADEON_USAGE_READWRITE)) ||
  89             (src &&
  90              ctx->ws->cs_is_buffer_referenced(ctx->dma_cs, src->buf,
  91                                                 RADEON_USAGE_WRITE)))
  92                 si_dma_emit_wait_idle(ctx);
  93
  94         if (dst) {
  95                 radeon_add_to_buffer_list(ctx, ctx->dma_cs, dst,
  96                                           RADEON_USAGE_WRITE,
  97                                           RADEON_PRIO_SDMA_BUFFER);
  98         }
  99         if (src) {
 100                 radeon_add_to_buffer_list(ctx, ctx->dma_cs, src,
 101                                           RADEON_USAGE_READ,
 102                                           RADEON_PRIO_SDMA_BUFFER);
 103         }
 104
 105         /* this function is called before all DMA calls, so increment this. */
 106         ctx->num_dma_calls++;
 107 }
 108
 109 void si_flush_dma_cs(struct si_context *ctx, unsigned flags,
 110                      struct pipe_fence_handle **fence)
 111 {
 112         struct radeon_winsys_cs *cs = ctx->dma_cs;
 113         struct radeon_saved_cs saved;
 114         bool check_vm = (ctx->screen->debug_flags & DBG(CHECK_VM)) != 0;
 115
 116         if (!radeon_emitted(cs, 0)) {
 117                 if (fence)
 118                         ctx->ws->fence_reference(fence, ctx->last_sdma_fence);
 119                 return;
 120         }
 121
 122         if (check_vm)
 123                 si_save_cs(ctx->ws, cs, &saved, true);
 124
 125         ctx->ws->cs_flush(cs, flags, &ctx->last_sdma_fence);
 126         if (fence)
 127                 ctx->ws->fence_reference(fence, ctx->last_sdma_fence);
 128
 129         if (check_vm) {
 130                 /* Use conservative timeout 800ms, after which we won't wait any
 131                  * longer and assume the GPU is hung.
 132                  */
 133                 ctx->ws->fence_wait(ctx->ws, ctx->last_sdma_fence, 800*1000*1000);
 134
 135                 si_check_vm_faults(ctx, &saved, RING_DMA);
 136                 si_clear_saved_cs(&saved);
 137         }
 138 }
 139
 140 void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst,
 141                             uint64_t offset, uint64_t size, unsigned value)
 142 {
 143         struct si_context *ctx = (struct si_context*)sscreen->aux_context;
 144
 145         mtx_lock(&sscreen->aux_context_lock);
 146         ctx->dma_clear_buffer(ctx, dst, offset, size, value);
 147         sscreen->aux_context->flush(sscreen->aux_context, NULL, 0);
 148         mtx_unlock(&sscreen->aux_context_lock);
 149 }