radeonsi: add CP DMA flags for greater control over synchronization

for L2 prefetch

Reviewed-by: Edward O'Callaghan <funfunctor@folklore1984.net>
Reviewed-by: Nicolai Hähnle <nicolai.haehnle@amd.com>

diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c
index e15fe4059290f0784609c30bb205bd84b8dea897..5b533732a4db95b7e60290a9b152027510322ebd 100644 (file)
--- a/src/gallium/drivers/radeonsi/si_blit.c
+++ b/src/gallium/drivers/radeonsi/si_blit.c
@@ -848,7 +848,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 
        /* Handle buffers first. */
        if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
-               si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
+               si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, 0);
                return;
        }
 

diff --git a/src/gallium/drivers/radeonsi/si_cp_dma.c b/src/gallium/drivers/radeonsi/si_cp_dma.c
index 1ca9a053d499f85ad78121c276cc2e88c59f7342..934fd8df2dbeaa354575142462afb465fc4825a5 100644 (file)
--- a/src/gallium/drivers/radeonsi/si_cp_dma.c
+++ b/src/gallium/drivers/radeonsi/si_cp_dma.c
@@ -130,7 +130,7 @@ static unsigned get_tc_l2_flag(struct si_context *sctx, enum r600_coherency cohe
 
 static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
                              struct pipe_resource *src, unsigned byte_count,
-                             uint64_t remaining_size,
+                             uint64_t remaining_size, unsigned user_flags,
                              bool *is_first, unsigned *packet_flags)
 {
        /* Count memory usage in so that need_cs_space can take it into account. */
@@ -138,7 +138,8 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
        if (src)
                r600_context_add_resource_size(&sctx->b.b, src);
 
-       si_need_cs_space(sctx);
+       if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
+               si_need_cs_space(sctx);
 
        /* This must be done after need_cs_space. */
        radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
@@ -152,10 +153,10 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
        /* Flush the caches for the first copy only.
         * Also wait for the previous CP DMA operations.
         */
-       if (sctx->b.flags)
+       if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->b.flags)
                si_emit_cache_flush(sctx);
 
-       if (*is_first)
+       if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first)
                *packet_flags |= CP_DMA_RAW_WAIT;
 
        *is_first = false;
@@ -163,7 +164,8 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
        /* Do the synchronization after the last dma, so that all data
         * is written to memory.
         */
-       if (byte_count == remaining_size)
+       if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) &&
+           byte_count == remaining_size)
                *packet_flags |= CP_DMA_SYNC;
 }
 
@@ -227,7 +229,7 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
                unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
                unsigned dma_flags = tc_l2_flag  | CP_DMA_CLEAR;
 
-               si_cp_dma_prepare(sctx, dst, NULL, byte_count, size,
+               si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, 0,
                                  &is_first, &dma_flags);
 
                /* Emit the clear packet. */
@@ -252,7 +254,7 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
  * \param size  Remaining size to the CP DMA alignment.
  */
 static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
-                                    bool *is_first)
+                                    unsigned user_flags, bool *is_first)
 {
        uint64_t va;
        unsigned dma_flags = 0;
@@ -275,7 +277,7 @@ static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
        }
 
        si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b,
-                         &sctx->scratch_buffer->b.b, size, size,
+                         &sctx->scratch_buffer->b.b, size, size, user_flags,
                          is_first, &dma_flags);
 
        va = sctx->scratch_buffer->gpu_address;
@@ -283,9 +285,15 @@ static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
                       R600_COHERENCY_SHADER);
 }
 
+/**
+ * Do memcpy between buffers using CP DMA.
+ *
+ * \param user_flags   bitmask of SI_CPDMA_*
+ */
 void si_copy_buffer(struct si_context *sctx,
                    struct pipe_resource *dst, struct pipe_resource *src,
-                   uint64_t dst_offset, uint64_t src_offset, unsigned size)
+                   uint64_t dst_offset, uint64_t src_offset, unsigned size,
+                   unsigned user_flags)
 {
        uint64_t main_dst_offset, main_src_offset;
        unsigned skipped_size = 0;
@@ -329,8 +337,9 @@ void si_copy_buffer(struct si_context *sctx,
        }
 
        /* Flush the caches. */
-       sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
-                        SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
+       if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC))
+               sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
+                                SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
 
        /* This is the main part doing the copying. Src is always aligned. */
        main_dst_offset = dst_offset + skipped_size;
@@ -342,7 +351,7 @@ void si_copy_buffer(struct si_context *sctx,
 
                si_cp_dma_prepare(sctx, dst, src, byte_count,
                                  size + skipped_size + realign_size,
-                                 &is_first, &dma_flags);
+                                 user_flags, &is_first, &dma_flags);
 
                si_emit_cp_dma(sctx, main_dst_offset, main_src_offset,
                               byte_count, dma_flags, R600_COHERENCY_SHADER);
@@ -357,7 +366,7 @@ void si_copy_buffer(struct si_context *sctx,
                unsigned dma_flags = tc_l2_flag;
 
                si_cp_dma_prepare(sctx, dst, src, skipped_size,
-                                 skipped_size + realign_size,
+                                 skipped_size + realign_size, user_flags,
                                  &is_first, &dma_flags);
 
                si_emit_cp_dma(sctx, dst_offset, src_offset, skipped_size,
@@ -366,7 +375,7 @@ void si_copy_buffer(struct si_context *sctx,
 
        /* Finally, realign the engine if the size wasn't aligned. */
        if (realign_size)
-               si_cp_dma_realign_engine(sctx, realign_size,
+               si_cp_dma_realign_engine(sctx, realign_size, user_flags,
                                         &is_first);
 
        if (tc_l2_flag)

diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index 9f79c2aa8046e2870c65ce34ce64394a38b962d7..dc37c8d28f3359e21f108ab396fde0416c3ca13c 100644 (file)
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -372,9 +372,15 @@ void si_resource_copy_region(struct pipe_context *ctx,
                             const struct pipe_box *src_box);
 
 /* si_cp_dma.c */
+#define SI_CPDMA_SKIP_CHECK_CS_SPACE   (1 << 0) /* don't call need_cs_space */
+#define SI_CPDMA_SKIP_SYNC_AFTER       (1 << 1) /* don't wait for DMA after the copy */
+#define SI_CPDMA_SKIP_SYNC_BEFORE      (1 << 2) /* don't wait for DMA before the copy (RAW hazards) */
+#define SI_CPDMA_SKIP_GFX_SYNC         (1 << 3) /* don't flush caches and don't wait for PS/CS */
+
 void si_copy_buffer(struct si_context *sctx,
                    struct pipe_resource *dst, struct pipe_resource *src,
-                   uint64_t dst_offset, uint64_t src_offset, unsigned size);
+                   uint64_t dst_offset, uint64_t src_offset, unsigned size,
+                   unsigned user_flags);
 void si_init_cp_dma_functions(struct si_context *sctx);
 
 /* si_debug.c */