gallium/radeon: fix (S)DMA read-after-write hazards

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Reviewed-by: Nicolai Hähnle <nicolai.haehnle@amd.com>

diff --git a/src/gallium/drivers/r600/evergreen_hw_context.c b/src/gallium/drivers/r600/evergreen_hw_context.c
index a0f468004037c27d9a262335879e3120905d8d60..c2dba8c19964f88aeaae348c98ef3c3f12187c56 100644 (file)
--- a/src/gallium/drivers/r600/evergreen_hw_context.c
+++ b/src/gallium/drivers/r600/evergreen_hw_context.c
@@ -77,6 +77,7 @@ void evergreen_dma_copy_buffer(struct r600_context *rctx,
                src_offset += csize << shift;
                size -= csize;
        }
+       r600_dma_emit_wait_idle(&rctx->b);
 }
 
 /* The max number of bytes to copy per packet. */

diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c
index 672ddd4fcbe9c3f71ac47f9af0fe1ca72810a195..acf60c6d8d1128159574d885fc707b73f051869a 100644 (file)
--- a/src/gallium/drivers/r600/evergreen_state.c
+++ b/src/gallium/drivers/r600/evergreen_state.c
@@ -3470,6 +3470,7 @@ static void evergreen_dma_copy_tile(struct r600_context *rctx,
                addr += cheight * pitch;
                y += cheight;
        }
+       r600_dma_emit_wait_idle(&rctx->b);
 }
 
 static void evergreen_dma_copy(struct pipe_context *ctx,

diff --git a/src/gallium/drivers/r600/r600_hw_context.c b/src/gallium/drivers/r600/r600_hw_context.c
index 2bc6d3ffce43d5de509fc77201e404d5e8e19f72..fa1028be7afffed044ddb57b3499f01574d261a4 100644 (file)
--- a/src/gallium/drivers/r600/r600_hw_context.c
+++ b/src/gallium/drivers/r600/r600_hw_context.c
@@ -484,4 +484,5 @@ void r600_dma_copy_buffer(struct r600_context *rctx,
                src_offset += csize << 2;
                size -= csize;
        }
+       r600_dma_emit_wait_idle(&rctx->b);
 }

diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c
index 48e3663ebf7c50e37d618499a4b5359d5aa929f2..715c6f11145b2ffa0b05d400ec562a19aa2b80a1 100644 (file)
--- a/src/gallium/drivers/r600/r600_state.c
+++ b/src/gallium/drivers/r600/r600_state.c
@@ -2941,6 +2941,7 @@ static boolean r600_dma_copy_tile(struct r600_context *rctx,
                addr += cheight * pitch;
                y += cheight;
        }
+       r600_dma_emit_wait_idle(&rctx->b);
        return TRUE;
 }
 

diff --git a/src/gallium/drivers/radeon/r600_pipe_common.c b/src/gallium/drivers/radeon/r600_pipe_common.c
index cea6b7bce8252ae253c45bd253933421af78566f..3e15b1437003499f83b0241ce914291fc9467cb8 100644 (file)
--- a/src/gallium/drivers/radeon/r600_pipe_common.c
+++ b/src/gallium/drivers/radeon/r600_pipe_common.c
@@ -149,6 +149,30 @@ void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw)
        }
 }
 
+/* This is required to prevent read-after-write hazards. */
+void r600_dma_emit_wait_idle(struct r600_common_context *rctx)
+{
+       struct radeon_winsys_cs *cs = rctx->dma.cs;
+
+       /* done at the end of DMA calls, so increment this. */
+       rctx->num_dma_calls++;
+
+       r600_need_dma_space(rctx, 1);
+
+       if (cs->cdw == 0) /* empty queue */
+               return;
+
+       /* NOP waits for idle on Evergreen and later. */
+       if (rctx->chip_class >= CIK)
+               radeon_emit(cs, 0x00000000); /* NOP */
+       else if (rctx->chip_class >= EVERGREEN)
+               radeon_emit(cs, 0xf0000000); /* NOP */
+       else {
+               /* TODO: R600-R700 should use the FENCE packet.
+                * CS checker support is required. */
+       }
+}
+
 static void r600_memory_barrier(struct pipe_context *ctx, unsigned flags)
 {
 }

diff --git a/src/gallium/drivers/radeon/r600_pipe_common.h b/src/gallium/drivers/radeon/r600_pipe_common.h
index 47b73653e9335432f1ef5689aa9948e01f948fa9..03449536b5b3bd5ecd204c6d12d99ccf7d772d1a 100644 (file)
--- a/src/gallium/drivers/radeon/r600_pipe_common.h
+++ b/src/gallium/drivers/radeon/r600_pipe_common.h
@@ -598,6 +598,7 @@ struct pipe_resource *r600_resource_create_common(struct pipe_screen *screen,
                                                  const struct pipe_resource *templ);
 const char *r600_get_llvm_processor_name(enum radeon_family family);
 void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw);
+void r600_dma_emit_wait_idle(struct r600_common_context *rctx);
 
 /* r600_gpu_load.c */
 void r600_gpu_load_kill_thread(struct r600_common_screen *rscreen);

diff --git a/src/gallium/drivers/radeonsi/cik_sdma.c b/src/gallium/drivers/radeonsi/cik_sdma.c
index 88a994ee390a981b9f12fa4cf40a5b22759361b7..5021578871b3924492d3e39b8e9c3f98d6c149df 100644 (file)
--- a/src/gallium/drivers/radeonsi/cik_sdma.c
+++ b/src/gallium/drivers/radeonsi/cik_sdma.c
@@ -87,6 +87,7 @@ static void cik_sdma_copy_buffer(struct si_context *ctx,
                       dst_offset + size);
 
        cik_sdma_do_copy_buffer(ctx, dst, src, dst_offset, src_offset, size);
+       r600_dma_emit_wait_idle(&ctx->b);
 }
 
 static void cik_sdma_copy(struct pipe_context *ctx,

diff --git a/src/gallium/drivers/radeonsi/si_dma.c b/src/gallium/drivers/radeonsi/si_dma.c
index 033eb7b147934824296d84b42eb31a6445fd4f9f..03321f583a5fb05a99ba4d7bb537b5d6353ae269 100644 (file)
--- a/src/gallium/drivers/radeonsi/si_dma.c
+++ b/src/gallium/drivers/radeonsi/si_dma.c
@@ -82,6 +82,7 @@ static void si_dma_copy_buffer(struct si_context *ctx,
                src_offset += csize << shift;
                size -= csize;
        }
+       r600_dma_emit_wait_idle(&ctx->b);
 }
 
 static void si_dma_copy_tile(struct si_context *ctx,
@@ -188,6 +189,7 @@ static void si_dma_copy_tile(struct si_context *ctx,
                addr += cheight * pitch;
                tiled_y += cheight;
        }
+       r600_dma_emit_wait_idle(&ctx->b);
 }
 
 static void si_dma_copy(struct pipe_context *ctx,