#include "sid.h"
#include "radeon/r600_cs.h"
-/* The max number of bytes to copy per packet. */
-#define CP_DMA_MAX_BYTE_COUNT ((1 << 21) - SI_CPDMA_ALIGNMENT)
-
/* Set this if you want the ME to wait until CP DMA is done.
* It should be set on the last CP DMA packet. */
#define CP_DMA_SYNC (1 << 0)
#define CP_DMA_USE_L2 (1 << 2) /* CIK+ */
#define CP_DMA_CLEAR (1 << 3)
+/* The max number of bytes that can be copied per packet. */
+static inline unsigned cp_dma_max_byte_count(struct si_context *sctx)
+{
+ unsigned max = sctx->b.chip_class >= GFX9 ?
+ S_414_BYTE_COUNT_GFX9(~0u) :
+ S_414_BYTE_COUNT_GFX6(~0u);
+
+ /* make it aligned for optimal performance */
+ return max & ~(SI_CPDMA_ALIGNMENT - 1);
+}
+
+
/* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
* a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
* clear value.
enum r600_coherency coher)
{
struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
- uint32_t header = 0, command = S_414_BYTE_COUNT(size);
+ uint32_t header = 0, command = 0;
assert(size);
- assert(size <= CP_DMA_MAX_BYTE_COUNT);
+ assert(size <= cp_dma_max_byte_count(sctx));
+
+ if (sctx->b.chip_class >= GFX9)
+ command |= S_414_BYTE_COUNT_GFX9(size);
+ else
+ command |= S_414_BYTE_COUNT_GFX6(size);
/* Sync flags. */
if (flags & CP_DMA_SYNC)
header |= S_411_CP_SYNC(1);
- else
- command |= S_414_DISABLE_WR_CONFIRM(1);
+ else {
+ if (sctx->b.chip_class >= GFX9)
+ command |= S_414_DISABLE_WR_CONFIRM_GFX9(1);
+ else
+ command |= S_414_DISABLE_WR_CONFIRM_GFX6(1);
+ }
if (flags & CP_DMA_RAW_WAIT)
command |= S_414_RAW_WAIT(1);
/* Src and dst flags. */
- if (flags & CP_DMA_USE_L2)
+ if (sctx->b.chip_class >= GFX9 && !(flags & CP_DMA_CLEAR) &&
+ src_va == dst_va)
+ header |= S_411_DSL_SEL(V_411_NOWHERE); /* prefetch only */
+ else if (flags & CP_DMA_USE_L2)
header |= S_411_DSL_SEL(V_411_DST_ADDR_TC_L2);
if (flags & CP_DMA_CLEAR)
struct r600_resource *rdst = r600_resource(dst);
unsigned tc_l2_flag = get_tc_l2_flag(sctx, coher);
unsigned flush_flags = get_flush_flags(sctx, coher);
+ uint64_t dma_clear_size;
bool is_first = true;
if (!size)
return;
+ dma_clear_size = size & ~3llu;
+
/* Mark the buffer range of destination as valid (initialized),
* so that transfer_map knows it should wait for the GPU when mapping
* that range. */
util_range_add(&rdst->valid_buffer_range, offset,
- offset + size);
-
- /* Fallback for unaligned clears. */
- if (offset % 4 != 0 || size % 4 != 0) {
- uint8_t *map = r600_buffer_map_sync_with_rings(&sctx->b, rdst,
- PIPE_TRANSFER_WRITE);
- map += offset;
- for (uint64_t i = 0; i < size; i++) {
- unsigned byte_within_dword = (offset + i) % 4;
- *map++ = (value >> (byte_within_dword * 8)) & 0xff;
- }
- return;
- }
+ offset + dma_clear_size);
/* dma_clear_buffer can use clear_buffer on failure. Make sure that
* doesn't happen. We don't want an infinite recursion: */
if (sctx->b.dma.cs &&
+ !(dst->flags & PIPE_RESOURCE_FLAG_SPARSE) &&
+ (offset % 4 == 0) &&
/* CP DMA is very slow. Always use SDMA for big clears. This
* alone improves DeusEx:MD performance by 70%. */
(size > 128 * 1024 ||
* of them are moved to SDMA thanks to this. */
!ws->cs_is_buffer_referenced(sctx->b.gfx.cs, rdst->buf,
RADEON_USAGE_READWRITE))) {
- sctx->b.dma_clear_buffer(ctx, dst, offset, size, value);
- return;
- }
+ sctx->b.dma_clear_buffer(ctx, dst, offset, dma_clear_size, value);
- uint64_t va = rdst->gpu_address + offset;
+ offset += dma_clear_size;
+ size -= dma_clear_size;
+ } else if (dma_clear_size >= 4) {
+ uint64_t va = rdst->gpu_address + offset;
- /* Flush the caches. */
- sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
- SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
+ offset += dma_clear_size;
+ size -= dma_clear_size;
- while (size) {
- unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
- unsigned dma_flags = tc_l2_flag | CP_DMA_CLEAR;
+ /* Flush the caches. */
+ sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
+ SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
- si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, 0,
- &is_first, &dma_flags);
+ while (dma_clear_size) {
+ unsigned byte_count = MIN2(dma_clear_size, cp_dma_max_byte_count(sctx));
+ unsigned dma_flags = tc_l2_flag | CP_DMA_CLEAR;
- /* Emit the clear packet. */
- si_emit_cp_dma(sctx, va, value, byte_count, dma_flags, coher);
+ si_cp_dma_prepare(sctx, dst, NULL, byte_count, dma_clear_size, 0,
+ &is_first, &dma_flags);
- size -= byte_count;
- va += byte_count;
+ /* Emit the clear packet. */
+ si_emit_cp_dma(sctx, va, value, byte_count, dma_flags, coher);
+
+ dma_clear_size -= byte_count;
+ va += byte_count;
+ }
+
+ if (tc_l2_flag)
+ rdst->TC_L2_dirty = true;
+
+ /* If it's not a framebuffer fast clear... */
+ if (coher == R600_COHERENCY_SHADER)
+ sctx->b.num_cp_dma_calls++;
}
- if (tc_l2_flag)
- rdst->TC_L2_dirty = true;
+ if (size) {
+ /* Handle non-dword alignment.
+ *
+ * This function is called for embedded texture metadata clears,
+ * but those should always be properly aligned. */
+ assert(dst->target == PIPE_BUFFER);
+ assert(size < 4);
- /* If it's not a framebuffer fast clear... */
- if (coher == R600_COHERENCY_SHADER)
- sctx->b.num_cp_dma_calls++;
+ pipe_buffer_write(ctx, dst, offset, size, &value);
+ }
}
/**
sctx->scratch_buffer->b.b.width0 < scratch_size) {
r600_resource_reference(&sctx->scratch_buffer, NULL);
sctx->scratch_buffer = (struct r600_resource*)
- pipe_buffer_create(&sctx->screen->b.b, 0,
- PIPE_USAGE_DEFAULT, scratch_size);
+ r600_aligned_buffer_create(&sctx->screen->b.b,
+ R600_RESOURCE_FLAG_UNMAPPABLE,
+ PIPE_USAGE_DEFAULT,
+ scratch_size, 256);
if (!sctx->scratch_buffer)
return;
while (size) {
unsigned dma_flags = tc_l2_flag;
- unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
+ unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
si_cp_dma_prepare(sctx, dst, src, byte_count,
size + skipped_size + realign_size,
if (sctx->vertex_buffer_pointer_dirty) {
cik_prefetch_TC_L2_async(sctx, &sctx->vertex_buffers.buffer->b.b,
sctx->vertex_buffers.buffer_offset,
- sctx->vertex_elements->count * 16);
+ sctx->vertex_elements->desc_list_byte_size);
}
if (si_pm4_state_changed(sctx, ps))
cik_prefetch_shader_async(sctx, sctx->queued.named.ps);
projects / mesa.git / blobdiff