* Authors:
* Jerome Glisse
*/
-#include "r600_hw_context_priv.h"
+#include "r600_pipe.h"
#include "evergreend.h"
#include "util/u_memory.h"
#include "util/u_math.h"
-static const struct r600_reg evergreen_context_reg_list[] = {
- {R_028644_SPI_PS_INPUT_CNTL_0, 0, 0},
- {R_028648_SPI_PS_INPUT_CNTL_1, 0, 0},
- {R_02864C_SPI_PS_INPUT_CNTL_2, 0, 0},
- {R_028650_SPI_PS_INPUT_CNTL_3, 0, 0},
- {R_028654_SPI_PS_INPUT_CNTL_4, 0, 0},
- {R_028658_SPI_PS_INPUT_CNTL_5, 0, 0},
- {R_02865C_SPI_PS_INPUT_CNTL_6, 0, 0},
- {R_028660_SPI_PS_INPUT_CNTL_7, 0, 0},
- {R_028664_SPI_PS_INPUT_CNTL_8, 0, 0},
- {R_028668_SPI_PS_INPUT_CNTL_9, 0, 0},
- {R_02866C_SPI_PS_INPUT_CNTL_10, 0, 0},
- {R_028670_SPI_PS_INPUT_CNTL_11, 0, 0},
- {R_028674_SPI_PS_INPUT_CNTL_12, 0, 0},
- {R_028678_SPI_PS_INPUT_CNTL_13, 0, 0},
- {R_02867C_SPI_PS_INPUT_CNTL_14, 0, 0},
- {R_028680_SPI_PS_INPUT_CNTL_15, 0, 0},
- {R_028684_SPI_PS_INPUT_CNTL_16, 0, 0},
- {R_028688_SPI_PS_INPUT_CNTL_17, 0, 0},
- {R_02868C_SPI_PS_INPUT_CNTL_18, 0, 0},
- {R_028690_SPI_PS_INPUT_CNTL_19, 0, 0},
- {R_028694_SPI_PS_INPUT_CNTL_20, 0, 0},
- {R_028698_SPI_PS_INPUT_CNTL_21, 0, 0},
- {R_02869C_SPI_PS_INPUT_CNTL_22, 0, 0},
- {R_0286A0_SPI_PS_INPUT_CNTL_23, 0, 0},
- {R_0286A4_SPI_PS_INPUT_CNTL_24, 0, 0},
- {R_0286A8_SPI_PS_INPUT_CNTL_25, 0, 0},
- {R_0286AC_SPI_PS_INPUT_CNTL_26, 0, 0},
- {R_0286B0_SPI_PS_INPUT_CNTL_27, 0, 0},
- {R_0286B4_SPI_PS_INPUT_CNTL_28, 0, 0},
- {R_0286B8_SPI_PS_INPUT_CNTL_29, 0, 0},
- {R_0286BC_SPI_PS_INPUT_CNTL_30, 0, 0},
- {R_0286C0_SPI_PS_INPUT_CNTL_31, 0, 0},
- {GROUP_FORCE_NEW_BLOCK, 0, 0},
- {R_0286CC_SPI_PS_IN_CONTROL_0, 0, 0},
- {R_0286D0_SPI_PS_IN_CONTROL_1, 0, 0},
- {R_0286D8_SPI_INPUT_Z, 0, 0},
- {R_0286E0_SPI_BARYC_CNTL, 0, 0},
- {R_0286E4_SPI_PS_IN_CONTROL_2, 0, 0},
- {R_028840_SQ_PGM_START_PS, REG_FLAG_NEED_BO, 0},
- {R_028844_SQ_PGM_RESOURCES_PS, 0, 0},
- {R_02884C_SQ_PGM_EXPORTS_PS, 0, 0},
-};
-
-static const struct r600_reg cayman_context_reg_list[] = {
- {R_028644_SPI_PS_INPUT_CNTL_0, 0, 0},
- {R_028648_SPI_PS_INPUT_CNTL_1, 0, 0},
- {R_02864C_SPI_PS_INPUT_CNTL_2, 0, 0},
- {R_028650_SPI_PS_INPUT_CNTL_3, 0, 0},
- {R_028654_SPI_PS_INPUT_CNTL_4, 0, 0},
- {R_028658_SPI_PS_INPUT_CNTL_5, 0, 0},
- {R_02865C_SPI_PS_INPUT_CNTL_6, 0, 0},
- {R_028660_SPI_PS_INPUT_CNTL_7, 0, 0},
- {R_028664_SPI_PS_INPUT_CNTL_8, 0, 0},
- {R_028668_SPI_PS_INPUT_CNTL_9, 0, 0},
- {R_02866C_SPI_PS_INPUT_CNTL_10, 0, 0},
- {R_028670_SPI_PS_INPUT_CNTL_11, 0, 0},
- {R_028674_SPI_PS_INPUT_CNTL_12, 0, 0},
- {R_028678_SPI_PS_INPUT_CNTL_13, 0, 0},
- {R_02867C_SPI_PS_INPUT_CNTL_14, 0, 0},
- {R_028680_SPI_PS_INPUT_CNTL_15, 0, 0},
- {R_028684_SPI_PS_INPUT_CNTL_16, 0, 0},
- {R_028688_SPI_PS_INPUT_CNTL_17, 0, 0},
- {R_02868C_SPI_PS_INPUT_CNTL_18, 0, 0},
- {R_028690_SPI_PS_INPUT_CNTL_19, 0, 0},
- {R_028694_SPI_PS_INPUT_CNTL_20, 0, 0},
- {R_028698_SPI_PS_INPUT_CNTL_21, 0, 0},
- {R_02869C_SPI_PS_INPUT_CNTL_22, 0, 0},
- {R_0286A0_SPI_PS_INPUT_CNTL_23, 0, 0},
- {R_0286A4_SPI_PS_INPUT_CNTL_24, 0, 0},
- {R_0286A8_SPI_PS_INPUT_CNTL_25, 0, 0},
- {R_0286AC_SPI_PS_INPUT_CNTL_26, 0, 0},
- {R_0286B0_SPI_PS_INPUT_CNTL_27, 0, 0},
- {R_0286B4_SPI_PS_INPUT_CNTL_28, 0, 0},
- {R_0286B8_SPI_PS_INPUT_CNTL_29, 0, 0},
- {R_0286BC_SPI_PS_INPUT_CNTL_30, 0, 0},
- {R_0286C0_SPI_PS_INPUT_CNTL_31, 0, 0},
- {R_0286CC_SPI_PS_IN_CONTROL_0, 0, 0},
- {R_0286D0_SPI_PS_IN_CONTROL_1, 0, 0},
- {R_0286D8_SPI_INPUT_Z, 0, 0},
- {R_0286E0_SPI_BARYC_CNTL, 0, 0},
- {R_0286E4_SPI_PS_IN_CONTROL_2, 0, 0},
- {R_028840_SQ_PGM_START_PS, REG_FLAG_NEED_BO, 0},
- {R_028844_SQ_PGM_RESOURCES_PS, 0, 0},
- {R_02884C_SQ_PGM_EXPORTS_PS, 0, 0},
-};
-
-int evergreen_context_init(struct r600_context *ctx)
+void evergreen_dma_copy_buffer(struct r600_context *rctx,
+ struct pipe_resource *dst,
+ struct pipe_resource *src,
+ uint64_t dst_offset,
+ uint64_t src_offset,
+ uint64_t size)
{
- int r = 0;
-
- /* add blocks */
- if (ctx->family >= CHIP_CAYMAN)
- r = r600_context_add_block(ctx, cayman_context_reg_list,
- Elements(cayman_context_reg_list), PKT3_SET_CONTEXT_REG, EVERGREEN_CONTEXT_REG_OFFSET);
- else
- r = r600_context_add_block(ctx, evergreen_context_reg_list,
- Elements(evergreen_context_reg_list), PKT3_SET_CONTEXT_REG, EVERGREEN_CONTEXT_REG_OFFSET);
- if (r)
- goto out_err;
-
- r = r600_setup_block_table(ctx);
- if (r)
- goto out_err;
-
- ctx->max_db = 8;
- return 0;
-out_err:
- r600_context_fini(ctx);
- return r;
-}
-
-void evergreen_flush_vgt_streamout(struct r600_context *ctx)
-{
- struct radeon_winsys_cs *cs = ctx->rings.gfx.cs;
-
- r600_write_config_reg(cs, R_0084FC_CP_STRMOUT_CNTL, 0);
-
- cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
- cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_SO_VGTSTREAMOUT_FLUSH) | EVENT_INDEX(0);
-
- cs->buf[cs->cdw++] = PKT3(PKT3_WAIT_REG_MEM, 5, 0);
- cs->buf[cs->cdw++] = WAIT_REG_MEM_EQUAL; /* wait until the register is equal to the reference value */
- cs->buf[cs->cdw++] = R_0084FC_CP_STRMOUT_CNTL >> 2; /* register */
- cs->buf[cs->cdw++] = 0;
- cs->buf[cs->cdw++] = S_0084FC_OFFSET_UPDATE_DONE(1); /* reference value */
- cs->buf[cs->cdw++] = S_0084FC_OFFSET_UPDATE_DONE(1); /* mask */
- cs->buf[cs->cdw++] = 4; /* poll interval */
-}
-
-void evergreen_set_streamout_enable(struct r600_context *ctx, unsigned buffer_enable_bit)
-{
- struct radeon_winsys_cs *cs = ctx->rings.gfx.cs;
-
- if (buffer_enable_bit) {
- r600_write_context_reg_seq(cs, R_028B94_VGT_STRMOUT_CONFIG, 2);
- r600_write_value(cs, S_028B94_STREAMOUT_0_EN(1)); /* R_028B94_VGT_STRMOUT_CONFIG */
- r600_write_value(cs, S_028B98_STREAM_0_BUFFER_EN(buffer_enable_bit)); /* R_028B98_VGT_STRMOUT_BUFFER_CONFIG */
- } else {
- r600_write_context_reg(cs, R_028B94_VGT_STRMOUT_CONFIG, S_028B94_STREAMOUT_0_EN(0));
- }
-}
-
-void evergreen_dma_copy(struct r600_context *rctx,
- struct pipe_resource *dst,
- struct pipe_resource *src,
- uint64_t dst_offset,
- uint64_t src_offset,
- uint64_t size)
-{
- struct radeon_winsys_cs *cs = rctx->rings.dma.cs;
+ struct radeon_cmdbuf *cs = rctx->b.dma.cs;
unsigned i, ncopy, csize, sub_cmd, shift;
struct r600_resource *rdst = (struct r600_resource*)dst;
struct r600_resource *rsrc = (struct r600_resource*)src;
- /* make sure that the dma ring is only one active */
- rctx->rings.gfx.flush(rctx, RADEON_FLUSH_ASYNC);
- dst_offset += r600_resource_va(&rctx->screen->screen, dst);
- src_offset += r600_resource_va(&rctx->screen->screen, src);
+ /* Mark the buffer range of destination as valid (initialized),
+ * so that transfer_map knows it should wait for the GPU when mapping
+ * that range. */
+ util_range_add(&rdst->b.b, &rdst->valid_buffer_range, dst_offset,
+ dst_offset + size);
+
+ dst_offset += rdst->gpu_address;
+ src_offset += rsrc->gpu_address;
/* see if we use dword or byte copy */
- if (!(dst_offset & 0x3) && !(src_offset & 0x3) && !(size & 0x3)) {
+ if (!(dst_offset % 4) && !(src_offset % 4) && !(size % 4)) {
size >>= 2;
- sub_cmd = 0x00;
+ sub_cmd = EG_DMA_COPY_DWORD_ALIGNED;
shift = 2;
} else {
- sub_cmd = 0x40;
+ sub_cmd = EG_DMA_COPY_BYTE_ALIGNED;
shift = 0;
}
- ncopy = (size / 0x000fffff) + !!(size % 0x000fffff);
+ ncopy = (size / EG_DMA_COPY_MAX_SIZE) + !!(size % EG_DMA_COPY_MAX_SIZE);
- r600_need_dma_space(rctx, ncopy * 5);
+ r600_need_dma_space(&rctx->b, ncopy * 5, rdst, rsrc);
for (i = 0; i < ncopy; i++) {
- csize = size < 0x000fffff ? size : 0x000fffff;
- /* emit reloc before writting cs so that cs is always in consistent state */
- r600_context_bo_reloc(rctx, &rctx->rings.dma, rsrc, RADEON_USAGE_READ);
- r600_context_bo_reloc(rctx, &rctx->rings.dma, rdst, RADEON_USAGE_WRITE);
- cs->buf[cs->cdw++] = DMA_PACKET(DMA_PACKET_COPY, sub_cmd, csize);
- cs->buf[cs->cdw++] = dst_offset & 0xffffffff;
- cs->buf[cs->cdw++] = src_offset & 0xffffffff;
- cs->buf[cs->cdw++] = (dst_offset >> 32UL) & 0xff;
- cs->buf[cs->cdw++] = (src_offset >> 32UL) & 0xff;
+ csize = size < EG_DMA_COPY_MAX_SIZE ? size : EG_DMA_COPY_MAX_SIZE;
+ /* emit reloc before writing cs so that cs is always in consistent state */
+ radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rsrc, RADEON_USAGE_READ, 0);
+ radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rdst, RADEON_USAGE_WRITE, 0);
+ radeon_emit(cs, DMA_PACKET(DMA_PACKET_COPY, sub_cmd, csize));
+ radeon_emit(cs, dst_offset & 0xffffffff);
+ radeon_emit(cs, src_offset & 0xffffffff);
+ radeon_emit(cs, (dst_offset >> 32UL) & 0xff);
+ radeon_emit(cs, (src_offset >> 32UL) & 0xff);
dst_offset += csize << shift;
src_offset += csize << shift;
size -= csize;
}
+}
- util_range_add(&rdst->valid_buffer_range, dst_offset,
- dst_offset + size);
+/* The max number of bytes to copy per packet. */
+#define CP_DMA_MAX_BYTE_COUNT ((1 << 21) - 8)
+
+void evergreen_cp_dma_clear_buffer(struct r600_context *rctx,
+ struct pipe_resource *dst, uint64_t offset,
+ unsigned size, uint32_t clear_value,
+ enum r600_coherency coher)
+{
+ struct radeon_cmdbuf *cs = rctx->b.gfx.cs;
+
+ assert(size);
+ assert(rctx->screen->b.has_cp_dma);
+
+ /* Mark the buffer range of destination as valid (initialized),
+ * so that transfer_map knows it should wait for the GPU when mapping
+ * that range. */
+ util_range_add(dst, &r600_resource(dst)->valid_buffer_range, offset,
+ offset + size);
+
+ offset += r600_resource(dst)->gpu_address;
+
+ /* Flush the cache where the resource is bound. */
+ rctx->b.flags |= r600_get_flush_flags(coher) |
+ R600_CONTEXT_WAIT_3D_IDLE;
+
+ while (size) {
+ unsigned sync = 0;
+ unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
+ unsigned reloc;
+
+ r600_need_cs_space(rctx,
+ 10 + (rctx->b.flags ? R600_MAX_FLUSH_CS_DWORDS : 0) +
+ R600_MAX_PFP_SYNC_ME_DWORDS, FALSE, 0);
+
+ /* Flush the caches for the first copy only. */
+ if (rctx->b.flags) {
+ r600_flush_emit(rctx);
+ }
+
+ /* Do the synchronization after the last copy, so that all data is written to memory. */
+ if (size == byte_count) {
+ sync = PKT3_CP_DMA_CP_SYNC;
+ }
+
+ /* This must be done after r600_need_cs_space. */
+ reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx,
+ (struct r600_resource*)dst, RADEON_USAGE_WRITE,
+ RADEON_PRIO_CP_DMA);
+
+ radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
+ radeon_emit(cs, clear_value); /* DATA [31:0] */
+ radeon_emit(cs, sync | PKT3_CP_DMA_SRC_SEL(2)); /* CP_SYNC [31] | SRC_SEL[30:29] */
+ radeon_emit(cs, offset); /* DST_ADDR_LO [31:0] */
+ radeon_emit(cs, (offset >> 32) & 0xff); /* DST_ADDR_HI [7:0] */
+ radeon_emit(cs, byte_count); /* COMMAND [29:22] | BYTE_COUNT [20:0] */
+
+ radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
+ radeon_emit(cs, reloc);
+
+ size -= byte_count;
+ offset += byte_count;
+ }
+
+ /* CP DMA is executed in ME, but index buffers are read by PFP.
+ * This ensures that ME (CP DMA) is idle before PFP starts fetching
+ * indices. If we wanted to execute CP DMA in PFP, this packet
+ * should precede it.
+ */
+ if (coher == R600_COHERENCY_SHADER)
+ r600_emit_pfp_sync_me(rctx);
}
projects / mesa.git / blobdiff