X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fr600%2Fr600_hw_context.c;h=ca7f41db68622db82c2616f120b0e4d3e8f172db;hb=5d8359ff4d8c379fdf1a78758f405bb4cdf69459;hp=7a6f957945be9e7a49dfdc8a6a57d029e1e2dac9;hpb=8140154ae92c6bd022e409790bb069966a857aed;p=mesa.git diff --git a/src/gallium/drivers/r600/r600_hw_context.c b/src/gallium/drivers/r600/r600_hw_context.c index 7a6f957945b..ca7f41db686 100644 --- a/src/gallium/drivers/r600/r600_hw_context.c +++ b/src/gallium/drivers/r600/r600_hw_context.c @@ -33,13 +33,12 @@ void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw, boolean count_draw_in) { - struct radeon_winsys_cs *dma = ctx->b.dma.cs; - /* Flush the DMA IB if it's not empty. */ - if (dma && dma->cdw) + if (radeon_emitted(ctx->b.dma.cs, 0)) ctx->b.dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL); - if (!ctx->b.ws->cs_memory_below_limit(ctx->b.gfx.cs, ctx->b.vram, ctx->b.gtt)) { + if (!radeon_cs_memory_below_limit(ctx->b.screen, ctx->b.gfx.cs, + ctx->b.vram, ctx->b.gtt)) { ctx->b.gtt = 0; ctx->b.vram = 0; ctx->b.gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL); @@ -49,9 +48,7 @@ void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw, ctx->b.gtt = 0; ctx->b.vram = 0; - /* The number of dwords we already used in the CS so far. */ - num_dw += ctx->b.gfx.cs->cdw; - + /* Check available space in CS. */ if (count_draw_in) { uint64_t mask; @@ -64,9 +61,8 @@ void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw, num_dw += R600_MAX_FLUSH_CS_DWORDS + R600_MAX_DRAW_CS_DWORDS; } - /* Count in queries_suspend. */ - num_dw += ctx->b.num_cs_dw_nontimer_queries_suspend + - ctx->b.num_cs_dw_timer_queries_suspend; + /* Count in r600_suspend_queries. */ + num_dw += ctx->b.num_cs_dw_queries_suspend; /* Count in streamout_end at the end of CS. */ if (ctx->b.streamout.begin_emitted) { @@ -85,7 +81,7 @@ void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw, num_dw += 10; /* Flush if there's not enough space. */ - if (num_dw > ctx->b.gfx.cs->max_dw) { + if (!ctx->b.ws->cs_check_space(ctx->b.gfx.cs, num_dw)) { ctx->b.gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL); } } @@ -100,6 +96,10 @@ void r600_flush_emit(struct r600_context *rctx) return; } + /* Ensure coherency between streamout and shaders. */ + if (rctx->b.flags & R600_CONTEXT_STREAMOUT_FLUSH) + rctx->b.flags |= r600_get_flush_flags(R600_COHERENCY_SHADER); + if (rctx->b.flags & R600_CONTEXT_WAIT_3D_IDLE) { wait_until |= S_008040_WAIT_3D_IDLE(1); } @@ -115,21 +115,32 @@ void r600_flush_emit(struct r600_context *rctx) } } + /* Wait packets must be executed first, because SURFACE_SYNC doesn't + * wait for shaders if it's not flushing CB or DB. + */ if (rctx->b.flags & R600_CONTEXT_PS_PARTIAL_FLUSH) { - cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0); - cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4)); + } + + if (wait_until) { + /* Use of WAIT_UNTIL is deprecated on Cayman+ */ + if (rctx->b.family < CHIP_CAYMAN) { + /* wait for things to settle */ + radeon_set_config_reg(cs, R_008040_WAIT_UNTIL, wait_until); + } } if (rctx->b.chip_class >= R700 && (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_CB_META)) { - cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0); - cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_CB_META) | EVENT_INDEX(0); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_CB_META) | EVENT_INDEX(0)); } if (rctx->b.chip_class >= R700 && (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_DB_META)) { - cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0); - cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_DB_META) | EVENT_INDEX(0); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_DB_META) | EVENT_INDEX(0)); /* Set FULL_CACHE_ENA for DB META flushes on r7xx and later. * @@ -142,8 +153,8 @@ void r600_flush_emit(struct r600_context *rctx) if (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV || (rctx->b.chip_class == R600 && rctx->b.flags & R600_CONTEXT_STREAMOUT_FLUSH)) { - cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0); - cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0)); } if (rctx->b.flags & R600_CONTEXT_INV_CONST_CACHE) { @@ -216,19 +227,21 @@ void r600_flush_emit(struct r600_context *rctx) } if (cp_coher_cntl) { - cs->buf[cs->cdw++] = PKT3(PKT3_SURFACE_SYNC, 3, 0); - cs->buf[cs->cdw++] = cp_coher_cntl; /* CP_COHER_CNTL */ - cs->buf[cs->cdw++] = 0xffffffff; /* CP_COHER_SIZE */ - cs->buf[cs->cdw++] = 0; /* CP_COHER_BASE */ - cs->buf[cs->cdw++] = 0x0000000A; /* POLL_INTERVAL */ + radeon_emit(cs, PKT3(PKT3_SURFACE_SYNC, 3, 0)); + radeon_emit(cs, cp_coher_cntl); /* CP_COHER_CNTL */ + radeon_emit(cs, 0xffffffff); /* CP_COHER_SIZE */ + radeon_emit(cs, 0); /* CP_COHER_BASE */ + radeon_emit(cs, 0x0000000A); /* POLL_INTERVAL */ } - if (wait_until) { - /* Use of WAIT_UNTIL is deprecated on Cayman+ */ - if (rctx->b.family < CHIP_CAYMAN) { - /* wait for things to settle */ - radeon_set_config_reg(cs, R_008040_WAIT_UNTIL, wait_until); - } + if (rctx->b.flags & R600_CONTEXT_START_PIPELINE_STATS) { + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PIPELINESTAT_START) | + EVENT_INDEX(0)); + } else if (rctx->b.flags & R600_CONTEXT_STOP_PIPELINE_STATS) { + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PIPELINESTAT_STOP) | + EVENT_INDEX(0)); } /* everything is properly flushed */ @@ -240,8 +253,12 @@ void r600_context_gfx_flush(void *context, unsigned flags, { struct r600_context *ctx = context; struct radeon_winsys_cs *cs = ctx->b.gfx.cs; + struct radeon_winsys *ws = ctx->b.ws; - if (cs->cdw == ctx->b.initial_gfx_cs_size && !fence) + if (!radeon_emitted(cs, ctx->b.initial_gfx_cs_size)) + return; + + if (r600_check_device_reset(&ctx->b)) return; r600_preflush_suspend_features(&ctx->b); @@ -257,17 +274,40 @@ void r600_context_gfx_flush(void *context, unsigned flags, r600_flush_emit(ctx); + if (ctx->trace_buf) + eg_trace_emit(ctx); /* old kernels and userspace don't set SX_MISC, so we must reset it to 0 here */ if (ctx->b.chip_class == R600) { radeon_set_context_reg(cs, R_028350_SX_MISC, 0); } - /* force to keep tiling flags */ - flags |= RADEON_FLUSH_KEEP_TILING_FLAGS; - + if (ctx->is_debug) { + /* Save the IB for debug contexts. */ + radeon_clear_saved_cs(&ctx->last_gfx); + radeon_save_cs(ws, cs, &ctx->last_gfx); + r600_resource_reference(&ctx->last_trace_buf, ctx->trace_buf); + r600_resource_reference(&ctx->trace_buf, NULL); + } /* Flush the CS. */ - ctx->b.ws->cs_flush(cs, flags, fence); - + ws->cs_flush(cs, flags, &ctx->b.last_gfx_fence); + if (fence) + ws->fence_reference(fence, ctx->b.last_gfx_fence); + ctx->b.num_gfx_cs_flushes++; + + if (ctx->is_debug) { + if (!ws->fence_wait(ws, ctx->b.last_gfx_fence, 10000000)) { + const char *fname = getenv("R600_TRACE"); + if (!fname) + exit(-1); + FILE *fl = fopen(fname, "w+"); + if (fl) { + eg_dump_debug_state(&ctx->b.b, fl, 0); + fclose(fl); + } else + perror(fname); + exit(-1); + } + } r600_begin_new_cs(ctx); } @@ -275,6 +315,23 @@ void r600_begin_new_cs(struct r600_context *ctx) { unsigned shader; + if (ctx->is_debug) { + uint32_t zero = 0; + + /* Create a buffer used for writing trace IDs and initialize it to 0. */ + assert(!ctx->trace_buf); + ctx->trace_buf = (struct r600_resource*) + pipe_buffer_create(ctx->b.b.screen, 0, + PIPE_USAGE_STAGING, 4); + if (ctx->trace_buf) + pipe_buffer_write_nooverlap(&ctx->b.b, &ctx->trace_buf->b.b, + 0, sizeof(zero), &zero); + ctx->trace_id = 0; + } + + if (ctx->trace_buf) + eg_trace_emit(ctx); + ctx->b.flags = 0; ctx->b.gtt = 0; ctx->b.vram = 0; @@ -295,12 +352,11 @@ void r600_begin_new_cs(struct r600_context *ctx) r600_mark_atom_dirty(ctx, &ctx->poly_offset_state.atom); r600_mark_atom_dirty(ctx, &ctx->vgt_state.atom); r600_mark_atom_dirty(ctx, &ctx->sample_mask.atom); - ctx->scissor.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1; - ctx->scissor.atom.num_dw = R600_MAX_VIEWPORTS * 4; - r600_mark_atom_dirty(ctx, &ctx->scissor.atom); - ctx->viewport.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1; - ctx->viewport.atom.num_dw = R600_MAX_VIEWPORTS * 8; - r600_mark_atom_dirty(ctx, &ctx->viewport.atom); + ctx->b.scissors.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1; + r600_mark_atom_dirty(ctx, &ctx->b.scissors.atom); + ctx->b.viewports.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1; + ctx->b.viewports.depth_range_dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1; + r600_mark_atom_dirty(ctx, &ctx->b.viewports.atom); if (ctx->b.chip_class <= EVERGREEN) { r600_mark_atom_dirty(ctx, &ctx->config_state.atom); } @@ -353,8 +409,71 @@ void r600_begin_new_cs(struct r600_context *ctx) /* Re-emit the draw state. */ ctx->last_primitive_type = -1; ctx->last_start_instance = -1; + ctx->last_rast_prim = -1; + ctx->current_rast_prim = -1; - ctx->b.initial_gfx_cs_size = ctx->b.gfx.cs->cdw; + assert(!ctx->b.gfx.cs->prev_dw); + ctx->b.initial_gfx_cs_size = ctx->b.gfx.cs->current.cdw; +} + +void r600_emit_pfp_sync_me(struct r600_context *rctx) +{ + struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + + if (rctx->b.chip_class >= EVERGREEN && + rctx->b.screen->info.drm_minor >= 46) { + radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(cs, 0); + } else { + /* Emulate PFP_SYNC_ME by writing a value to memory in ME and + * waiting for it in PFP. + */ + struct r600_resource *buf = NULL; + unsigned offset, reloc; + uint64_t va; + + /* 16-byte address alignment is required by WAIT_REG_MEM. */ + u_suballocator_alloc(rctx->b.allocator_zeroed_memory, 4, 16, + &offset, (struct pipe_resource**)&buf); + if (!buf) { + /* This is too heavyweight, but will work. */ + rctx->b.gfx.flush(rctx, RADEON_FLUSH_ASYNC, NULL); + return; + } + + reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, buf, + RADEON_USAGE_READWRITE, + RADEON_PRIO_FENCE); + + va = buf->gpu_address + offset; + assert(va % 16 == 0); + + /* Write 1 to memory in ME. */ + radeon_emit(cs, PKT3(PKT3_MEM_WRITE, 3, 0)); + radeon_emit(cs, va); + radeon_emit(cs, ((va >> 32) & 0xff) | MEM_WRITE_32_BITS); + radeon_emit(cs, 1); + radeon_emit(cs, 0); + + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, reloc); + + /* Wait in PFP (PFP can only do GEQUAL against memory). */ + radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); + radeon_emit(cs, WAIT_REG_MEM_GEQUAL | + WAIT_REG_MEM_MEMORY | + WAIT_REG_MEM_PFP); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + radeon_emit(cs, 1); /* reference value */ + radeon_emit(cs, 0xffffffff); /* mask */ + radeon_emit(cs, 4); /* poll interval */ + + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, reloc); + + r600_resource_reference(&buf, NULL); + } } /* The max number of bytes to copy per packet. */ @@ -380,15 +499,7 @@ void r600_cp_dma_copy_buffer(struct r600_context *rctx, src_offset += r600_resource(src)->gpu_address; /* Flush the caches where the resources are bound. */ - rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE | - R600_CONTEXT_INV_VERTEX_CACHE | - R600_CONTEXT_INV_TEX_CACHE | - R600_CONTEXT_FLUSH_AND_INV | - R600_CONTEXT_FLUSH_AND_INV_CB | - R600_CONTEXT_FLUSH_AND_INV_DB | - R600_CONTEXT_FLUSH_AND_INV_CB_META | - R600_CONTEXT_FLUSH_AND_INV_DB_META | - R600_CONTEXT_STREAMOUT_FLUSH | + rctx->b.flags |= r600_get_flush_flags(R600_COHERENCY_SHADER) | R600_CONTEXT_WAIT_3D_IDLE; /* There are differences between R700 and EG in CP DMA, @@ -398,7 +509,9 @@ void r600_cp_dma_copy_buffer(struct r600_context *rctx, unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT); unsigned src_reloc, dst_reloc; - r600_need_cs_space(rctx, 10 + (rctx->b.flags ? R600_MAX_FLUSH_CS_DWORDS : 0), FALSE); + r600_need_cs_space(rctx, + 10 + (rctx->b.flags ? R600_MAX_FLUSH_CS_DWORDS : 0) + + 3 + R600_MAX_PFP_SYNC_ME_DWORDS, FALSE); /* Flush the caches for the first copy only. */ if (rctx->b.flags) { @@ -433,10 +546,17 @@ void r600_cp_dma_copy_buffer(struct r600_context *rctx, dst_offset += byte_count; } - /* Invalidate the read caches. */ - rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE | - R600_CONTEXT_INV_VERTEX_CACHE | - R600_CONTEXT_INV_TEX_CACHE; + /* CP_DMA_CP_SYNC doesn't wait for idle on R6xx, but this does. */ + if (rctx->b.chip_class == R600) + radeon_set_config_reg(cs, R_008040_WAIT_UNTIL, + S_008040_WAIT_CP_DMA_IDLE(1)); + + /* CP DMA is executed in ME, but index buffers are read by PFP. + * This ensures that ME (CP DMA) is idle before PFP starts fetching + * indices. If we wanted to execute CP DMA in PFP, this packet + * should precede it. + */ + r600_emit_pfp_sync_me(rctx); } void r600_dma_copy_buffer(struct r600_context *rctx, @@ -460,7 +580,7 @@ void r600_dma_copy_buffer(struct r600_context *rctx, size >>= 2; /* convert to dwords */ ncopy = (size / R600_DMA_COPY_MAX_SIZE_DW) + !!(size % R600_DMA_COPY_MAX_SIZE_DW); - r600_need_dma_space(&rctx->b, ncopy * 5); + r600_need_dma_space(&rctx->b, ncopy * 5, rdst, rsrc); for (i = 0; i < ncopy; i++) { csize = size < R600_DMA_COPY_MAX_SIZE_DW ? size : R600_DMA_COPY_MAX_SIZE_DW; /* emit reloc before writing cs so that cs is always in consistent state */ @@ -468,11 +588,11 @@ void r600_dma_copy_buffer(struct r600_context *rctx, RADEON_PRIO_SDMA_BUFFER); radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rdst, RADEON_USAGE_WRITE, RADEON_PRIO_SDMA_BUFFER); - cs->buf[cs->cdw++] = DMA_PACKET(DMA_PACKET_COPY, 0, 0, csize); - cs->buf[cs->cdw++] = dst_offset & 0xfffffffc; - cs->buf[cs->cdw++] = src_offset & 0xfffffffc; - cs->buf[cs->cdw++] = (dst_offset >> 32UL) & 0xff; - cs->buf[cs->cdw++] = (src_offset >> 32UL) & 0xff; + radeon_emit(cs, DMA_PACKET(DMA_PACKET_COPY, 0, 0, csize)); + radeon_emit(cs, dst_offset & 0xfffffffc); + radeon_emit(cs, src_offset & 0xfffffffc); + radeon_emit(cs, (dst_offset >> 32UL) & 0xff); + radeon_emit(cs, (src_offset >> 32UL) & 0xff); dst_offset += csize << 2; src_offset += csize << 2; size -= csize;