From: Francisco Jerez Date: Fri, 24 Apr 2020 00:58:48 +0000 (-0700) Subject: iris: Bracket batch operations which access memory within sync regions. X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=e81c07de41c7f6f585a2c6aa0c67b1082b609b8f;hp=8cbe9535482f3efd27fbcbb90a329e6567e8c961;p=mesa.git iris: Bracket batch operations which access memory within sync regions. This delimits all batch operations which access memory between iris_batch_sync_region_start() and iris_batch_sync_region_end() calls. This makes sure that any buffer objects accessed within the region are considered in use through the same caching domain until the end of the region. Adding any buffer to the batch validation list outside of a sync region will lead to an assertion failure in a future commit, unless the caller explicitly opted out of the cache tracking mechanism. Reviewed-by: Kenneth Graunke Part-of: --- diff --git a/src/gallium/drivers/iris/iris_blit.c b/src/gallium/drivers/iris/iris_blit.c index f0a43e959a5..51667f4c5db 100644 --- a/src/gallium/drivers/iris/iris_blit.c +++ b/src/gallium/drivers/iris/iris_blit.c @@ -487,6 +487,7 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) if (info->mask & main_mask) { for (int slice = 0; slice < info->dst.box.depth; slice++) { iris_batch_maybe_flush(batch, 1500); + iris_batch_sync_region_start(batch); blorp_blit(&blorp_batch, &src_surf, info->src.level, info->src.box.z + slice, @@ -496,6 +497,8 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) src_x0, src_y0, src_x1, src_y1, dst_x0, dst_y0, dst_x1, dst_y1, filter, mirror_x, mirror_y); + + iris_batch_sync_region_end(batch); } } @@ -536,6 +539,7 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) for (int slice = 0; slice < info->dst.box.depth; slice++) { iris_batch_maybe_flush(batch, 1500); + iris_batch_sync_region_start(batch); blorp_blit(&blorp_batch, &src_surf, info->src.level, info->src.box.z + slice, @@ -545,6 +549,8 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) src_x0, src_y0, src_x1, src_y1, dst_x0, dst_y0, dst_x1, dst_y1, filter, mirror_x, mirror_y); + + iris_batch_sync_region_end(batch); } } @@ -660,9 +666,11 @@ iris_copy_region(struct blorp_context *blorp, iris_batch_maybe_flush(batch, 1500); + iris_batch_sync_region_start(batch); blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width); blorp_batch_finish(&blorp_batch); + iris_batch_sync_region_end(batch); } else { // XXX: what about one surface being a buffer and not the other? @@ -684,10 +692,12 @@ iris_copy_region(struct blorp_context *blorp, for (int slice = 0; slice < src_box->depth; slice++) { iris_batch_maybe_flush(batch, 1500); + iris_batch_sync_region_start(batch); blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice, &dst_surf, dst_level, dstz + slice, src_box->x, src_box->y, dstx, dsty, src_box->width, src_box->height); + iris_batch_sync_region_end(batch); } blorp_batch_finish(&blorp_batch); diff --git a/src/gallium/drivers/iris/iris_clear.c b/src/gallium/drivers/iris/iris_clear.c index 3800dfcc145..67730e391ee 100644 --- a/src/gallium/drivers/iris/iris_clear.c +++ b/src/gallium/drivers/iris/iris_clear.c @@ -296,6 +296,8 @@ fast_clear_color(struct iris_context *ice, "fast clear: pre-flush", PIPE_CONTROL_RENDER_TARGET_FLUSH); + iris_batch_sync_region_start(batch); + /* If we reach this point, we need to fast clear to change the state to * ISL_AUX_STATE_CLEAR, or to update the fast clear color (or both). */ @@ -323,6 +325,7 @@ fast_clear_color(struct iris_context *ice, iris_emit_end_of_pipe_sync(batch, "fast clear: post flush", PIPE_CONTROL_RENDER_TARGET_FLUSH); + iris_batch_sync_region_end(batch); iris_resource_set_aux_state(ice, res, level, box->z, box->depth, ISL_AUX_STATE_CLEAR); @@ -380,6 +383,8 @@ clear_color(struct iris_context *ice, iris_blorp_surf_for_resource(&batch->screen->isl_dev, &surf, p_res, aux_usage, level, true); + iris_batch_sync_region_start(batch); + struct blorp_batch blorp_batch; blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags); @@ -393,6 +398,8 @@ clear_color(struct iris_context *ice, color, color_write_disable); blorp_batch_finish(&blorp_batch); + iris_batch_sync_region_end(batch); + iris_flush_and_dirty_for_history(ice, batch, res, PIPE_CONTROL_RENDER_TARGET_FLUSH, "cache history: post color clear"); @@ -594,9 +601,6 @@ clear_depth_stencil(struct iris_context *ice, level, true); } - struct blorp_batch blorp_batch; - blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags); - uint8_t stencil_mask = clear_stencil && stencil_res ? 0xff : 0; if (stencil_mask) { iris_resource_prepare_access(ice, batch, stencil_res, level, 1, box->z, @@ -606,6 +610,11 @@ clear_depth_stencil(struct iris_context *ice, stencil_res->aux.usage, level, true); } + iris_batch_sync_region_start(batch); + + struct blorp_batch blorp_batch; + blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags); + blorp_clear_depth_stencil(&blorp_batch, &z_surf, &stencil_surf, level, box->z, box->depth, box->x, box->y, @@ -615,6 +624,8 @@ clear_depth_stencil(struct iris_context *ice, stencil_mask, stencil); blorp_batch_finish(&blorp_batch); + iris_batch_sync_region_end(batch); + iris_flush_and_dirty_for_history(ice, batch, res, 0, "cache history: post slow ZS clear"); diff --git a/src/gallium/drivers/iris/iris_query.c b/src/gallium/drivers/iris/iris_query.c index 8888a8fd284..aaed40e5e08 100644 --- a/src/gallium/drivers/iris/iris_query.c +++ b/src/gallium/drivers/iris/iris_query.c @@ -711,6 +711,8 @@ iris_get_query_result_resource(struct pipe_context *ctx, struct gen_mi_builder b; gen_mi_builder_init(&b, batch); + iris_batch_sync_region_start(batch); + struct gen_mi_value result = calculate_result_on_gpu(devinfo, &b, q); struct gen_mi_value dst = result_type <= PIPE_QUERY_TYPE_U32 ? gen_mi_mem32(rw_bo(dst_bo, offset)) @@ -723,6 +725,8 @@ iris_get_query_result_resource(struct pipe_context *ctx, } else { gen_mi_store(&b, dst, result); } + + iris_batch_sync_region_end(batch); } static void @@ -763,6 +767,8 @@ set_predicate_for_result(struct iris_context *ice, struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); + iris_batch_sync_region_start(batch); + /* The CPU doesn't have the query result yet; use hardware predication */ ice->state.predicate = IRIS_PREDICATE_STATE_USE_BIT; @@ -809,6 +815,8 @@ set_predicate_for_result(struct iris_context *ice, gen_mi_store(&b, query_mem64(q, offsetof(struct iris_query_snapshots, predicate_result)), result); ice->state.compute_predicate = bo; + + iris_batch_sync_region_end(batch); } static void diff --git a/src/gallium/drivers/iris/iris_resolve.c b/src/gallium/drivers/iris/iris_resolve.c index f763f45f01e..1b0c22fc09a 100644 --- a/src/gallium/drivers/iris/iris_resolve.c +++ b/src/gallium/drivers/iris/iris_resolve.c @@ -499,6 +499,7 @@ iris_resolve_color(struct iris_context *ice, iris_emit_end_of_pipe_sync(batch, "color resolve: pre-flush", PIPE_CONTROL_RENDER_TARGET_FLUSH); + iris_batch_sync_region_start(batch); struct blorp_batch blorp_batch; blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); /* On Gen >= 12, Stencil buffer with lossless compression needs to be @@ -517,6 +518,7 @@ iris_resolve_color(struct iris_context *ice, /* See comment above */ iris_emit_end_of_pipe_sync(batch, "color resolve: post-flush", PIPE_CONTROL_RENDER_TARGET_FLUSH); + iris_batch_sync_region_end(batch); } static void @@ -536,11 +538,13 @@ iris_mcs_partial_resolve(struct iris_context *ice, &res->base, res->aux.usage, 0, true); struct blorp_batch blorp_batch; + iris_batch_sync_region_start(batch); blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); blorp_mcs_partial_resolve(&blorp_batch, &surf, isl_format_srgb_to_linear(res->surf.format), start_layer, num_layers); blorp_batch_finish(&blorp_batch); + iris_batch_sync_region_end(batch); } @@ -681,6 +685,8 @@ iris_hiz_exec(struct iris_context *ice, iris_batch_maybe_flush(batch, 1500); + iris_batch_sync_region_start(batch); + struct blorp_surf surf; iris_blorp_surf_for_resource(&batch->screen->isl_dev, &surf, &res->base, res->aux.usage, level, true); @@ -712,6 +718,8 @@ iris_hiz_exec(struct iris_context *ice, "hiz op: post flush", PIPE_CONTROL_DEPTH_CACHE_FLUSH | PIPE_CONTROL_DEPTH_STALL); + + iris_batch_sync_region_end(batch); } static bool diff --git a/src/gallium/drivers/iris/iris_state.c b/src/gallium/drivers/iris/iris_state.c index bc394938536..54291868e7d 100644 --- a/src/gallium/drivers/iris/iris_state.c +++ b/src/gallium/drivers/iris/iris_state.c @@ -526,10 +526,12 @@ static void iris_load_register_mem32(struct iris_batch *batch, uint32_t reg, struct iris_bo *bo, uint32_t offset) { + iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) { lrm.RegisterAddress = reg; lrm.MemoryAddress = ro_bo(bo, offset); } + iris_batch_sync_region_end(batch); } /** @@ -549,11 +551,13 @@ iris_store_register_mem32(struct iris_batch *batch, uint32_t reg, struct iris_bo *bo, uint32_t offset, bool predicated) { + iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_STORE_REGISTER_MEM), srm) { srm.RegisterAddress = reg; srm.MemoryAddress = rw_bo(bo, offset); srm.PredicateEnable = predicated; } + iris_batch_sync_region_end(batch); } static void @@ -570,10 +574,12 @@ iris_store_data_imm32(struct iris_batch *batch, struct iris_bo *bo, uint32_t offset, uint32_t imm) { + iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_STORE_DATA_IMM), sdi) { sdi.Address = rw_bo(bo, offset); sdi.ImmediateData = imm; } + iris_batch_sync_region_end(batch); } static void @@ -585,11 +591,13 @@ iris_store_data_imm64(struct iris_batch *batch, * 2 in genxml but it's actually variable length and we need 5 DWords. */ void *map = iris_get_command_space(batch, 4 * 5); + iris_batch_sync_region_start(batch); _iris_pack_command(batch, GENX(MI_STORE_DATA_IMM), map, sdi) { sdi.DWordLength = 5 - 2; sdi.Address = rw_bo(bo, offset); sdi.ImmediateData = imm; } + iris_batch_sync_region_end(batch); } static void @@ -602,6 +610,7 @@ iris_copy_mem_mem(struct iris_batch *batch, assert(bytes % 4 == 0); assert(dst_offset % 4 == 0); assert(src_offset % 4 == 0); + iris_batch_sync_region_start(batch); for (unsigned i = 0; i < bytes; i += 4) { iris_emit_cmd(batch, GENX(MI_COPY_MEM_MEM), cp) { @@ -609,6 +618,8 @@ iris_copy_mem_mem(struct iris_batch *batch, cp.SourceMemoryAddress = ro_bo(src_bo, src_offset + i); } } + + iris_batch_sync_region_end(batch); } static void @@ -898,6 +909,8 @@ iris_init_render_context(struct iris_batch *batch) UNUSED const struct gen_device_info *devinfo = &batch->screen->devinfo; uint32_t reg_val; + iris_batch_sync_region_start(batch); + emit_pipeline_select(batch, _3D); iris_emit_l3_config(batch, batch->screen->l3_config_3d); @@ -1003,9 +1016,12 @@ iris_init_render_context(struct iris_batch *batch) iris_alloc_push_constants(batch); + #if GEN_GEN >= 12 init_aux_map_state(batch); #endif + + iris_batch_sync_region_end(batch); } static void @@ -1013,6 +1029,8 @@ iris_init_compute_context(struct iris_batch *batch) { UNUSED const struct gen_device_info *devinfo = &batch->screen->devinfo; + iris_batch_sync_region_start(batch); + /* GEN:BUG:1607854226: * * Start with pipeline in 3D mode to set the STATE_BASE_ADDRESS. @@ -1040,6 +1058,7 @@ iris_init_compute_context(struct iris_batch *batch) init_aux_map_state(batch); #endif + iris_batch_sync_region_end(batch); } struct iris_vertex_buffer_state { @@ -5079,6 +5098,8 @@ iris_update_surface_base_address(struct iris_batch *batch, uint32_t mocs = batch->screen->isl_dev.mocs.internal; + iris_batch_sync_region_start(batch); + flush_before_state_base_change(batch); #if GEN_GEN == 12 @@ -5119,6 +5140,7 @@ iris_update_surface_base_address(struct iris_batch *batch, #endif flush_after_state_base_change(batch); + iris_batch_sync_region_end(batch); batch->last_surface_base_address = binder->bo->gtt_offset; } @@ -6290,6 +6312,8 @@ iris_upload_render_state(struct iris_context *ice, { bool use_predicate = ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT; + iris_batch_sync_region_start(batch); + /* Always pin the binder. If we're emitting new binding table pointers, * we need it. If not, we're probably inheriting old tables via the * context, and need it anyway. Since true zero-bindings cases are @@ -6496,6 +6520,8 @@ iris_upload_render_state(struct iris_context *ice, } } } + + iris_batch_sync_region_end(batch); } static void @@ -6518,6 +6544,8 @@ iris_upload_compute_state(struct iris_context *ice, brw_cs_simd_size_for_group_size(devinfo, cs_prog_data, group_size); const unsigned threads = DIV_ROUND_UP(group_size, simd_size); + iris_batch_sync_region_start(batch); + /* Always pin the binder. If we're emitting new binding table pointers, * we need it. If not, we're probably inheriting old tables via the * context, and need it anyway. Since true zero-bindings cases are @@ -6676,6 +6704,8 @@ iris_upload_compute_state(struct iris_context *ice, iris_restore_compute_saved_bos(ice, batch, grid); batch->contains_draw = true; } + + iris_batch_sync_region_end(batch); } /** @@ -7297,6 +7327,8 @@ iris_emit_raw_pipe_control(struct iris_batch *batch, imm, reason); } + iris_batch_sync_region_start(batch); + iris_emit_cmd(batch, GENX(PIPE_CONTROL), pc) { #if GEN_GEN >= 12 pc.TileCacheFlushEnable = flags & PIPE_CONTROL_TILE_CACHE_FLUSH; @@ -7334,6 +7366,8 @@ iris_emit_raw_pipe_control(struct iris_batch *batch, pc.Address = rw_bo(bo, offset); pc.ImmediateData = imm; } + + iris_batch_sync_region_end(batch); } #if GEN_GEN == 9 @@ -7418,10 +7452,12 @@ iris_emit_mi_report_perf_count(struct iris_batch *batch, uint32_t offset_in_bytes, uint32_t report_id) { + iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_REPORT_PERF_COUNT), mi_rpc) { mi_rpc.MemoryAddress = rw_bo(bo, offset_in_bytes); mi_rpc.ReportID = report_id; } + iris_batch_sync_region_end(batch); } /**