iris: Bracket batch operations which access memory within sync regions.
authorFrancisco Jerez <currojerez@riseup.net>
Fri, 24 Apr 2020 00:58:48 +0000 (17:58 -0700)
committerMarge Bot <eric+marge@anholt.net>
Wed, 3 Jun 2020 23:12:22 +0000 (23:12 +0000)
This delimits all batch operations which access memory between
iris_batch_sync_region_start() and iris_batch_sync_region_end() calls.
This makes sure that any buffer objects accessed within the region are
considered in use through the same caching domain until the end of the
region.

Adding any buffer to the batch validation list outside of a sync
region will lead to an assertion failure in a future commit, unless
the caller explicitly opted out of the cache tracking mechanism.

Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3875>

src/gallium/drivers/iris/iris_blit.c
src/gallium/drivers/iris/iris_clear.c
src/gallium/drivers/iris/iris_query.c
src/gallium/drivers/iris/iris_resolve.c
src/gallium/drivers/iris/iris_state.c

index f0a43e959a58c1d7602ccdeb0801659ad5f46e10..51667f4c5dbc79f2a04173f2bc878b0aebb782eb 100644 (file)
@@ -487,6 +487,7 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
    if (info->mask & main_mask) {
       for (int slice = 0; slice < info->dst.box.depth; slice++) {
          iris_batch_maybe_flush(batch, 1500);
+         iris_batch_sync_region_start(batch);
 
          blorp_blit(&blorp_batch,
                     &src_surf, info->src.level, info->src.box.z + slice,
@@ -496,6 +497,8 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
                     src_x0, src_y0, src_x1, src_y1,
                     dst_x0, dst_y0, dst_x1, dst_y1,
                     filter, mirror_x, mirror_y);
+
+         iris_batch_sync_region_end(batch);
       }
    }
 
@@ -536,6 +539,7 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
 
       for (int slice = 0; slice < info->dst.box.depth; slice++) {
          iris_batch_maybe_flush(batch, 1500);
+         iris_batch_sync_region_start(batch);
 
          blorp_blit(&blorp_batch,
                     &src_surf, info->src.level, info->src.box.z + slice,
@@ -545,6 +549,8 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
                     src_x0, src_y0, src_x1, src_y1,
                     dst_x0, dst_y0, dst_x1, dst_y1,
                     filter, mirror_x, mirror_y);
+
+         iris_batch_sync_region_end(batch);
       }
    }
 
@@ -660,9 +666,11 @@ iris_copy_region(struct blorp_context *blorp,
 
       iris_batch_maybe_flush(batch, 1500);
 
+      iris_batch_sync_region_start(batch);
       blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
       blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
       blorp_batch_finish(&blorp_batch);
+      iris_batch_sync_region_end(batch);
    } else {
       // XXX: what about one surface being a buffer and not the other?
 
@@ -684,10 +692,12 @@ iris_copy_region(struct blorp_context *blorp,
       for (int slice = 0; slice < src_box->depth; slice++) {
          iris_batch_maybe_flush(batch, 1500);
 
+         iris_batch_sync_region_start(batch);
          blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice,
                     &dst_surf, dst_level, dstz + slice,
                     src_box->x, src_box->y, dstx, dsty,
                     src_box->width, src_box->height);
+         iris_batch_sync_region_end(batch);
       }
       blorp_batch_finish(&blorp_batch);
 
index 3800dfcc145e906768e55222c2f05f71e4fa24b1..67730e391eed3ce5f20aab8c8388a359df917e6f 100644 (file)
@@ -296,6 +296,8 @@ fast_clear_color(struct iris_context *ice,
                               "fast clear: pre-flush",
                               PIPE_CONTROL_RENDER_TARGET_FLUSH);
 
+   iris_batch_sync_region_start(batch);
+
    /* If we reach this point, we need to fast clear to change the state to
     * ISL_AUX_STATE_CLEAR, or to update the fast clear color (or both).
     */
@@ -323,6 +325,7 @@ fast_clear_color(struct iris_context *ice,
    iris_emit_end_of_pipe_sync(batch,
                               "fast clear: post flush",
                               PIPE_CONTROL_RENDER_TARGET_FLUSH);
+   iris_batch_sync_region_end(batch);
 
    iris_resource_set_aux_state(ice, res, level, box->z,
                                box->depth, ISL_AUX_STATE_CLEAR);
@@ -380,6 +383,8 @@ clear_color(struct iris_context *ice,
    iris_blorp_surf_for_resource(&batch->screen->isl_dev, &surf,
                                 p_res, aux_usage, level, true);
 
+   iris_batch_sync_region_start(batch);
+
    struct blorp_batch blorp_batch;
    blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
 
@@ -393,6 +398,8 @@ clear_color(struct iris_context *ice,
                color, color_write_disable);
 
    blorp_batch_finish(&blorp_batch);
+   iris_batch_sync_region_end(batch);
+
    iris_flush_and_dirty_for_history(ice, batch, res,
                                     PIPE_CONTROL_RENDER_TARGET_FLUSH,
                                     "cache history: post color clear");
@@ -594,9 +601,6 @@ clear_depth_stencil(struct iris_context *ice,
                                    level, true);
    }
 
-   struct blorp_batch blorp_batch;
-   blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
-
    uint8_t stencil_mask = clear_stencil && stencil_res ? 0xff : 0;
    if (stencil_mask) {
       iris_resource_prepare_access(ice, batch, stencil_res, level, 1, box->z,
@@ -606,6 +610,11 @@ clear_depth_stencil(struct iris_context *ice,
                                    stencil_res->aux.usage, level, true);
    }
 
+   iris_batch_sync_region_start(batch);
+
+   struct blorp_batch blorp_batch;
+   blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
+
    blorp_clear_depth_stencil(&blorp_batch, &z_surf, &stencil_surf,
                              level, box->z, box->depth,
                              box->x, box->y,
@@ -615,6 +624,8 @@ clear_depth_stencil(struct iris_context *ice,
                              stencil_mask, stencil);
 
    blorp_batch_finish(&blorp_batch);
+   iris_batch_sync_region_end(batch);
+
    iris_flush_and_dirty_for_history(ice, batch, res, 0,
                                     "cache history: post slow ZS clear");
 
index 8888a8fd2842ddbb82f17013d8629a1881c2db01..aaed40e5e08f8c9931d759c50e22687a6f356327 100644 (file)
@@ -711,6 +711,8 @@ iris_get_query_result_resource(struct pipe_context *ctx,
    struct gen_mi_builder b;
    gen_mi_builder_init(&b, batch);
 
+   iris_batch_sync_region_start(batch);
+
    struct gen_mi_value result = calculate_result_on_gpu(devinfo, &b, q);
    struct gen_mi_value dst =
       result_type <= PIPE_QUERY_TYPE_U32 ? gen_mi_mem32(rw_bo(dst_bo, offset))
@@ -723,6 +725,8 @@ iris_get_query_result_resource(struct pipe_context *ctx,
    } else {
       gen_mi_store(&b, dst, result);
    }
+
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -763,6 +767,8 @@ set_predicate_for_result(struct iris_context *ice,
    struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
    struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res);
 
+   iris_batch_sync_region_start(batch);
+
    /* The CPU doesn't have the query result yet; use hardware predication */
    ice->state.predicate = IRIS_PREDICATE_STATE_USE_BIT;
 
@@ -809,6 +815,8 @@ set_predicate_for_result(struct iris_context *ice,
    gen_mi_store(&b, query_mem64(q, offsetof(struct iris_query_snapshots,
                                             predicate_result)), result);
    ice->state.compute_predicate = bo;
+
+   iris_batch_sync_region_end(batch);
 }
 
 static void
index f763f45f01e55fafc922d1305d3929b20f0d5a97..1b0c22fc09af1f0f46bc0c05e2df013c75149ca2 100644 (file)
@@ -499,6 +499,7 @@ iris_resolve_color(struct iris_context *ice,
    iris_emit_end_of_pipe_sync(batch, "color resolve: pre-flush",
                               PIPE_CONTROL_RENDER_TARGET_FLUSH);
 
+   iris_batch_sync_region_start(batch);
    struct blorp_batch blorp_batch;
    blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
    /* On Gen >= 12, Stencil buffer with lossless compression needs to be
@@ -517,6 +518,7 @@ iris_resolve_color(struct iris_context *ice,
    /* See comment above */
    iris_emit_end_of_pipe_sync(batch, "color resolve: post-flush",
                               PIPE_CONTROL_RENDER_TARGET_FLUSH);
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -536,11 +538,13 @@ iris_mcs_partial_resolve(struct iris_context *ice,
                                 &res->base, res->aux.usage, 0, true);
 
    struct blorp_batch blorp_batch;
+   iris_batch_sync_region_start(batch);
    blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
    blorp_mcs_partial_resolve(&blorp_batch, &surf,
                              isl_format_srgb_to_linear(res->surf.format),
                              start_layer, num_layers);
    blorp_batch_finish(&blorp_batch);
+   iris_batch_sync_region_end(batch);
 }
 
 
@@ -681,6 +685,8 @@ iris_hiz_exec(struct iris_context *ice,
 
    iris_batch_maybe_flush(batch, 1500);
 
+   iris_batch_sync_region_start(batch);
+
    struct blorp_surf surf;
    iris_blorp_surf_for_resource(&batch->screen->isl_dev, &surf,
                                 &res->base, res->aux.usage, level, true);
@@ -712,6 +718,8 @@ iris_hiz_exec(struct iris_context *ice,
                                 "hiz op: post flush",
                                 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
                                 PIPE_CONTROL_DEPTH_STALL);
+
+   iris_batch_sync_region_end(batch);
 }
 
 static bool
index bc3949385369207e2bad1f81a8f92be3b6055a5a..54291868e7d413a0231a73034af6b076fe94642d 100644 (file)
@@ -526,10 +526,12 @@ static void
 iris_load_register_mem32(struct iris_batch *batch, uint32_t reg,
                          struct iris_bo *bo, uint32_t offset)
 {
+   iris_batch_sync_region_start(batch);
    iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
       lrm.RegisterAddress = reg;
       lrm.MemoryAddress = ro_bo(bo, offset);
    }
+   iris_batch_sync_region_end(batch);
 }
 
 /**
@@ -549,11 +551,13 @@ iris_store_register_mem32(struct iris_batch *batch, uint32_t reg,
                           struct iris_bo *bo, uint32_t offset,
                           bool predicated)
 {
+   iris_batch_sync_region_start(batch);
    iris_emit_cmd(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
       srm.RegisterAddress = reg;
       srm.MemoryAddress = rw_bo(bo, offset);
       srm.PredicateEnable = predicated;
    }
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -570,10 +574,12 @@ iris_store_data_imm32(struct iris_batch *batch,
                       struct iris_bo *bo, uint32_t offset,
                       uint32_t imm)
 {
+   iris_batch_sync_region_start(batch);
    iris_emit_cmd(batch, GENX(MI_STORE_DATA_IMM), sdi) {
       sdi.Address = rw_bo(bo, offset);
       sdi.ImmediateData = imm;
    }
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -585,11 +591,13 @@ iris_store_data_imm64(struct iris_batch *batch,
     * 2 in genxml but it's actually variable length and we need 5 DWords.
     */
    void *map = iris_get_command_space(batch, 4 * 5);
+   iris_batch_sync_region_start(batch);
    _iris_pack_command(batch, GENX(MI_STORE_DATA_IMM), map, sdi) {
       sdi.DWordLength = 5 - 2;
       sdi.Address = rw_bo(bo, offset);
       sdi.ImmediateData = imm;
    }
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -602,6 +610,7 @@ iris_copy_mem_mem(struct iris_batch *batch,
    assert(bytes % 4 == 0);
    assert(dst_offset % 4 == 0);
    assert(src_offset % 4 == 0);
+   iris_batch_sync_region_start(batch);
 
    for (unsigned i = 0; i < bytes; i += 4) {
       iris_emit_cmd(batch, GENX(MI_COPY_MEM_MEM), cp) {
@@ -609,6 +618,8 @@ iris_copy_mem_mem(struct iris_batch *batch,
          cp.SourceMemoryAddress = ro_bo(src_bo, src_offset + i);
       }
    }
+
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -898,6 +909,8 @@ iris_init_render_context(struct iris_batch *batch)
    UNUSED const struct gen_device_info *devinfo = &batch->screen->devinfo;
    uint32_t reg_val;
 
+   iris_batch_sync_region_start(batch);
+
    emit_pipeline_select(batch, _3D);
 
    iris_emit_l3_config(batch, batch->screen->l3_config_3d);
@@ -1003,9 +1016,12 @@ iris_init_render_context(struct iris_batch *batch)
 
    iris_alloc_push_constants(batch);
 
+
 #if GEN_GEN >= 12
    init_aux_map_state(batch);
 #endif
+
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -1013,6 +1029,8 @@ iris_init_compute_context(struct iris_batch *batch)
 {
    UNUSED const struct gen_device_info *devinfo = &batch->screen->devinfo;
 
+   iris_batch_sync_region_start(batch);
+
    /* GEN:BUG:1607854226:
     *
     *  Start with pipeline in 3D mode to set the STATE_BASE_ADDRESS.
@@ -1040,6 +1058,7 @@ iris_init_compute_context(struct iris_batch *batch)
    init_aux_map_state(batch);
 #endif
 
+   iris_batch_sync_region_end(batch);
 }
 
 struct iris_vertex_buffer_state {
@@ -5079,6 +5098,8 @@ iris_update_surface_base_address(struct iris_batch *batch,
 
    uint32_t mocs = batch->screen->isl_dev.mocs.internal;
 
+   iris_batch_sync_region_start(batch);
+
    flush_before_state_base_change(batch);
 
 #if GEN_GEN == 12
@@ -5119,6 +5140,7 @@ iris_update_surface_base_address(struct iris_batch *batch,
 #endif
 
    flush_after_state_base_change(batch);
+   iris_batch_sync_region_end(batch);
 
    batch->last_surface_base_address = binder->bo->gtt_offset;
 }
@@ -6290,6 +6312,8 @@ iris_upload_render_state(struct iris_context *ice,
 {
    bool use_predicate = ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT;
 
+   iris_batch_sync_region_start(batch);
+
    /* Always pin the binder.  If we're emitting new binding table pointers,
     * we need it.  If not, we're probably inheriting old tables via the
     * context, and need it anyway.  Since true zero-bindings cases are
@@ -6496,6 +6520,8 @@ iris_upload_render_state(struct iris_context *ice,
          }
       }
    }
+
+   iris_batch_sync_region_end(batch);
 }
 
 static void
@@ -6518,6 +6544,8 @@ iris_upload_compute_state(struct iris_context *ice,
       brw_cs_simd_size_for_group_size(devinfo, cs_prog_data, group_size);
    const unsigned threads = DIV_ROUND_UP(group_size, simd_size);
 
+   iris_batch_sync_region_start(batch);
+
    /* Always pin the binder.  If we're emitting new binding table pointers,
     * we need it.  If not, we're probably inheriting old tables via the
     * context, and need it anyway.  Since true zero-bindings cases are
@@ -6676,6 +6704,8 @@ iris_upload_compute_state(struct iris_context *ice,
       iris_restore_compute_saved_bos(ice, batch, grid);
       batch->contains_draw = true;
    }
+
+   iris_batch_sync_region_end(batch);
 }
 
 /**
@@ -7297,6 +7327,8 @@ iris_emit_raw_pipe_control(struct iris_batch *batch,
               imm, reason);
    }
 
+   iris_batch_sync_region_start(batch);
+
    iris_emit_cmd(batch, GENX(PIPE_CONTROL), pc) {
 #if GEN_GEN >= 12
       pc.TileCacheFlushEnable = flags & PIPE_CONTROL_TILE_CACHE_FLUSH;
@@ -7334,6 +7366,8 @@ iris_emit_raw_pipe_control(struct iris_batch *batch,
       pc.Address = rw_bo(bo, offset);
       pc.ImmediateData = imm;
    }
+
+   iris_batch_sync_region_end(batch);
 }
 
 #if GEN_GEN == 9
@@ -7418,10 +7452,12 @@ iris_emit_mi_report_perf_count(struct iris_batch *batch,
                                uint32_t offset_in_bytes,
                                uint32_t report_id)
 {
+   iris_batch_sync_region_start(batch);
    iris_emit_cmd(batch, GENX(MI_REPORT_PERF_COUNT), mi_rpc) {
       mi_rpc.MemoryAddress = rw_bo(bo, offset_in_bytes);
       mi_rpc.ReportID = report_id;
    }
+   iris_batch_sync_region_end(batch);
 }
 
 /**