From eb5d1c27227302167d299bcaa2dabe623a242eb3 Mon Sep 17 00:00:00 2001 From: Francisco Jerez Date: Fri, 29 May 2020 16:38:43 -0700 Subject: [PATCH] iris: Annotate all BO uses with domain and sequence number information. Probably the most annoying patch to review from the whole series -- Mark every buffer object use as accessed through some caching domain with the sequence number of the current synchronization section of the batch. The additional argument of iris_use_pinned_bo() makes sure I'd have gotten a compile error if I had missed any buffer added to the batch validation list. There are only a few exceptions where a buffer is left untracked while adding it to the validation list, justified below: - Batch buffers: These are strictly read-only for the moment. - BLORP buffer objects: Their seqnos are bumped manually at the end of iris_blorp_exec() instead, in order to avoid plumbing domain information through BLORP address combining. - Scratch buffers: The contents of these are strictly thread-local. - Shader images and SSBOs: Accesses of these buffers are explicitly synchronized at the API level. v2: Opt out of tracking more aggressively (Ken): In addition to the above, surface states, binding tables, instructions and most dynamic states are now left untracked, which means a *lot* more BO uses marked IRIS_DOMAIN_NONE which need to be reviewed extremely carefully, since the cache tracker won't be able to provide any coherency guarantees for them. Reviewed-by: Kenneth Graunke Part-of: --- src/gallium/drivers/iris/iris_batch.c | 11 +- src/gallium/drivers/iris/iris_batch.h | 2 +- src/gallium/drivers/iris/iris_blorp.c | 20 +- src/gallium/drivers/iris/iris_genx_macros.h | 10 +- src/gallium/drivers/iris/iris_program_cache.c | 4 +- src/gallium/drivers/iris/iris_query.c | 8 +- src/gallium/drivers/iris/iris_screen.h | 1 + src/gallium/drivers/iris/iris_state.c | 197 +++++++++++------- 8 files changed, 161 insertions(+), 92 deletions(-) diff --git a/src/gallium/drivers/iris/iris_batch.c b/src/gallium/drivers/iris/iris_batch.c index e3be9c3df8a..a3455933471 100644 --- a/src/gallium/drivers/iris/iris_batch.c +++ b/src/gallium/drivers/iris/iris_batch.c @@ -271,7 +271,7 @@ ensure_exec_obj_space(struct iris_batch *batch, uint32_t count) void iris_use_pinned_bo(struct iris_batch *batch, struct iris_bo *bo, - bool writable) + bool writable, enum iris_domain access) { assert(bo->kflags & EXEC_OBJECT_PINNED); @@ -283,6 +283,11 @@ iris_use_pinned_bo(struct iris_batch *batch, if (bo == batch->screen->workaround_bo) writable = false; + if (access < NUM_IRIS_DOMAINS) { + assert(batch->sync_region_depth); + iris_bo_bump_seqno(bo, batch->next_seqno, access); + } + struct drm_i915_gem_exec_object2 *existing_entry = find_validation_entry(batch, bo); @@ -357,7 +362,7 @@ create_batch(struct iris_batch *batch) batch->map = iris_bo_map(NULL, batch->bo, MAP_READ | MAP_WRITE); batch->map_next = batch->map; - iris_use_pinned_bo(batch, batch->bo, false); + iris_use_pinned_bo(batch, batch->bo, false, IRIS_DOMAIN_NONE); } static void @@ -404,7 +409,7 @@ iris_batch_reset(struct iris_batch *batch) /* Always add the workaround BO, it contains a driver identifier at the * beginning quite helpful to debug error states. */ - iris_use_pinned_bo(batch, screen->workaround_bo, false); + iris_use_pinned_bo(batch, screen->workaround_bo, false, IRIS_DOMAIN_NONE); iris_batch_maybe_noop(batch); } diff --git a/src/gallium/drivers/iris/iris_batch.h b/src/gallium/drivers/iris/iris_batch.h index 50de99289cf..bfea20d268c 100644 --- a/src/gallium/drivers/iris/iris_batch.h +++ b/src/gallium/drivers/iris/iris_batch.h @@ -179,7 +179,7 @@ bool iris_batch_prepare_noop(struct iris_batch *batch, bool noop_enable); #define RELOC_WRITE EXEC_OBJECT_WRITE void iris_use_pinned_bo(struct iris_batch *batch, struct iris_bo *bo, - bool writable); + bool writable, enum iris_domain access); enum pipe_reset_status iris_batch_check_for_reset(struct iris_batch *batch); diff --git a/src/gallium/drivers/iris/iris_blorp.c b/src/gallium/drivers/iris/iris_blorp.c index 1da234d9467..a9d1e89cd72 100644 --- a/src/gallium/drivers/iris/iris_blorp.c +++ b/src/gallium/drivers/iris/iris_blorp.c @@ -60,7 +60,7 @@ stream_state(struct iris_batch *batch, u_upload_alloc(uploader, 0, size, alignment, out_offset, &res, &ptr); struct iris_bo *bo = iris_resource_bo(res); - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); iris_record_state_size(batch->state_sizes, bo->gtt_offset + *out_offset, size); @@ -93,7 +93,8 @@ combine_and_pin_address(struct blorp_batch *blorp_batch, struct iris_batch *batch = blorp_batch->driver_batch; struct iris_bo *bo = addr.buffer; - iris_use_pinned_bo(batch, bo, addr.reloc_flags & RELOC_WRITE); + iris_use_pinned_bo(batch, bo, addr.reloc_flags & RELOC_WRITE, + IRIS_DOMAIN_NONE); /* Assume this is a general address, not relative to a base. */ return bo->gtt_offset + addr.offset; @@ -162,7 +163,7 @@ blorp_alloc_binding_table(struct blorp_batch *blorp_batch, bt_map[i] = surface_offsets[i] - (uint32_t) binder->bo->gtt_offset; } - iris_use_pinned_bo(batch, binder->bo, false); + iris_use_pinned_bo(batch, binder->bo, false, IRIS_DOMAIN_NONE); batch->screen->vtbl.update_surface_base_address(batch, binder); } @@ -375,6 +376,19 @@ iris_blorp_exec(struct blorp_batch *blorp_batch, iris_depth_cache_add_bo(batch, params->depth.addr.buffer); if (params->stencil.enabled) iris_depth_cache_add_bo(batch, params->stencil.addr.buffer); + + if (params->src.enabled) + iris_bo_bump_seqno(params->src.addr.buffer, batch->next_seqno, + IRIS_DOMAIN_OTHER_READ); + if (params->dst.enabled) + iris_bo_bump_seqno(params->dst.addr.buffer, batch->next_seqno, + IRIS_DOMAIN_RENDER_WRITE); + if (params->depth.enabled) + iris_bo_bump_seqno(params->depth.addr.buffer, batch->next_seqno, + IRIS_DOMAIN_DEPTH_WRITE); + if (params->stencil.enabled) + iris_bo_bump_seqno(params->stencil.addr.buffer, batch->next_seqno, + IRIS_DOMAIN_DEPTH_WRITE); } void diff --git a/src/gallium/drivers/iris/iris_genx_macros.h b/src/gallium/drivers/iris/iris_genx_macros.h index 74fe9139add..9312175d9b1 100644 --- a/src/gallium/drivers/iris/iris_genx_macros.h +++ b/src/gallium/drivers/iris/iris_genx_macros.h @@ -52,7 +52,7 @@ __gen_combine_address(struct iris_batch *batch, void *location, uint64_t result = addr.offset + delta; if (addr.bo) { - iris_use_pinned_bo(batch, addr.bo, addr.write); + iris_use_pinned_bo(batch, addr.bo, addr.write, addr.access); /* Assume this is a general address, not relative to a base. */ result += addr.bo->gtt_offset; } @@ -121,11 +121,13 @@ __gen_combine_address(struct iris_batch *batch, void *location, UNUSED static struct iris_address ro_bo(struct iris_bo *bo, uint64_t offset) { - return (struct iris_address) { .bo = bo, .offset = offset }; + return (struct iris_address) { .bo = bo, .offset = offset, + .access = IRIS_DOMAIN_OTHER_READ }; } UNUSED static struct iris_address -rw_bo(struct iris_bo *bo, uint64_t offset) +rw_bo(struct iris_bo *bo, uint64_t offset, enum iris_domain access) { - return (struct iris_address) { .bo = bo, .offset = offset, .write = true }; + return (struct iris_address) { .bo = bo, .offset = offset, + .write = true, .access = access }; } diff --git a/src/gallium/drivers/iris/iris_program_cache.c b/src/gallium/drivers/iris/iris_program_cache.c index 62393cacf5b..aa0d5c29c37 100644 --- a/src/gallium/drivers/iris/iris_program_cache.c +++ b/src/gallium/drivers/iris/iris_program_cache.c @@ -216,7 +216,7 @@ iris_blorp_lookup_shader(struct blorp_batch *blorp_batch, iris_bo_offset_from_base_address(bo) + shader->assembly.offset; *((void **) prog_data_out) = shader->prog_data; - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); return true; } @@ -248,7 +248,7 @@ iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage, iris_bo_offset_from_base_address(bo) + shader->assembly.offset; *((void **) prog_data_out) = shader->prog_data; - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); return true; } diff --git a/src/gallium/drivers/iris/iris_query.c b/src/gallium/drivers/iris/iris_query.c index aaed40e5e08..7f93d0f7ec7 100644 --- a/src/gallium/drivers/iris/iris_query.c +++ b/src/gallium/drivers/iris/iris_query.c @@ -101,7 +101,8 @@ query_mem64(struct iris_query *q, uint32_t offset) struct iris_address addr = { .bo = iris_resource_bo(q->query_state_ref.res), .offset = q->query_state_ref.offset + offset, - .write = true + .write = true, + .access = IRIS_DOMAIN_OTHER_WRITE }; return gen_mi_mem64(addr); } @@ -715,8 +716,9 @@ iris_get_query_result_resource(struct pipe_context *ctx, struct gen_mi_value result = calculate_result_on_gpu(devinfo, &b, q); struct gen_mi_value dst = - result_type <= PIPE_QUERY_TYPE_U32 ? gen_mi_mem32(rw_bo(dst_bo, offset)) - : gen_mi_mem64(rw_bo(dst_bo, offset)); + result_type <= PIPE_QUERY_TYPE_U32 ? + gen_mi_mem32(rw_bo(dst_bo, offset, IRIS_DOMAIN_OTHER_WRITE)) : + gen_mi_mem64(rw_bo(dst_bo, offset, IRIS_DOMAIN_OTHER_WRITE)); if (predicated) { gen_mi_store(&b, gen_mi_reg32(MI_PREDICATE_RESULT), diff --git a/src/gallium/drivers/iris/iris_screen.h b/src/gallium/drivers/iris/iris_screen.h index b536e10642b..4d300e302e9 100644 --- a/src/gallium/drivers/iris/iris_screen.h +++ b/src/gallium/drivers/iris/iris_screen.h @@ -137,6 +137,7 @@ struct iris_vtable { }; struct iris_address { + enum iris_domain access; struct iris_bo *bo; uint64_t offset; bool write; diff --git a/src/gallium/drivers/iris/iris_state.c b/src/gallium/drivers/iris/iris_state.c index 54291868e7d..786a188746e 100644 --- a/src/gallium/drivers/iris/iris_state.c +++ b/src/gallium/drivers/iris/iris_state.c @@ -339,7 +339,7 @@ stream_state(struct iris_batch *batch, u_upload_alloc(uploader, 0, size, alignment, out_offset, out_res, &ptr); struct iris_bo *bo = iris_resource_bo(*out_res); - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); iris_record_state_size(batch->state_sizes, bo->gtt_offset + *out_offset, size); @@ -554,7 +554,7 @@ iris_store_register_mem32(struct iris_batch *batch, uint32_t reg, iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_STORE_REGISTER_MEM), srm) { srm.RegisterAddress = reg; - srm.MemoryAddress = rw_bo(bo, offset); + srm.MemoryAddress = rw_bo(bo, offset, IRIS_DOMAIN_OTHER_WRITE); srm.PredicateEnable = predicated; } iris_batch_sync_region_end(batch); @@ -576,7 +576,7 @@ iris_store_data_imm32(struct iris_batch *batch, { iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_STORE_DATA_IMM), sdi) { - sdi.Address = rw_bo(bo, offset); + sdi.Address = rw_bo(bo, offset, IRIS_DOMAIN_OTHER_WRITE); sdi.ImmediateData = imm; } iris_batch_sync_region_end(batch); @@ -594,7 +594,7 @@ iris_store_data_imm64(struct iris_batch *batch, iris_batch_sync_region_start(batch); _iris_pack_command(batch, GENX(MI_STORE_DATA_IMM), map, sdi) { sdi.DWordLength = 5 - 2; - sdi.Address = rw_bo(bo, offset); + sdi.Address = rw_bo(bo, offset, IRIS_DOMAIN_OTHER_WRITE); sdi.ImmediateData = imm; } iris_batch_sync_region_end(batch); @@ -614,7 +614,8 @@ iris_copy_mem_mem(struct iris_batch *batch, for (unsigned i = 0; i < bytes; i += 4) { iris_emit_cmd(batch, GENX(MI_COPY_MEM_MEM), cp) { - cp.DestinationMemoryAddress = rw_bo(dst_bo, dst_offset + i); + cp.DestinationMemoryAddress = rw_bo(dst_bo, dst_offset + i, + IRIS_DOMAIN_OTHER_WRITE); cp.SourceMemoryAddress = ro_bo(src_bo, src_offset + i); } } @@ -3664,7 +3665,8 @@ iris_set_stream_output_targets(struct pipe_context *ctx, sob._3DCommandSubOpcode = SO_BUFFER_INDEX_0_CMD + i; #endif sob.SurfaceBaseAddress = - rw_bo(NULL, res->bo->gtt_offset + tgt->base.buffer_offset); + rw_bo(NULL, res->bo->gtt_offset + tgt->base.buffer_offset, + IRIS_DOMAIN_OTHER_WRITE); sob.SOBufferEnable = true; sob.StreamOffsetWriteEnable = true; sob.StreamOutputBufferOffsetAddressEnable = true; @@ -3674,7 +3676,7 @@ iris_set_stream_output_targets(struct pipe_context *ctx, sob.StreamOffset = offset; sob.StreamOutputBufferOffsetAddress = rw_bo(NULL, iris_resource_bo(tgt->offset.res)->gtt_offset + - tgt->offset.offset); + tgt->offset.offset, IRIS_DOMAIN_OTHER_WRITE); } } @@ -4164,7 +4166,8 @@ KSP(const struct iris_compiled_shader *shader) iris_get_scratch_space(ice, prog_data->total_scratch, stage); \ uint32_t scratch_addr = bo->gtt_offset; \ pkt.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11; \ - pkt.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr); \ + pkt.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr, \ + IRIS_DOMAIN_NONE); \ } /** @@ -4362,7 +4365,8 @@ iris_store_fs_state(struct iris_context *ice, MESA_SHADER_FRAGMENT); uint32_t scratch_addr = bo->gtt_offset; ps.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11; - ps.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr); + ps.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr, + IRIS_DOMAIN_NONE); } } @@ -4493,7 +4497,7 @@ use_null_surface(struct iris_batch *batch, struct iris_context *ice) { struct iris_bo *state_bo = iris_resource_bo(ice->state.unbound_tex.res); - iris_use_pinned_bo(batch, state_bo, false); + iris_use_pinned_bo(batch, state_bo, false, IRIS_DOMAIN_NONE); return ice->state.unbound_tex.offset; } @@ -4507,7 +4511,7 @@ use_null_fb_surface(struct iris_batch *batch, struct iris_context *ice) struct iris_bo *state_bo = iris_resource_bo(ice->state.null_fb.res); - iris_use_pinned_bo(batch, state_bo, false); + iris_use_pinned_bo(batch, state_bo, false, IRIS_DOMAIN_NONE); return ice->state.null_fb.offset; } @@ -4617,23 +4621,28 @@ use_surface(struct iris_context *ice, struct pipe_surface *p_surf, bool writeable, enum isl_aux_usage aux_usage, - bool is_read_surface) + bool is_read_surface, + enum iris_domain access) { struct iris_surface *surf = (void *) p_surf; struct iris_resource *res = (void *) p_surf->texture; uint32_t offset = 0; - iris_use_pinned_bo(batch, iris_resource_bo(p_surf->texture), writeable); + iris_use_pinned_bo(batch, iris_resource_bo(p_surf->texture), + writeable, access); if (GEN_GEN == 8 && is_read_surface) { - iris_use_pinned_bo(batch, iris_resource_bo(surf->surface_state_read.ref.res), false); + iris_use_pinned_bo(batch, iris_resource_bo(surf->surface_state_read.ref.res), false, + IRIS_DOMAIN_NONE); } else { - iris_use_pinned_bo(batch, iris_resource_bo(surf->surface_state.ref.res), false); + iris_use_pinned_bo(batch, iris_resource_bo(surf->surface_state.ref.res), false, + IRIS_DOMAIN_NONE); } if (res->aux.bo) { - iris_use_pinned_bo(batch, res->aux.bo, writeable); + iris_use_pinned_bo(batch, res->aux.bo, writeable, access); if (res->aux.clear_color_bo) - iris_use_pinned_bo(batch, res->aux.clear_color_bo, false); + iris_use_pinned_bo(batch, res->aux.clear_color_bo, + false, IRIS_DOMAIN_OTHER_READ); if (memcmp(&res->aux.clear_color, &surf->clear_color, sizeof(surf->clear_color)) != 0) { @@ -4663,13 +4672,16 @@ use_sampler_view(struct iris_context *ice, enum isl_aux_usage aux_usage = iris_resource_texture_aux_usage(ice, isv->res, isv->view.format); - iris_use_pinned_bo(batch, isv->res->bo, false); - iris_use_pinned_bo(batch, iris_resource_bo(isv->surface_state.ref.res), false); + iris_use_pinned_bo(batch, isv->res->bo, false, IRIS_DOMAIN_OTHER_READ); + iris_use_pinned_bo(batch, iris_resource_bo(isv->surface_state.ref.res), false, + IRIS_DOMAIN_NONE); if (isv->res->aux.bo) { - iris_use_pinned_bo(batch, isv->res->aux.bo, false); + iris_use_pinned_bo(batch, isv->res->aux.bo, + false, IRIS_DOMAIN_OTHER_READ); if (isv->res->aux.clear_color_bo) - iris_use_pinned_bo(batch, isv->res->aux.clear_color_bo, false); + iris_use_pinned_bo(batch, isv->res->aux.clear_color_bo, + false, IRIS_DOMAIN_OTHER_READ); if (memcmp(&isv->res->aux.clear_color, &isv->clear_color, sizeof(isv->clear_color)) != 0) { update_clear_value(ice, batch, isv->res, &isv->surface_state, @@ -4688,13 +4700,14 @@ use_ubo_ssbo(struct iris_batch *batch, struct iris_context *ice, struct pipe_shader_buffer *buf, struct iris_state_ref *surf_state, - bool writable) + bool writable, enum iris_domain access) { if (!buf->buffer || !surf_state->res) return use_null_surface(batch, ice); - iris_use_pinned_bo(batch, iris_resource_bo(buf->buffer), writable); - iris_use_pinned_bo(batch, iris_resource_bo(surf_state->res), false); + iris_use_pinned_bo(batch, iris_resource_bo(buf->buffer), writable, access); + iris_use_pinned_bo(batch, iris_resource_bo(surf_state->res), false, + IRIS_DOMAIN_NONE); return surf_state->offset; } @@ -4712,11 +4725,12 @@ use_image(struct iris_batch *batch, struct iris_context *ice, bool write = iv->base.shader_access & PIPE_IMAGE_ACCESS_WRITE; - iris_use_pinned_bo(batch, res->bo, write); - iris_use_pinned_bo(batch, iris_resource_bo(iv->surface_state.ref.res), false); + iris_use_pinned_bo(batch, res->bo, write, IRIS_DOMAIN_NONE); + iris_use_pinned_bo(batch, iris_resource_bo(iv->surface_state.ref.res), + false, IRIS_DOMAIN_NONE); if (res->aux.bo) - iris_use_pinned_bo(batch, res->aux.bo, write); + iris_use_pinned_bo(batch, res->aux.bo, write, IRIS_DOMAIN_NONE); enum isl_aux_usage aux_usage = iris_image_view_aux_usage(ice, &iv->base, info); @@ -4773,8 +4787,10 @@ iris_populate_binding_table(struct iris_context *ice, /* surface for gl_NumWorkGroups */ struct iris_state_ref *grid_data = &ice->state.grid_size; struct iris_state_ref *grid_state = &ice->state.grid_surf_state; - iris_use_pinned_bo(batch, iris_resource_bo(grid_data->res), false); - iris_use_pinned_bo(batch, iris_resource_bo(grid_state->res), false); + iris_use_pinned_bo(batch, iris_resource_bo(grid_data->res), false, + IRIS_DOMAIN_OTHER_READ); + iris_use_pinned_bo(batch, iris_resource_bo(grid_state->res), false, + IRIS_DOMAIN_NONE); push_bt_entry(grid_state->offset); } @@ -4786,7 +4802,8 @@ iris_populate_binding_table(struct iris_context *ice, uint32_t addr; if (cso_fb->cbufs[i]) { addr = use_surface(ice, batch, cso_fb->cbufs[i], true, - ice->state.draw_aux_usage[i], false); + ice->state.draw_aux_usage[i], false, + IRIS_DOMAIN_RENDER_WRITE); } else { addr = use_null_fb_surface(batch, ice); } @@ -4809,7 +4826,8 @@ iris_populate_binding_table(struct iris_context *ice, uint32_t addr; if (cso_fb->cbufs[i]) { addr = use_surface(ice, batch, cso_fb->cbufs[i], - true, ice->state.draw_aux_usage[i], true); + false, ice->state.draw_aux_usage[i], true, + IRIS_DOMAIN_OTHER_READ); push_bt_entry(addr); } } @@ -4831,9 +4849,10 @@ iris_populate_binding_table(struct iris_context *ice, if (i == bt->sizes[IRIS_SURFACE_GROUP_UBO] - 1) { if (ish->const_data) { - iris_use_pinned_bo(batch, iris_resource_bo(ish->const_data), false); + iris_use_pinned_bo(batch, iris_resource_bo(ish->const_data), false, + IRIS_DOMAIN_OTHER_READ); iris_use_pinned_bo(batch, iris_resource_bo(ish->const_data_state.res), - false); + false, IRIS_DOMAIN_NONE); addr = ish->const_data_state.offset; } else { /* This can only happen with INTEL_DISABLE_COMPACT_BINDING_TABLE=1. */ @@ -4841,7 +4860,8 @@ iris_populate_binding_table(struct iris_context *ice, } } else { addr = use_ubo_ssbo(batch, ice, &shs->constbuf[i], - &shs->constbuf_surf_state[i], false); + &shs->constbuf_surf_state[i], false, + IRIS_DOMAIN_OTHER_READ); } push_bt_entry(addr); @@ -4850,7 +4870,7 @@ iris_populate_binding_table(struct iris_context *ice, foreach_surface_used(i, IRIS_SURFACE_GROUP_SSBO) { uint32_t addr = use_ubo_ssbo(batch, ice, &shs->ssbo[i], &shs->ssbo_surf_state[i], - shs->writable_ssbos & (1u << i)); + shs->writable_ssbos & (1u << i), IRIS_DOMAIN_NONE); push_bt_entry(addr); } @@ -4864,11 +4884,12 @@ iris_populate_binding_table(struct iris_context *ice, static void iris_use_optional_res(struct iris_batch *batch, struct pipe_resource *res, - bool writeable) + bool writeable, + enum iris_domain access) { if (res) { struct iris_bo *bo = iris_resource_bo(res); - iris_use_pinned_bo(batch, bo, writeable); + iris_use_pinned_bo(batch, bo, writeable, access); } } @@ -4884,15 +4905,21 @@ pin_depth_and_stencil_buffers(struct iris_batch *batch, iris_get_depth_stencil_resources(zsbuf->texture, &zres, &sres); if (zres) { - iris_use_pinned_bo(batch, zres->bo, cso_zsa->depth_writes_enabled); + const enum iris_domain access = cso_zsa->depth_writes_enabled ? + IRIS_DOMAIN_DEPTH_WRITE : IRIS_DOMAIN_OTHER_READ; + iris_use_pinned_bo(batch, zres->bo, cso_zsa->depth_writes_enabled, + access); if (zres->aux.bo) { iris_use_pinned_bo(batch, zres->aux.bo, - cso_zsa->depth_writes_enabled); + cso_zsa->depth_writes_enabled, access); } } if (sres) { - iris_use_pinned_bo(batch, sres->bo, cso_zsa->stencil_writes_enabled); + const enum iris_domain access = cso_zsa->stencil_writes_enabled ? + IRIS_DOMAIN_DEPTH_WRITE : IRIS_DOMAIN_OTHER_READ; + iris_use_pinned_bo(batch, sres->bo, cso_zsa->stencil_writes_enabled, + access); } } @@ -4922,23 +4949,28 @@ iris_restore_render_saved_bos(struct iris_context *ice, const uint64_t stage_clean = ~ice->state.stage_dirty; if (clean & IRIS_DIRTY_CC_VIEWPORT) { - iris_use_optional_res(batch, ice->state.last_res.cc_vp, false); + iris_use_optional_res(batch, ice->state.last_res.cc_vp, false, + IRIS_DOMAIN_NONE); } if (clean & IRIS_DIRTY_SF_CL_VIEWPORT) { - iris_use_optional_res(batch, ice->state.last_res.sf_cl_vp, false); + iris_use_optional_res(batch, ice->state.last_res.sf_cl_vp, false, + IRIS_DOMAIN_NONE); } if (clean & IRIS_DIRTY_BLEND_STATE) { - iris_use_optional_res(batch, ice->state.last_res.blend, false); + iris_use_optional_res(batch, ice->state.last_res.blend, false, + IRIS_DOMAIN_NONE); } if (clean & IRIS_DIRTY_COLOR_CALC_STATE) { - iris_use_optional_res(batch, ice->state.last_res.color_calc, false); + iris_use_optional_res(batch, ice->state.last_res.color_calc, false, + IRIS_DOMAIN_NONE); } if (clean & IRIS_DIRTY_SCISSOR_RECT) { - iris_use_optional_res(batch, ice->state.last_res.scissor, false); + iris_use_optional_res(batch, ice->state.last_res.scissor, false, + IRIS_DOMAIN_NONE); } if (ice->state.streamout_active && (clean & IRIS_DIRTY_SO_BUFFERS)) { @@ -4947,9 +4979,9 @@ iris_restore_render_saved_bos(struct iris_context *ice, (void *) ice->state.so_target[i]; if (tgt) { iris_use_pinned_bo(batch, iris_resource_bo(tgt->base.buffer), - true); + true, IRIS_DOMAIN_OTHER_WRITE); iris_use_pinned_bo(batch, iris_resource_bo(tgt->offset.res), - true); + true, IRIS_DOMAIN_OTHER_WRITE); } } } @@ -4981,9 +5013,10 @@ iris_restore_render_saved_bos(struct iris_context *ice, struct iris_resource *res = (void *) cbuf->buffer; if (res) - iris_use_pinned_bo(batch, res->bo, false); + iris_use_pinned_bo(batch, res->bo, false, IRIS_DOMAIN_OTHER_READ); else - iris_use_pinned_bo(batch, batch->screen->workaround_bo, false); + iris_use_pinned_bo(batch, batch->screen->workaround_bo, false, + IRIS_DOMAIN_OTHER_READ); } } @@ -4998,7 +5031,8 @@ iris_restore_render_saved_bos(struct iris_context *ice, struct iris_shader_state *shs = &ice->state.shaders[stage]; struct pipe_resource *res = shs->sampler_table.res; if (res) - iris_use_pinned_bo(batch, iris_resource_bo(res), false); + iris_use_pinned_bo(batch, iris_resource_bo(res), false, + IRIS_DOMAIN_NONE); } for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) { @@ -5007,14 +5041,14 @@ iris_restore_render_saved_bos(struct iris_context *ice, if (shader) { struct iris_bo *bo = iris_resource_bo(shader->assembly.res); - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); struct brw_stage_prog_data *prog_data = shader->prog_data; if (prog_data->total_scratch > 0) { struct iris_bo *bo = iris_get_scratch_space(ice, prog_data->total_scratch, stage); - iris_use_pinned_bo(batch, bo, true); + iris_use_pinned_bo(batch, bo, true, IRIS_DOMAIN_NONE); } } } @@ -5026,14 +5060,16 @@ iris_restore_render_saved_bos(struct iris_context *ice, pin_depth_and_stencil_buffers(batch, cso_fb->zsbuf, ice->state.cso_zsa); } - iris_use_optional_res(batch, ice->state.last_res.index_buffer, false); + iris_use_optional_res(batch, ice->state.last_res.index_buffer, false, + IRIS_DOMAIN_OTHER_READ); if (clean & IRIS_DIRTY_VERTEX_BUFFERS) { uint64_t bound = ice->state.bound_vertex_buffers; while (bound) { const int i = u_bit_scan64(&bound); struct pipe_resource *res = genx->vertex_buffers[i].resource; - iris_use_pinned_bo(batch, iris_resource_bo(res), false); + iris_use_pinned_bo(batch, iris_resource_bo(res), false, + IRIS_DOMAIN_OTHER_READ); } } } @@ -5055,13 +5091,15 @@ iris_restore_compute_saved_bos(struct iris_context *ice, struct pipe_resource *sampler_res = shs->sampler_table.res; if (sampler_res) - iris_use_pinned_bo(batch, iris_resource_bo(sampler_res), false); + iris_use_pinned_bo(batch, iris_resource_bo(sampler_res), false, + IRIS_DOMAIN_NONE); if ((stage_clean & IRIS_STAGE_DIRTY_SAMPLER_STATES_CS) && (stage_clean & IRIS_STAGE_DIRTY_BINDINGS_CS) && (stage_clean & IRIS_STAGE_DIRTY_CONSTANTS_CS) && (stage_clean & IRIS_STAGE_DIRTY_CS)) { - iris_use_optional_res(batch, ice->state.last_res.cs_desc, false); + iris_use_optional_res(batch, ice->state.last_res.cs_desc, false, + IRIS_DOMAIN_NONE); } if (stage_clean & IRIS_STAGE_DIRTY_CS) { @@ -5069,18 +5107,18 @@ iris_restore_compute_saved_bos(struct iris_context *ice, if (shader) { struct iris_bo *bo = iris_resource_bo(shader->assembly.res); - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE); struct iris_bo *curbe_bo = iris_resource_bo(ice->state.last_res.cs_thread_ids); - iris_use_pinned_bo(batch, curbe_bo, false); + iris_use_pinned_bo(batch, curbe_bo, false, IRIS_DOMAIN_NONE); struct brw_stage_prog_data *prog_data = shader->prog_data; if (prog_data->total_scratch > 0) { struct iris_bo *bo = iris_get_scratch_space(ice, prog_data->total_scratch, stage); - iris_use_pinned_bo(batch, bo, true); + iris_use_pinned_bo(batch, bo, true, IRIS_DOMAIN_NONE); } } } @@ -5645,7 +5683,8 @@ iris_upload_dirty_render_state(struct iris_context *ice, struct iris_shader_state *shs = &ice->state.shaders[stage]; struct pipe_resource *res = shs->sampler_table.res; if (res) - iris_use_pinned_bo(batch, iris_resource_bo(res), false); + iris_use_pinned_bo(batch, iris_resource_bo(res), false, + IRIS_DOMAIN_NONE); iris_emit_cmd(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS_VS), ptr) { ptr._3DCommandSubOpcode = 43 + stage; @@ -5654,7 +5693,8 @@ iris_upload_dirty_render_state(struct iris_context *ice, } if (ice->state.need_border_colors) - iris_use_pinned_bo(batch, ice->state.border_color_pool.bo, false); + iris_use_pinned_bo(batch, ice->state.border_color_pool.bo, false, + IRIS_DOMAIN_NONE); if (dirty & IRIS_DIRTY_MULTISAMPLE) { iris_emit_cmd(batch, GENX(3DSTATE_MULTISAMPLE), ms) { @@ -5680,12 +5720,12 @@ iris_upload_dirty_render_state(struct iris_context *ice, if (shader) { struct brw_stage_prog_data *prog_data = shader->prog_data; struct iris_resource *cache = (void *) shader->assembly.res; - iris_use_pinned_bo(batch, cache->bo, false); + iris_use_pinned_bo(batch, cache->bo, false, IRIS_DOMAIN_NONE); if (prog_data->total_scratch > 0) { struct iris_bo *bo = iris_get_scratch_space(ice, prog_data->total_scratch, stage); - iris_use_pinned_bo(batch, bo, true); + iris_use_pinned_bo(batch, bo, true, IRIS_DOMAIN_NONE); } if (stage == MESA_SHADER_FRAGMENT) { @@ -5776,9 +5816,9 @@ iris_upload_dirty_render_state(struct iris_context *ice, if (tgt) { tgt->zeroed = true; iris_use_pinned_bo(batch, iris_resource_bo(tgt->base.buffer), - true); + true, IRIS_DOMAIN_OTHER_WRITE); iris_use_pinned_bo(batch, iris_resource_bo(tgt->offset.res), - true); + true, IRIS_DOMAIN_OTHER_WRITE); } } } @@ -6105,7 +6145,7 @@ iris_upload_dirty_render_state(struct iris_context *ice, while (bound) { const int i = u_bit_scan64(&bound); iris_use_optional_res(batch, genx->vertex_buffers[i].resource, - false); + false, IRIS_DOMAIN_OTHER_READ); } #else /* The VF cache designers cut corners, and made the cache key's @@ -6127,7 +6167,7 @@ iris_upload_dirty_render_state(struct iris_context *ice, struct iris_resource *res = (void *) genx->vertex_buffers[i].resource; if (res) { - iris_use_pinned_bo(batch, res->bo, false); + iris_use_pinned_bo(batch, res->bo, false, IRIS_DOMAIN_OTHER_READ); high_bits = res->bo->gtt_offset >> 32ull; if (high_bits != ice->state.last_vbo_high_bits[i]) { @@ -6319,7 +6359,8 @@ iris_upload_render_state(struct iris_context *ice, * context, and need it anyway. Since true zero-bindings cases are * practically non-existent, just pin it and avoid last_res tracking. */ - iris_use_pinned_bo(batch, ice->state.binder.bo, false); + iris_use_pinned_bo(batch, ice->state.binder.bo, false, + IRIS_DOMAIN_NONE); if (!batch->contains_draw) { iris_restore_render_saved_bos(ice, batch, draw); @@ -6358,7 +6399,7 @@ iris_upload_render_state(struct iris_context *ice, if (memcmp(genx->last_index_buffer, ib_packet, sizeof(ib_packet)) != 0) { memcpy(genx->last_index_buffer, ib_packet, sizeof(ib_packet)); iris_batch_emit(batch, ib_packet, sizeof(ib_packet)); - iris_use_pinned_bo(batch, bo, false); + iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_OTHER_READ); } #if GEN_GEN < 11 @@ -6551,7 +6592,7 @@ iris_upload_compute_state(struct iris_context *ice, * context, and need it anyway. Since true zero-bindings cases are * practically non-existent, just pin it and avoid last_res tracking. */ - iris_use_pinned_bo(batch, ice->state.binder.bo, false); + iris_use_pinned_bo(batch, ice->state.binder.bo, false, IRIS_DOMAIN_NONE); if ((stage_dirty & IRIS_STAGE_DIRTY_CONSTANTS_CS) && shs->sysvals_need_upload) @@ -6563,11 +6604,14 @@ iris_upload_compute_state(struct iris_context *ice, if (stage_dirty & IRIS_STAGE_DIRTY_SAMPLER_STATES_CS) iris_upload_sampler_states(ice, MESA_SHADER_COMPUTE); - iris_use_optional_res(batch, shs->sampler_table.res, false); - iris_use_pinned_bo(batch, iris_resource_bo(shader->assembly.res), false); + iris_use_optional_res(batch, shs->sampler_table.res, false, + IRIS_DOMAIN_NONE); + iris_use_pinned_bo(batch, iris_resource_bo(shader->assembly.res), false, + IRIS_DOMAIN_NONE); if (ice->state.need_border_colors) - iris_use_pinned_bo(batch, ice->state.border_color_pool.bo, false); + iris_use_pinned_bo(batch, ice->state.border_color_pool.bo, false, + IRIS_DOMAIN_NONE); #if GEN_GEN >= 12 genX(invalidate_aux_map_state)(batch); @@ -6592,7 +6636,7 @@ iris_upload_compute_state(struct iris_context *ice, iris_get_scratch_space(ice, prog_data->total_scratch, MESA_SHADER_COMPUTE); vfe.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11; - vfe.ScratchSpaceBasePointer = rw_bo(bo, 0); + vfe.ScratchSpaceBasePointer = rw_bo(bo, 0, IRIS_DOMAIN_NONE); } vfe.MaximumNumberofThreads = @@ -7363,7 +7407,7 @@ iris_emit_raw_pipe_control(struct iris_batch *batch, flags & PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE; pc.TextureCacheInvalidationEnable = flags & PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; - pc.Address = rw_bo(bo, offset); + pc.Address = rw_bo(bo, offset, IRIS_DOMAIN_OTHER_WRITE); pc.ImmediateData = imm; } @@ -7454,7 +7498,8 @@ iris_emit_mi_report_perf_count(struct iris_batch *batch, { iris_batch_sync_region_start(batch); iris_emit_cmd(batch, GENX(MI_REPORT_PERF_COUNT), mi_rpc) { - mi_rpc.MemoryAddress = rw_bo(bo, offset_in_bytes); + mi_rpc.MemoryAddress = rw_bo(bo, offset_in_bytes, + IRIS_DOMAIN_OTHER_WRITE); mi_rpc.ReportID = report_id; } iris_batch_sync_region_end(batch); -- 2.30.2