X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Firis%2Firis_resource.c;h=e0811827692b5cc84a8d3d697f13512ac4a15cac;hb=82b46667836647226387442b2feb9d7f1475bd36;hp=5b1fde7aac65a115b16d0cf005e8adf286c5829e;hpb=410894c643274975aab781e9c412f7a339a7547b;p=mesa.git diff --git a/src/gallium/drivers/iris/iris_resource.c b/src/gallium/drivers/iris/iris_resource.c index 5b1fde7aac6..e0811827692 100644 --- a/src/gallium/drivers/iris/iris_resource.c +++ b/src/gallium/drivers/iris/iris_resource.c @@ -37,7 +37,7 @@ #include "util/os_memory.h" #include "util/u_cpu_detect.h" #include "util/u_inlines.h" -#include "util/u_format.h" +#include "util/format/u_format.h" #include "util/u_threaded_context.h" #include "util/u_transfer.h" #include "util/u_transfer_helper.h" @@ -47,6 +47,7 @@ #include "iris_context.h" #include "iris_resource.h" #include "iris_screen.h" +#include "intel/common/gen_aux_map.h" #include "intel/dev/gen_debug.h" #include "isl/isl.h" #include "drm-uapi/drm_fourcc.h" @@ -58,6 +59,7 @@ enum modifier_priority { MODIFIER_PRIORITY_X, MODIFIER_PRIORITY_Y, MODIFIER_PRIORITY_Y_CCS, + MODIFIER_PRIORITY_Y_GEN12_RC_CCS, }; static const uint64_t priority_to_modifier[] = { @@ -66,14 +68,35 @@ static const uint64_t priority_to_modifier[] = { [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED, [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED, [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS, + [MODIFIER_PRIORITY_Y_GEN12_RC_CCS] = I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS, }; static bool modifier_is_supported(const struct gen_device_info *devinfo, enum pipe_format pfmt, uint64_t modifier) { - /* XXX: do something real */ + /* Check for basic device support. */ switch (modifier) { + case DRM_FORMAT_MOD_LINEAR: + case I915_FORMAT_MOD_X_TILED: + case I915_FORMAT_MOD_Y_TILED: + break; + case I915_FORMAT_MOD_Y_TILED_CCS: + if (devinfo->gen <= 8 || devinfo->gen >= 12) + return false; + break; + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: + if (devinfo->gen != 12) + return false; + break; + case DRM_FORMAT_MOD_INVALID: + default: + return false; + } + + /* Check remaining requirements. */ + switch (modifier) { + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: case I915_FORMAT_MOD_Y_TILED_CCS: { if (unlikely(INTEL_DEBUG & DEBUG_NO_RBC)) return false; @@ -82,21 +105,15 @@ modifier_is_supported(const struct gen_device_info *devinfo, iris_format_for_usage(devinfo, pfmt, ISL_SURF_USAGE_RENDER_TARGET_BIT).fmt; - enum isl_format linear_format = isl_format_srgb_to_linear(rt_format); - - if (!isl_format_supports_ccs_e(devinfo, linear_format)) + if (rt_format == ISL_FORMAT_UNSUPPORTED || + !isl_format_supports_ccs_e(devinfo, rt_format)) return false; - - return true; } - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_X_TILED: - case DRM_FORMAT_MOD_LINEAR: - return true; - case DRM_FORMAT_MOD_INVALID: default: - return false; + break; } + + return true; } static uint64_t @@ -111,6 +128,9 @@ select_best_modifier(struct gen_device_info *devinfo, enum pipe_format pfmt, continue; switch (modifiers[i]) { + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: + prio = MAX2(prio, MODIFIER_PRIORITY_Y_GEN12_RC_CCS); + break; case I915_FORMAT_MOD_Y_TILED_CCS: prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS); break; @@ -170,6 +190,7 @@ iris_query_dmabuf_modifiers(struct pipe_screen *pscreen, I915_FORMAT_MOD_X_TILED, I915_FORMAT_MOD_Y_TILED, I915_FORMAT_MOD_Y_TILED_CCS, + I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS, }; int supported_mods = 0; @@ -206,12 +227,38 @@ pipe_bind_to_isl_usage(unsigned bindings) if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER)) usage |= ISL_SURF_USAGE_STORAGE_BIT; - if (bindings & PIPE_BIND_DISPLAY_TARGET) + if (bindings & PIPE_BIND_SCANOUT) usage |= ISL_SURF_USAGE_DISPLAY_BIT; return usage; } +enum isl_format +iris_image_view_get_format(struct iris_context *ice, + const struct pipe_image_view *img) +{ + struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen; + const struct gen_device_info *devinfo = &screen->devinfo; + + isl_surf_usage_flags_t usage = ISL_SURF_USAGE_STORAGE_BIT; + enum isl_format isl_fmt = + iris_format_for_usage(devinfo, img->format, usage).fmt; + + if (img->shader_access & PIPE_IMAGE_ACCESS_READ) { + /* On Gen8, try to use typed surfaces reads (which support a + * limited number of formats), and if not possible, fall back + * to untyped reads. + */ + if (devinfo->gen == 8 && + !isl_has_matching_typed_storage_image_format(devinfo, isl_fmt)) + return ISL_FORMAT_RAW; + else + return isl_lower_storage_image_format(devinfo, isl_fmt); + } + + return isl_fmt; +} + struct pipe_resource * iris_resource_get_separate_stencil(struct pipe_resource *p_res) { @@ -292,8 +339,10 @@ iris_resource_disable_aux(struct iris_resource *res) res->aux.usage = ISL_AUX_USAGE_NONE; res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE; res->aux.sampler_usages = 1 << ISL_AUX_USAGE_NONE; + res->aux.has_hiz = 0; res->aux.surf.size_B = 0; res->aux.bo = NULL; + res->aux.extra_aux.surf.size_B = 0; res->aux.clear_color_bo = NULL; res->aux.state = NULL; } @@ -310,6 +359,8 @@ iris_resource_destroy(struct pipe_screen *screen, iris_resource_disable_aux(res); iris_bo_unreference(res->bo); + iris_pscreen_unref(res->base.screen); + free(res); } @@ -322,7 +373,7 @@ iris_alloc_resource(struct pipe_screen *pscreen, return NULL; res->base = *templ; - res->base.screen = pscreen; + res->base.screen = iris_pscreen_ref(pscreen); pipe_reference_init(&res->base.reference, 1); res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE; @@ -346,6 +397,8 @@ iris_get_num_logical_layers(const struct iris_resource *res, unsigned level) static enum isl_aux_state ** create_aux_state_map(struct iris_resource *res, enum isl_aux_state initial) { + assert(res->aux.state == NULL); + uint32_t total_slices = 0; for (uint32_t level = 0; level < res->surf.levels; level++) total_slices += iris_get_num_logical_layers(res, level); @@ -384,10 +437,50 @@ iris_get_aux_clear_color_state_size(struct iris_screen *screen) return devinfo->gen >= 10 ? screen->isl_dev.ss.clear_color_state_size : 0; } +static void +map_aux_addresses(struct iris_screen *screen, struct iris_resource *res) +{ + const struct gen_device_info *devinfo = &screen->devinfo; + if (devinfo->gen >= 12 && isl_aux_usage_has_ccs(res->aux.usage)) { + void *aux_map_ctx = iris_bufmgr_get_aux_map_context(screen->bufmgr); + assert(aux_map_ctx); + const unsigned aux_offset = res->aux.extra_aux.surf.size_B > 0 ? + res->aux.extra_aux.offset : res->aux.offset; + gen_aux_map_add_image(aux_map_ctx, &res->surf, res->bo->gtt_offset, + res->aux.bo->gtt_offset + aux_offset); + res->bo->aux_map_address = res->aux.bo->gtt_offset; + } +} + +static bool +want_ccs_e_for_format(const struct gen_device_info *devinfo, + enum isl_format format) +{ + if (!isl_format_supports_ccs_e(devinfo, format)) + return false; + + const struct isl_format_layout *fmtl = isl_format_get_layout(format); + + /* CCS_E seems to significantly hurt performance with 32-bit floating + * point formats. For example, Paraview's "Wavelet Volume" case uses + * both R32_FLOAT and R32G32B32A32_FLOAT, and enabling CCS_E for those + * formats causes a 62% FPS drop. + * + * However, many benchmarks seem to use 16-bit float with no issues. + */ + if (fmtl->channels.r.bits == 32 && fmtl->channels.r.type == ISL_SFLOAT) + return false; + + return true; +} + /** * Configure aux for the resource, but don't allocate it. For images which * might be shared with modifiers, we must allocate the image and aux data in * a single bo. + * + * Returns false on unexpected error (e.g. allocation failed, or invalid + * configuration result). */ static bool iris_resource_configure_aux(struct iris_screen *screen, @@ -395,24 +488,99 @@ iris_resource_configure_aux(struct iris_screen *screen, uint64_t *aux_size_B, uint32_t *alloc_flags) { - struct isl_device *isl_dev = &screen->isl_dev; - enum isl_aux_state initial_state; - UNUSED bool ok = false; + const struct gen_device_info *devinfo = &screen->devinfo; + + /* Try to create the auxiliary surfaces allowed by the modifier or by + * the user if no modifier is specified. + */ + assert(!res->mod_info || + res->mod_info->aux_usage == ISL_AUX_USAGE_NONE || + res->mod_info->aux_usage == ISL_AUX_USAGE_CCS_E || + res->mod_info->aux_usage == ISL_AUX_USAGE_GEN12_CCS_E); + + const bool has_mcs = !res->mod_info && + isl_surf_get_mcs_surf(&screen->isl_dev, &res->surf, &res->aux.surf); + + const bool has_hiz = !res->mod_info && !(INTEL_DEBUG & DEBUG_NO_HIZ) && + isl_surf_get_hiz_surf(&screen->isl_dev, &res->surf, &res->aux.surf); + + const bool has_ccs = + ((!res->mod_info && !(INTEL_DEBUG & DEBUG_NO_RBC)) || + (res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE)) && + isl_surf_get_ccs_surf(&screen->isl_dev, &res->surf, &res->aux.surf, + &res->aux.extra_aux.surf, 0); + + /* Having both HIZ and MCS is impossible. */ + assert(!has_mcs || !has_hiz); + + /* Ensure aux surface creation for MCS_CCS and HIZ_CCS is correct. */ + if (has_ccs && (has_mcs || has_hiz)) { + assert(res->aux.extra_aux.surf.size_B > 0 && + res->aux.extra_aux.surf.usage & ISL_SURF_USAGE_CCS_BIT); + assert(res->aux.surf.size_B > 0 && + res->aux.surf.usage & + (ISL_SURF_USAGE_HIZ_BIT | ISL_SURF_USAGE_MCS_BIT)); + } + + if (res->mod_info && has_ccs) { + /* Only allow a CCS modifier if the aux was created successfully. */ + res->aux.possible_usages |= 1 << res->mod_info->aux_usage; + } else if (has_mcs) { + res->aux.possible_usages |= + 1 << (has_ccs ? ISL_AUX_USAGE_MCS_CCS : ISL_AUX_USAGE_MCS); + } else if (has_hiz) { + if (!has_ccs) { + res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ; + } else if (res->surf.samples == 1 && + (res->surf.usage & ISL_SURF_USAGE_TEXTURE_BIT)) { + /* If this resource is single-sampled and will be used as a texture, + * put the HiZ surface in write-through mode so that we can sample + * from it. + */ + res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ_CCS_WT; + } else { + res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ_CCS; + } + } else if (has_ccs && isl_surf_usage_is_stencil(res->surf.usage)) { + res->aux.possible_usages |= 1 << ISL_AUX_USAGE_STC_CCS; + } else if (has_ccs) { + if (want_ccs_e_for_format(devinfo, res->surf.format)) { + res->aux.possible_usages |= devinfo->gen < 12 ? + 1 << ISL_AUX_USAGE_CCS_E : 1 << ISL_AUX_USAGE_GEN12_CCS_E; + } else if (isl_format_supports_ccs_d(devinfo, res->surf.format)) { + res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_D; + } + } + res->aux.usage = util_last_bit(res->aux.possible_usages) - 1; + + res->aux.sampler_usages = res->aux.possible_usages; + + /* We don't always support sampling with hiz. But when we do, it must be + * single sampled. + */ + if (!devinfo->has_sample_with_hiz || res->surf.samples > 1) + res->aux.sampler_usages &= ~(1 << ISL_AUX_USAGE_HIZ); + + /* ISL_AUX_USAGE_HIZ_CCS doesn't support sampling at all */ + res->aux.sampler_usages &= ~(1 << ISL_AUX_USAGE_HIZ_CCS); + + enum isl_aux_state initial_state; *aux_size_B = 0; *alloc_flags = 0; assert(!res->aux.bo); switch (res->aux.usage) { case ISL_AUX_USAGE_NONE: - res->aux.surf.size_B = 0; - ok = true; - break; + /* Having no aux buffer is only okay if there's no modifier with aux. */ + return !res->mod_info || res->mod_info->aux_usage == ISL_AUX_USAGE_NONE; case ISL_AUX_USAGE_HIZ: + case ISL_AUX_USAGE_HIZ_CCS: + case ISL_AUX_USAGE_HIZ_CCS_WT: initial_state = ISL_AUX_STATE_AUX_INVALID; - ok = isl_surf_get_hiz_surf(isl_dev, &res->surf, &res->aux.surf); break; case ISL_AUX_USAGE_MCS: + case ISL_AUX_USAGE_MCS_CCS: /* The Ivybridge PRM, Vol 2 Part 1 p326 says: * * "When MCS buffer is enabled and bound to MSRT, it is required @@ -423,10 +591,11 @@ iris_resource_configure_aux(struct iris_screen *screen, * 1's, so we simply memset it to 0xff. */ initial_state = ISL_AUX_STATE_CLEAR; - ok = isl_surf_get_mcs_surf(isl_dev, &res->surf, &res->aux.surf); break; case ISL_AUX_USAGE_CCS_D: case ISL_AUX_USAGE_CCS_E: + case ISL_AUX_USAGE_GEN12_CCS_E: + case ISL_AUX_USAGE_STC_CCS: /* When CCS_E is used, we need to ensure that the CCS starts off in * a valid state. From the Sky Lake PRM, "MCS Buffer for Render * Target(s)": @@ -440,44 +609,54 @@ iris_resource_configure_aux(struct iris_screen *screen, * For CCS_D, do the same thing. On Gen9+, this avoids having any * undefined bits in the aux buffer. */ - if (imported) + if (imported) { + assert(res->aux.usage != ISL_AUX_USAGE_STC_CCS); initial_state = isl_drm_modifier_get_default_aux_state(res->mod_info->modifier); - else + } else { initial_state = ISL_AUX_STATE_PASS_THROUGH; + } *alloc_flags |= BO_ALLOC_ZEROED; - ok = isl_surf_get_ccs_surf(isl_dev, &res->surf, &res->aux.surf, 0); break; + case ISL_AUX_USAGE_MC: + unreachable("Unsupported aux mode"); } - /* We should have a valid aux_surf. */ - if (!ok) + /* Create the aux_state for the auxiliary buffer. */ + res->aux.state = create_aux_state_map(res, initial_state); + if (!res->aux.state) return false; - /* No work is needed for a zero-sized auxiliary buffer. */ - if (res->aux.surf.size_B == 0) - return true; + /* Increase the aux offset if the main and aux surfaces will share a BO. */ + res->aux.offset = + !res->mod_info || res->mod_info->aux_usage == res->aux.usage ? + ALIGN(res->surf.size_B, res->aux.surf.alignment_B) : 0; + uint64_t size = res->aux.surf.size_B; - if (!res->aux.state) { - /* Create the aux_state for the auxiliary buffer. */ - res->aux.state = create_aux_state_map(res, initial_state); - if (!res->aux.state) - return false; + /* Allocate space in the buffer for storing the CCS. */ + if (res->aux.extra_aux.surf.size_B > 0) { + const uint64_t padded_aux_size = + ALIGN(size, res->aux.extra_aux.surf.alignment_B); + res->aux.extra_aux.offset = res->aux.offset + padded_aux_size; + size = padded_aux_size + res->aux.extra_aux.surf.size_B; } - uint64_t size = res->aux.surf.size_B; - /* Allocate space in the buffer for storing the clear color. On modern * platforms (gen > 9), we can read it directly from such buffer. * * On gen <= 9, we are going to store the clear color on the buffer * anyways, and copy it back to the surface state during state emission. + * + * Also add some padding to make sure the fast clear color state buffer + * starts at a 4K alignment. We believe that 256B might be enough, but due + * to lack of testing we will leave this as 4K for now. */ - res->aux.clear_color_offset = size; + size = ALIGN(size, 4096); + res->aux.clear_color_offset = res->aux.offset + size; size += iris_get_aux_clear_color_state_size(screen); *aux_size_B = size; - if (res->aux.usage == ISL_AUX_USAGE_HIZ) { + if (isl_aux_usage_has_hiz(res->aux.usage)) { for (unsigned level = 0; level < res->surf.levels; ++level) { uint32_t width = u_minify(res->surf.phys_level0_sa.width, level); uint32_t height = u_minify(res->surf.phys_level0_sa.height, level); @@ -495,6 +674,8 @@ iris_resource_configure_aux(struct iris_screen *screen, /** * Initialize the aux buffer contents. + * + * Returns false on unexpected error (e.g. mapping a BO failed). */ static bool iris_resource_init_aux_buf(struct iris_resource *res, uint32_t alloc_flags, @@ -503,17 +684,18 @@ iris_resource_init_aux_buf(struct iris_resource *res, uint32_t alloc_flags, if (!(alloc_flags & BO_ALLOC_ZEROED)) { void *map = iris_bo_map(NULL, res->aux.bo, MAP_WRITE | MAP_RAW); - if (!map) { - iris_resource_disable_aux(res); + if (!map) return false; - } if (iris_resource_get_aux_state(res, 0, 0) != ISL_AUX_STATE_AUX_INVALID) { - uint8_t memset_value = res->aux.usage == ISL_AUX_USAGE_MCS ? 0xFF : 0; + uint8_t memset_value = isl_aux_usage_has_mcs(res->aux.usage) ? 0xFF : 0; memset((char*)map + res->aux.offset, memset_value, res->aux.surf.size_B); } + memset((char*)map + res->aux.extra_aux.offset, + 0, res->aux.extra_aux.surf.size_B); + /* Zero the indirect clear color to match ::fast_clear_color. */ memset((char *)map + res->aux.clear_color_offset, 0, clear_color_state_size); @@ -531,6 +713,9 @@ iris_resource_init_aux_buf(struct iris_resource *res, uint32_t alloc_flags, /** * Allocate the initial aux surface for a resource based on aux.usage + * + * Returns false on unexpected error (e.g. allocation failed, or invalid + * configuration result). */ static bool iris_resource_alloc_separate_aux(struct iris_screen *screen, @@ -550,7 +735,8 @@ iris_resource_alloc_separate_aux(struct iris_screen *screen, * block sizes. */ res->aux.bo = iris_bo_alloc_tiled(screen->bufmgr, "aux buffer", size, 4096, - IRIS_MEMZONE_OTHER, I915_TILING_Y, + IRIS_MEMZONE_OTHER, + isl_tiling_to_i915_tiling(res->aux.surf.tiling), res->aux.surf.row_pitch_B, alloc_flags); if (!res->aux.bo) { return false; @@ -560,6 +746,8 @@ iris_resource_alloc_separate_aux(struct iris_screen *screen, iris_get_aux_clear_color_state_size(screen))) return false; + map_aux_addresses(screen, res); + return true; } @@ -599,33 +787,8 @@ iris_resource_finish_aux_import(struct pipe_screen *pscreen, iris_resource_destroy(&screen->base, res->base.next); res->base.next = NULL; -} - -static bool -supports_mcs(const struct isl_surf *surf) -{ - /* MCS compression only applies to multisampled resources. */ - if (surf->samples <= 1) - return false; - - /* Depth and stencil buffers use the IMS (interleaved) layout. */ - if (isl_surf_usage_is_depth_or_stencil(surf->usage)) - return false; - - return true; -} - -static bool -supports_ccs(const struct gen_device_info *devinfo, - const struct isl_surf *surf) -{ - /* CCS only supports singlesampled resources. */ - if (surf->samples > 1) - return false; - - /* Note: still need to check the format! */ - return true; + map_aux_addresses(screen, res); } static struct pipe_resource * @@ -663,6 +826,9 @@ iris_resource_create_for_buffer(struct pipe_screen *pscreen, return NULL; } + if (templ->bind & PIPE_BIND_SHARED) + iris_bo_make_external(res->bo); + return &res->base; } @@ -694,28 +860,19 @@ iris_resource_create_with_modifiers(struct pipe_screen *pscreen, } else { if (modifiers_count > 0) { fprintf(stderr, "Unsupported modifier, resource creation failed.\n"); - return NULL; - } - - /* No modifiers - we can select our own tiling. */ - - if (has_depth) { - /* Depth must be Y-tiled */ - tiling_flags = ISL_TILING_Y0_BIT; - } else if (templ->format == PIPE_FORMAT_S8_UINT) { - /* Stencil must be W-tiled */ - tiling_flags = ISL_TILING_W_BIT; - } else if (templ->target == PIPE_BUFFER || - templ->target == PIPE_TEXTURE_1D || - templ->target == PIPE_TEXTURE_1D_ARRAY) { - /* Use linear for buffers and 1D textures */ - tiling_flags = ISL_TILING_LINEAR_BIT; + goto fail; } /* Use linear for staging buffers */ if (templ->usage == PIPE_USAGE_STAGING || - templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) ) + templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) ) { tiling_flags = ISL_TILING_LINEAR_BIT; + } else if (templ->bind & PIPE_BIND_SCANOUT) { + if (devinfo->has_tiling_uapi) + tiling_flags = ISL_TILING_X_BIT; + else + tiling_flags = ISL_TILING_LINEAR_BIT; + } } isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind); @@ -756,33 +913,6 @@ iris_resource_create_with_modifiers(struct pipe_screen *pscreen, .tiling_flags = tiling_flags); assert(isl_surf_created_successfully); - if (res->mod_info) { - res->aux.possible_usages |= 1 << res->mod_info->aux_usage; - } else if (supports_mcs(&res->surf)) { - res->aux.possible_usages |= 1 << ISL_AUX_USAGE_MCS; - } else if (has_depth) { - if (likely(!(INTEL_DEBUG & DEBUG_NO_HIZ))) - res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ; - } else if (likely(!(INTEL_DEBUG & DEBUG_NO_RBC)) && - supports_ccs(devinfo, &res->surf)) { - if (isl_format_supports_ccs_e(devinfo, res->surf.format)) - res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_E; - - if (isl_format_supports_ccs_d(devinfo, res->surf.format)) - res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_D; - } - - res->aux.usage = util_last_bit(res->aux.possible_usages) - 1; - - res->aux.sampler_usages = res->aux.possible_usages; - - /* We don't always support sampling with hiz. But when we do, it must be - * single sampled. - */ - if (!devinfo->has_sample_with_hiz || res->surf.samples > 1) { - res->aux.sampler_usages &= ~(1 << ISL_AUX_USAGE_HIZ); - } - const char *name = "miptree"; enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER; @@ -797,51 +927,37 @@ iris_resource_create_with_modifiers(struct pipe_screen *pscreen, uint32_t aux_preferred_alloc_flags; uint64_t aux_size = 0; - bool aux_enabled = - iris_resource_configure_aux(screen, res, false, &aux_size, - &aux_preferred_alloc_flags); - aux_enabled = aux_enabled && res->aux.surf.size_B > 0; - const bool separate_aux = aux_enabled && !res->mod_info; - uint64_t aux_offset; - uint64_t bo_size; - - if (aux_enabled && !separate_aux) { - /* Allocate aux data with main surface. This is required for modifiers - * with aux data (ccs). - */ - aux_offset = ALIGN(res->surf.size_B, res->aux.surf.alignment_B); - bo_size = aux_offset + aux_size; - } else { - aux_offset = 0; - bo_size = res->surf.size_B; + if (!iris_resource_configure_aux(screen, res, false, &aux_size, + &aux_preferred_alloc_flags)) { + goto fail; } - res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, bo_size, 4096, memzone, + /* Modifiers require the aux data to be in the same buffer as the main + * surface, but we combine them even when a modifiers is not being used. + */ + const uint64_t bo_size = + MAX2(res->surf.size_B, res->aux.offset + aux_size); + uint32_t alignment = MAX2(4096, res->surf.alignment_B); + res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, bo_size, alignment, + memzone, isl_tiling_to_i915_tiling(res->surf.tiling), res->surf.row_pitch_B, flags); if (!res->bo) goto fail; - if (aux_enabled) { - if (separate_aux) { - if (!iris_resource_alloc_separate_aux(screen, res)) - aux_enabled = false; - } else { - res->aux.bo = res->bo; - iris_bo_reference(res->aux.bo); - res->aux.offset += aux_offset; - unsigned clear_color_state_size = - iris_get_aux_clear_color_state_size(screen); - if (clear_color_state_size > 0) - res->aux.clear_color_offset += aux_offset; - if (!iris_resource_init_aux_buf(res, flags, clear_color_state_size)) - aux_enabled = false; - } + if (aux_size > 0) { + res->aux.bo = res->bo; + iris_bo_reference(res->aux.bo); + unsigned clear_color_state_size = + iris_get_aux_clear_color_state_size(screen); + if (!iris_resource_init_aux_buf(res, flags, clear_color_state_size)) + goto fail; + map_aux_addresses(screen, res); } - if (!aux_enabled) - iris_resource_disable_aux(res); + if (templ->bind & PIPE_BIND_SHARED) + iris_bo_make_external(res->bo); return &res->base; @@ -894,11 +1010,11 @@ iris_resource_from_user_memory(struct pipe_screen *pscreen, user_memory, templ->width0, IRIS_MEMZONE_OTHER); if (!res->bo) { - free(res); + iris_resource_destroy(pscreen, &res->base); return NULL; } - util_range_add(&res->valid_buffer_range, 0, templ->width0); + util_range_add(&res->base, &res->valid_buffer_range, 0, templ->width0); return &res->base; } @@ -913,12 +1029,21 @@ iris_resource_from_handle(struct pipe_screen *pscreen, struct gen_device_info *devinfo = &screen->devinfo; struct iris_bufmgr *bufmgr = screen->bufmgr; struct iris_resource *res = iris_alloc_resource(pscreen, templ); + const struct isl_drm_modifier_info *mod_inf = + isl_drm_modifier_get_info(whandle->modifier); + int tiling; + if (!res) return NULL; switch (whandle->type) { case WINSYS_HANDLE_TYPE_FD: - res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle); + if (mod_inf) + tiling = isl_tiling_to_i915_tiling(mod_inf->tiling); + else + tiling = -1; + res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle, + tiling, whandle->stride); break; case WINSYS_HANDLE_TYPE_SHARED: res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image", @@ -928,16 +1053,18 @@ iris_resource_from_handle(struct pipe_screen *pscreen, unreachable("invalid winsys handle type"); } if (!res->bo) - return NULL; + goto fail; res->offset = whandle->offset; - uint64_t modifier = whandle->modifier; - if (modifier == DRM_FORMAT_MOD_INVALID) { - modifier = tiling_to_modifier(res->bo->tiling_mode); + if (mod_inf == NULL) { + mod_inf = + isl_drm_modifier_get_info(tiling_to_modifier(res->bo->tiling_mode)); } - res->mod_info = isl_drm_modifier_get_info(modifier); - assert(res->mod_info); + assert(mod_inf); + + res->external_format = whandle->format; + res->mod_info = mod_inf; isl_surf_usage_flags_t isl_usage = pipe_bind_to_isl_usage(templ->bind); @@ -948,7 +1075,8 @@ iris_resource_from_handle(struct pipe_screen *pscreen, if (templ->target == PIPE_BUFFER) { res->surf.tiling = ISL_TILING_LINEAR; } else { - if (whandle->modifier == DRM_FORMAT_MOD_INVALID || whandle->plane == 0) { + /* Create a surface for each plane specified by the external format. */ + if (whandle->plane < util_format_get_num_planes(whandle->format)) { UNUSED const bool isl_surf_created_successfully = isl_surf_init(&screen->isl_dev, &res->surf, .dim = target_to_isl_surf_dim(templ->target), @@ -975,9 +1103,6 @@ iris_resource_from_handle(struct pipe_screen *pscreen, if (res->mod_info->aux_usage != ISL_AUX_USAGE_NONE) { uint32_t alloc_flags; uint64_t size; - res->aux.usage = res->mod_info->aux_usage; - res->aux.possible_usages = 1 << res->mod_info->aux_usage; - res->aux.sampler_usages = res->aux.possible_usages; bool ok = iris_resource_configure_aux(screen, res, true, &size, &alloc_flags); assert(ok); @@ -1012,32 +1137,60 @@ static void iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource) { struct iris_context *ice = (struct iris_context *)ctx; - struct iris_batch *render_batch = &ice->batches[IRIS_BATCH_RENDER]; struct iris_resource *res = (void *) resource; const struct isl_drm_modifier_info *mod = res->mod_info; - iris_resource_prepare_access(ice, render_batch, res, + iris_resource_prepare_access(ice, res, 0, INTEL_REMAINING_LEVELS, 0, INTEL_REMAINING_LAYERS, mod ? mod->aux_usage : ISL_AUX_USAGE_NONE, mod ? mod->supports_clear_color : false); } +static void +iris_resource_disable_aux_on_first_query(struct pipe_resource *resource, + unsigned usage) +{ + struct iris_resource *res = (struct iris_resource *)resource; + bool mod_with_aux = + res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE; + + /* Disable aux usage if explicit flush not set and this is the first time + * we are dealing with this resource and the resource was not created with + * a modifier with aux. + */ + if (!mod_with_aux && + (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && res->aux.usage != 0) && + p_atomic_read(&resource->reference.count) == 1) { + iris_resource_disable_aux(res); + } +} + static bool -iris_resource_get_param(struct pipe_screen *screen, +iris_resource_get_param(struct pipe_screen *pscreen, + struct pipe_context *context, struct pipe_resource *resource, - unsigned int plane, + unsigned plane, + unsigned layer, enum pipe_resource_param param, + unsigned handle_usage, uint64_t *value) { + struct iris_screen *screen = (struct iris_screen *)pscreen; struct iris_resource *res = (struct iris_resource *)resource; bool mod_with_aux = res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE; bool wants_aux = mod_with_aux && plane > 0; - struct iris_bo *bo = wants_aux ? res->aux.bo : res->bo; bool result; unsigned handle; + if (iris_resource_unfinished_aux_import(res)) + iris_resource_finish_aux_import(pscreen, res); + + struct iris_bo *bo = wants_aux ? res->aux.bo : res->bo; + + iris_resource_disable_aux_on_first_query(resource, handle_usage); + switch (param) { case PIPE_RESOURCE_PARAM_NPLANES: if (mod_with_aux) { @@ -1064,9 +1217,19 @@ iris_resource_get_param(struct pipe_screen *screen, if (result) *value = handle; return result; - case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS: - *value = iris_bo_export_gem_handle(bo); + case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS: { + /* Because we share the same drm file across multiple iris_screen, when + * we export a GEM handle we must make sure it is valid in the DRM file + * descriptor the caller is using (this is the FD given at screen + * creation). + */ + uint32_t handle; + if (iris_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle)) + return false; + *value = handle; return true; + } + case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD: result = iris_bo_export_dmabuf(bo, (int *) &handle) == 0; if (result) @@ -1084,19 +1247,12 @@ iris_resource_get_handle(struct pipe_screen *pscreen, struct winsys_handle *whandle, unsigned usage) { + struct iris_screen *screen = (struct iris_screen *) pscreen; struct iris_resource *res = (struct iris_resource *)resource; bool mod_with_aux = res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE; - /* Disable aux usage if explicit flush not set and this is the first time - * we are dealing with this resource and the resource was not created with - * a modifier with aux. - */ - if (!mod_with_aux && - (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && res->aux.usage != 0) && - p_atomic_read(&resource->reference.count) == 1) { - iris_resource_disable_aux(res); - } + iris_resource_disable_aux_on_first_query(resource, usage); struct iris_bo *bo; if (mod_with_aux && whandle->plane > 0) { @@ -1109,6 +1265,8 @@ iris_resource_get_handle(struct pipe_screen *pscreen, whandle->stride = res->surf.row_pitch_B; bo = res->bo; } + + whandle->format = res->external_format; whandle->modifier = res->mod_info ? res->mod_info->modifier : tiling_to_modifier(res->bo->tiling_mode); @@ -1127,9 +1285,18 @@ iris_resource_get_handle(struct pipe_screen *pscreen, switch (whandle->type) { case WINSYS_HANDLE_TYPE_SHARED: return iris_bo_flink(bo, &whandle->handle) == 0; - case WINSYS_HANDLE_TYPE_KMS: - whandle->handle = iris_bo_export_gem_handle(bo); + case WINSYS_HANDLE_TYPE_KMS: { + /* Because we share the same drm file across multiple iris_screen, when + * we export a GEM handle we must make sure it is valid in the DRM file + * descriptor the caller is using (this is the FD given at screen + * creation). + */ + uint32_t handle; + if (iris_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle)) + return false; + whandle->handle = handle; return true; + } case WINSYS_HANDLE_TYPE_FD: return iris_bo_export_dmabuf(bo, (int *) &whandle->handle) == 0; } @@ -1160,6 +1327,10 @@ iris_invalidate_resource(struct pipe_context *ctx, if (resource->target != PIPE_BUFFER) return; + /* If it's already invalidated, don't bother doing anything. */ + if (res->valid_buffer_range.start > res->valid_buffer_range.end) + return; + if (!resource_is_busy(ice, res)) { /* The resource is idle, so just mark that it contains no data and * keep using the same underlying buffer object. @@ -1191,7 +1362,7 @@ iris_invalidate_resource(struct pipe_context *ctx, /* Rebind the buffer, replacing any state referring to the old BO's * address, and marking state dirty so it's reemitted. */ - ice->vtbl.rebind_buffer(ice, res, old_bo->gtt_offset); + screen->vtbl.rebind_buffer(ice, res); util_range_set_empty(&res->valid_buffer_range); @@ -1688,6 +1859,9 @@ iris_transfer_map(struct pipe_context *ctx, struct iris_resource *res = (struct iris_resource *)resource; struct isl_surf *surf = &res->surf; + if (iris_resource_unfinished_aux_import(res)) + iris_resource_finish_aux_import(ctx->screen, res); + if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) { /* Replace the backing storage with a fresh buffer for non-async maps */ if (!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | @@ -1708,13 +1882,17 @@ iris_transfer_map(struct pipe_context *ctx, if (resource->target != PIPE_BUFFER) { bool need_hiz_resolve = iris_resource_level_has_hiz(res, level); + bool need_stencil_resolve = res->aux.usage == ISL_AUX_USAGE_STC_CCS; need_color_resolve = (res->aux.usage == ISL_AUX_USAGE_CCS_D || - res->aux.usage == ISL_AUX_USAGE_CCS_E) && + res->aux.usage == ISL_AUX_USAGE_CCS_E || + res->aux.usage == ISL_AUX_USAGE_GEN12_CCS_E) && iris_has_color_unresolved(res, level, 1, box->z, box->depth); - need_resolve = need_color_resolve || need_hiz_resolve; + need_resolve = need_color_resolve || + need_hiz_resolve || + need_stencil_resolve; } bool map_would_stall = false; @@ -1751,7 +1929,7 @@ iris_transfer_map(struct pipe_context *ctx, box->x + box->width); if (usage & PIPE_TRANSFER_WRITE) - util_range_add(&res->valid_buffer_range, box->x, box->x + box->width); + util_range_add(&res->base, &res->valid_buffer_range, box->x, box->x + box->width); /* Avoid using GPU copies for persistent/coherent buffers, as the idea * there is to access them simultaneously on the CPU & GPU. This also @@ -1779,7 +1957,9 @@ iris_transfer_map(struct pipe_context *ctx, if (fmtl->txc == ISL_TXC_ASTC) no_gpu = true; - if ((map_would_stall || res->aux.usage == ISL_AUX_USAGE_CCS_E) && !no_gpu) { + if ((map_would_stall || + res->aux.usage == ISL_AUX_USAGE_CCS_E || + res->aux.usage == ISL_AUX_USAGE_GEN12_CCS_E) && !no_gpu) { /* If we need a synchronous mapping and the resource is busy, or needs * resolving, we copy to/from a linear temporary buffer using the GPU. */ @@ -1790,8 +1970,7 @@ iris_transfer_map(struct pipe_context *ctx, /* Otherwise we're free to map on the CPU. */ if (need_resolve) { - iris_resource_access_raw(ice, &ice->batches[IRIS_BATCH_RENDER], res, - level, box->z, box->depth, + iris_resource_access_raw(ice, res, level, box->z, box->depth, usage & PIPE_TRANSFER_WRITE); } @@ -1836,7 +2015,7 @@ iris_transfer_flush_region(struct pipe_context *ctx, if (map->dest_had_defined_contents) history_flush |= iris_flush_bits_for_history(res); - util_range_add(&res->valid_buffer_range, box->x, box->x + box->width); + util_range_add(&res->base, &res->valid_buffer_range, box->x, box->x + box->width); } if (history_flush & ~PIPE_CONTROL_CS_STALL) { @@ -1888,19 +2067,14 @@ void iris_dirty_for_history(struct iris_context *ice, struct iris_resource *res) { - uint64_t dirty = 0ull; + uint64_t stage_dirty = 0ull; if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) { - dirty |= IRIS_DIRTY_CONSTANTS_VS | - IRIS_DIRTY_CONSTANTS_TCS | - IRIS_DIRTY_CONSTANTS_TES | - IRIS_DIRTY_CONSTANTS_GS | - IRIS_DIRTY_CONSTANTS_FS | - IRIS_DIRTY_CONSTANTS_CS | - IRIS_ALL_DIRTY_BINDINGS; + stage_dirty |= ((uint64_t)res->bind_stages) + << IRIS_SHIFT_FOR_STAGE_DIRTY_CONSTANTS; } - ice->state.dirty |= dirty; + ice->state.stage_dirty |= stage_dirty; } /**