X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;ds=sidebyside;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fintel_mipmap_tree.c;h=e28ac562a154b2ceab37c4a83e4cf9dd82a27de3;hb=3a4667e502c458b6ac0ae3d4641612a9186a1924;hp=ead0c359c0fd35349a9f7aba8cd0a41d596e95f9;hpb=fd3fc5f5479410ffc87852c7d3a391e3ec548009;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c index ead0c359c0f..e28ac562a15 100644 --- a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c +++ b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c @@ -25,7 +25,7 @@ #include #include -#include +#include "drm-uapi/drm_fourcc.h" #include "intel_batchbuffer.h" #include "intel_image.h" @@ -45,6 +45,9 @@ #include "main/texcompress_etc.h" #include "main/teximage.h" #include "main/streaming-load-memcpy.h" + +#include "util/format_srgb.h" + #include "x86/common_x86_asm.h" #define FILE_DEBUG_FLAG DEBUG_MIPTREE @@ -55,202 +58,6 @@ static void *intel_miptree_map_raw(struct brw_context *brw, static void intel_miptree_unmap_raw(struct intel_mipmap_tree *mt); -static bool -intel_miptree_alloc_aux(struct brw_context *brw, - struct intel_mipmap_tree *mt); - -static bool -intel_miptree_supports_mcs(struct brw_context *brw, - const struct intel_mipmap_tree *mt) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - /* MCS compression only applies to multisampled miptrees */ - if (mt->surf.samples <= 1) - return false; - - /* Prior to Gen7, all MSAA surfaces used IMS layout. */ - if (devinfo->gen < 7) - return false; - - /* See isl_surf_get_mcs_surf for details. */ - if (mt->surf.samples == 16 && mt->surf.logical_level0_px.width > 8192) - return false; - - /* In Gen7, IMS layout is only used for depth and stencil buffers. */ - switch (_mesa_get_format_base_format(mt->format)) { - case GL_DEPTH_COMPONENT: - case GL_STENCIL_INDEX: - case GL_DEPTH_STENCIL: - return false; - default: - /* From the Ivy Bridge PRM, Vol4 Part1 p77 ("MCS Enable"): - * - * This field must be set to 0 for all SINT MSRTs when all RT channels - * are not written - * - * In practice this means that we have to disable MCS for all signed - * integer MSAA buffers. The alternative, to disable MCS only when one - * of the render target channels is disabled, is impractical because it - * would require converting between CMS and UMS MSAA layouts on the fly, - * which is expensive. - */ - if (devinfo->gen == 7 && _mesa_get_format_datatype(mt->format) == GL_INT) { - return false; - } else { - return true; - } - } -} - -static bool -intel_tiling_supports_ccs(const struct brw_context *brw, - enum isl_tiling tiling) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render - * Target(s)", beneath the "Fast Color Clear" bullet (p326): - * - * - Support is limited to tiled render targets. - * - * Gen9 changes the restriction to Y-tile only. - */ - if (devinfo->gen >= 9) - return tiling == ISL_TILING_Y0; - else if (devinfo->gen >= 7) - return tiling != ISL_TILING_LINEAR; - else - return false; -} - -/** - * For a single-sampled render target ("non-MSRT"), determine if an MCS buffer - * can be used. This doesn't (and should not) inspect any of the properties of - * the miptree's BO. - * - * From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render Target(s)", - * beneath the "Fast Color Clear" bullet (p326): - * - * - Support is for non-mip-mapped and non-array surface types only. - * - * And then later, on p327: - * - * - MCS buffer for non-MSRT is supported only for RT formats 32bpp, - * 64bpp, and 128bpp. - * - * From the Skylake documentation, it is made clear that X-tiling is no longer - * supported: - * - * - MCS and Lossless compression is supported for TiledY/TileYs/TileYf - * non-MSRTs only. - */ -static bool -intel_miptree_supports_ccs(struct brw_context *brw, - const struct intel_mipmap_tree *mt) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - /* MCS support does not exist prior to Gen7 */ - if (devinfo->gen < 7) - return false; - - /* This function applies only to non-multisampled render targets. */ - if (mt->surf.samples > 1) - return false; - - /* MCS is only supported for color buffers */ - switch (_mesa_get_format_base_format(mt->format)) { - case GL_DEPTH_COMPONENT: - case GL_DEPTH_STENCIL: - case GL_STENCIL_INDEX: - return false; - } - - if (mt->cpp != 4 && mt->cpp != 8 && mt->cpp != 16) - return false; - - const bool mip_mapped = mt->first_level != 0 || mt->last_level != 0; - const bool arrayed = mt->surf.logical_level0_px.array_len > 1 || - mt->surf.logical_level0_px.depth > 1; - - if (arrayed) { - /* Multisample surfaces with the CMS layout are not layered surfaces, - * yet still have physical_depth0 > 1. Assert that we don't - * accidentally reject a multisampled surface here. We should have - * rejected it earlier by explicitly checking the sample count. - */ - assert(mt->surf.samples == 1); - } - - /* Handle the hardware restrictions... - * - * All GENs have the following restriction: "MCS buffer for non-MSRT is - * supported only for RT formats 32bpp, 64bpp, and 128bpp." - * - * From the HSW PRM Volume 7: 3D-Media-GPGPU, page 652: (Color Clear of - * Non-MultiSampler Render Target Restrictions) Support is for - * non-mip-mapped and non-array surface types only. - * - * From the BDW PRM Volume 7: 3D-Media-GPGPU, page 649: (Color Clear of - * Non-MultiSampler Render Target Restriction). Mip-mapped and arrayed - * surfaces are supported with MCS buffer layout with these alignments in - * the RT space: Horizontal Alignment = 256 and Vertical Alignment = 128. - * - * From the SKL PRM Volume 7: 3D-Media-GPGPU, page 632: (Color Clear of - * Non-MultiSampler Render Target Restriction). Mip-mapped and arrayed - * surfaces are supported with MCS buffer layout with these alignments in - * the RT space: Horizontal Alignment = 128 and Vertical Alignment = 64. - */ - if (devinfo->gen < 8 && (mip_mapped || arrayed)) - return false; - - /* There's no point in using an MCS buffer if the surface isn't in a - * renderable format. - */ - if (!brw->mesa_format_supports_render[mt->format]) - return false; - - if (devinfo->gen >= 9) { - mesa_format linear_format = _mesa_get_srgb_format_linear(mt->format); - const enum isl_format isl_format = - brw_isl_format_for_mesa_format(linear_format); - return isl_format_supports_ccs_e(&brw->screen->devinfo, isl_format); - } else - return true; -} - -static bool -intel_tiling_supports_hiz(const struct brw_context *brw, - enum isl_tiling tiling) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (devinfo->gen < 6) - return false; - - return tiling == ISL_TILING_Y0; -} - -static bool -intel_miptree_supports_hiz(const struct brw_context *brw, - const struct intel_mipmap_tree *mt) -{ - if (!brw->has_hiz) - return false; - - switch (mt->format) { - case MESA_FORMAT_Z_FLOAT32: - case MESA_FORMAT_Z32_FLOAT_S8X24_UINT: - case MESA_FORMAT_Z24_UNORM_X8_UINT: - case MESA_FORMAT_Z24_UNORM_S8_UINT: - case MESA_FORMAT_Z_UNORM16: - return true; - default: - return false; - } -} - /** * Return true if the format that will be used to access the miptree is * CCS_E-compatible with the miptree's linear/non-sRGB format. @@ -260,7 +67,7 @@ intel_miptree_supports_hiz(const struct brw_context *brw, * our HW tends to support more linear formats than sRGB ones, we use this * format variant for check for CCS_E compatibility. */ -MAYBE_UNUSED static bool +static bool format_ccs_e_compat_with_miptree(const struct gen_device_info *devinfo, const struct intel_mipmap_tree *mt, enum isl_format access_format) @@ -272,15 +79,10 @@ format_ccs_e_compat_with_miptree(const struct gen_device_info *devinfo, return isl_formats_are_ccs_e_compatible(devinfo, isl_format, access_format); } +/* Determine if CCS_E is supported for a given platform and mesa format. */ static bool -intel_miptree_supports_ccs_e(struct brw_context *brw, - const struct intel_mipmap_tree *mt) +format_supports_ccs_e(const struct brw_context *brw, mesa_format format) { - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (devinfo->gen < 9) - return false; - /* For now compression is only enabled for integer formats even though * there exist supported floating point formats also. This is a heuristic * decision based on current public benchmarks. In none of the cases these @@ -288,18 +90,16 @@ intel_miptree_supports_ccs_e(struct brw_context *brw, * Hence these are left to to be enabled in the future when they are known * to improve things. */ - if (_mesa_get_format_datatype(mt->format) == GL_FLOAT) - return false; - - if (!intel_miptree_supports_ccs(brw, mt)) + if (_mesa_get_format_datatype(format) == GL_FLOAT) return false; - /* Fast clear can be also used to clear srgb surfaces by using equivalent - * linear format. This trick, however, can't be extended to be used with - * lossless compression and therefore a check is needed to see if the format - * really is linear. + /* Many window system buffers are sRGB even if they are never rendered as + * sRGB. For those, we want CCS_E for when sRGBEncode is false. When the + * surface is used as sRGB, we fall back to CCS_D. */ - return _mesa_get_srgb_format_linear(mt->format) == mt->format; + mesa_format linear_format = _mesa_get_srgb_format_linear(format); + enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format); + return isl_format_supports_ccs_e(&brw->screen->devinfo, isl_format); } /** @@ -353,8 +153,7 @@ needs_separate_stencil(const struct brw_context *brw, if (devinfo->must_use_separate_stencil) return true; - return brw->has_separate_stencil && - intel_miptree_supports_hiz(brw, mt); + return brw->has_separate_stencil && brw->has_hiz; } /** @@ -367,19 +166,16 @@ intel_miptree_choose_aux_usage(struct brw_context *brw, { assert(mt->aux_usage == ISL_AUX_USAGE_NONE); - if (intel_miptree_supports_mcs(brw, mt)) { - assert(mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY); - mt->aux_usage = ISL_AUX_USAGE_MCS; - } else if (intel_tiling_supports_ccs(brw, mt->surf.tiling) && - intel_miptree_supports_ccs(brw, mt)) { - if (!unlikely(INTEL_DEBUG & DEBUG_NO_RBC) && - intel_miptree_supports_ccs_e(brw, mt)) { + if (_mesa_is_format_color_format(mt->format)) { + if (mt->surf.samples > 1) { + mt->aux_usage = ISL_AUX_USAGE_MCS; + } else if (!unlikely(INTEL_DEBUG & DEBUG_NO_RBC) && + format_supports_ccs_e(brw, mt->format)) { mt->aux_usage = ISL_AUX_USAGE_CCS_E; - } else { + } else if (brw->mesa_format_supports_render[mt->format]) { mt->aux_usage = ISL_AUX_USAGE_CCS_D; } - } else if (intel_tiling_supports_hiz(brw, mt->surf.tiling) && - intel_miptree_supports_hiz(brw, mt)) { + } else if (isl_surf_usage_is_depth(mt->surf.usage) && brw->has_hiz) { mt->aux_usage = ISL_AUX_USAGE_HIZ; } @@ -513,7 +309,7 @@ free_aux_state_map(enum isl_aux_state **state) } static bool -need_to_retile_as_linear(struct brw_context *brw, unsigned row_pitch, +need_to_retile_as_linear(struct brw_context *brw, unsigned blt_pitch, enum isl_tiling tiling, unsigned samples) { if (samples > 1) @@ -522,13 +318,9 @@ need_to_retile_as_linear(struct brw_context *brw, unsigned row_pitch, if (tiling == ISL_TILING_LINEAR) return false; - /* If the width is much smaller than a tile, don't bother tiling. */ - if (row_pitch < 64) - return true; - - if (ALIGN(row_pitch, 512) >= 32768) { - perf_debug("row pitch %u too large to blit, falling back to untiled", - row_pitch); + if (blt_pitch >= 32768) { + perf_debug("blt pitch %u too large to blit, falling back to untiled", + blt_pitch); return true; } @@ -558,7 +350,7 @@ make_surface(struct brw_context *brw, GLenum target, mesa_format format, unsigned width0, unsigned height0, unsigned depth0, unsigned num_samples, isl_tiling_flags_t tiling_flags, isl_surf_usage_flags_t isl_usage_flags, uint32_t alloc_flags, - unsigned row_pitch, struct brw_bo *bo) + unsigned row_pitch_B, struct brw_bo *bo) { struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1); if (!mt) @@ -592,7 +384,7 @@ make_surface(struct brw_context *brw, GLenum target, mesa_format format, .levels = last_level - first_level + 1, .array_len = target == GL_TEXTURE_3D ? 1 : depth0, .samples = num_samples, - .row_pitch = row_pitch, + .row_pitch_B = row_pitch_B, .usage = isl_usage_flags, .tiling_flags = tiling_flags, }; @@ -608,12 +400,12 @@ make_surface(struct brw_context *brw, GLenum target, mesa_format format, bool is_depth_stencil = mt->surf.usage & (ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_DEPTH_BIT); if (!is_depth_stencil) { - if (need_to_retile_as_linear(brw, mt->surf.row_pitch, + if (need_to_retile_as_linear(brw, intel_miptree_blt_pitch(mt), mt->surf.tiling, mt->surf.samples)) { init_info.tiling_flags = 1u << ISL_TILING_LINEAR; if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info)) goto fail; - } else if (need_to_retile_as_x(brw, mt->surf.size, mt->surf.tiling)) { + } else if (need_to_retile_as_x(brw, mt->surf.size_B, mt->surf.tiling)) { init_info.tiling_flags = 1u << ISL_TILING_X; if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info)) goto fail; @@ -625,14 +417,15 @@ make_surface(struct brw_context *brw, GLenum target, mesa_format format, * See isl_apply_surface_padding(). */ if (mt->surf.tiling != ISL_TILING_LINEAR) - assert(mt->surf.size % mt->surf.row_pitch == 0); + assert(mt->surf.size_B % mt->surf.row_pitch_B == 0); if (!bo) { mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "isl-miptree", - mt->surf.size, + mt->surf.size_B, + BRW_MEMZONE_OTHER, isl_tiling_to_i915_tiling( mt->surf.tiling), - mt->surf.row_pitch, alloc_flags); + mt->surf.row_pitch_B, alloc_flags); if (!mt->bo) goto fail; } else { @@ -655,28 +448,21 @@ fail: return NULL; } -static bool -make_separate_stencil_surface(struct brw_context *brw, - struct intel_mipmap_tree *mt) +/* Return the usual surface usage flags for the given format. */ +static isl_surf_usage_flags_t +mt_surf_usage(mesa_format format) { - mt->stencil_mt = make_surface(brw, mt->target, MESA_FORMAT_S_UINT8, - 0, mt->surf.levels - 1, - mt->surf.logical_level0_px.width, - mt->surf.logical_level0_px.height, - mt->surf.dim == ISL_SURF_DIM_3D ? - mt->surf.logical_level0_px.depth : - mt->surf.logical_level0_px.array_len, - mt->surf.samples, ISL_TILING_W_BIT, - ISL_SURF_USAGE_STENCIL_BIT | - ISL_SURF_USAGE_TEXTURE_BIT, - BO_ALLOC_BUSY, 0, NULL); - - if (!mt->stencil_mt) - return false; - - mt->stencil_mt->r8stencil_needs_update = true; - - return true; + switch(_mesa_get_format_base_format(format)) { + case GL_DEPTH_COMPONENT: + return ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT; + case GL_DEPTH_STENCIL: + return ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_STENCIL_BIT | + ISL_SURF_USAGE_TEXTURE_BIT; + case GL_STENCIL_INDEX: + return ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_TEXTURE_BIT; + default: + return ISL_SURF_USAGE_RENDER_TARGET_BIT | ISL_SURF_USAGE_TEXTURE_BIT; + } } static struct intel_mipmap_tree * @@ -692,75 +478,56 @@ miptree_create(struct brw_context *brw, enum intel_miptree_create_flags flags) { const struct gen_device_info *devinfo = &brw->screen->devinfo; + const uint32_t alloc_flags = + (flags & MIPTREE_CREATE_BUSY || num_samples > 1) ? BO_ALLOC_BUSY : 0; + isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK; - if (format == MESA_FORMAT_S_UINT8) - return make_surface(brw, target, format, first_level, last_level, - width0, height0, depth0, num_samples, - ISL_TILING_W_BIT, - ISL_SURF_USAGE_STENCIL_BIT | - ISL_SURF_USAGE_TEXTURE_BIT, - BO_ALLOC_BUSY, - 0, - NULL); + /* TODO: This used to be because there wasn't BLORP to handle Y-tiling. */ + if (devinfo->gen < 6 && _mesa_is_format_color_format(format)) + tiling_flags &= ~ISL_TILING_Y0_BIT; - const GLenum base_format = _mesa_get_format_base_format(format); - if ((base_format == GL_DEPTH_COMPONENT || - base_format == GL_DEPTH_STENCIL) && - !(flags & MIPTREE_CREATE_LINEAR)) { + mesa_format mt_fmt = format; + if (!_mesa_is_format_color_format(format) && devinfo->gen >= 6) { /* Fix up the Z miptree format for how we're splitting out separate - * stencil. Gen7 expects there to be no stencil bits in its depth buffer. + * stencil. Gen7 expects there to be no stencil bits in its depth buffer. */ - const mesa_format depth_only_format = - intel_depth_format_for_depthstencil_format(format); - struct intel_mipmap_tree *mt = make_surface( - brw, target, devinfo->gen >= 6 ? depth_only_format : format, - first_level, last_level, - width0, height0, depth0, num_samples, ISL_TILING_Y0_BIT, - ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT, - BO_ALLOC_BUSY, 0, NULL); - - if (needs_separate_stencil(brw, mt, format) && - !make_separate_stencil_surface(brw, mt)) { - intel_miptree_release(&mt); - return NULL; - } - - if (!(flags & MIPTREE_CREATE_NO_AUX)) - intel_miptree_choose_aux_usage(brw, mt); - - return mt; + mt_fmt = intel_depth_format_for_depthstencil_format(format); } - mesa_format tex_format = format; - mesa_format etc_format = MESA_FORMAT_NONE; - uint32_t alloc_flags = 0; - - format = intel_lower_compressed_format(brw, format); - - etc_format = (format != tex_format) ? tex_format : MESA_FORMAT_NONE; - - if (flags & MIPTREE_CREATE_BUSY) - alloc_flags |= BO_ALLOC_BUSY; + struct intel_mipmap_tree *mt = + make_surface(brw, target, mt_fmt, first_level, last_level, + width0, height0, depth0, num_samples, + tiling_flags, mt_surf_usage(mt_fmt), + alloc_flags, 0, NULL); - isl_tiling_flags_t tiling_flags = (flags & MIPTREE_CREATE_LINEAR) ? - ISL_TILING_LINEAR_BIT : ISL_TILING_ANY_MASK; + if (mt == NULL) + return NULL; - /* TODO: This used to be because there wasn't BLORP to handle Y-tiling. */ - if (devinfo->gen < 6) - tiling_flags &= ~ISL_TILING_Y0_BIT; + if (intel_miptree_needs_fake_etc(brw, mt)) { + mesa_format decomp_format = intel_lower_compressed_format(brw, format); + mt->shadow_mt = make_surface(brw, target, decomp_format, first_level, + last_level, width0, height0, depth0, + num_samples, tiling_flags, + mt_surf_usage(decomp_format), + alloc_flags, 0, NULL); - struct intel_mipmap_tree *mt = make_surface( - brw, target, format, - first_level, last_level, - width0, height0, depth0, - num_samples, tiling_flags, - ISL_SURF_USAGE_RENDER_TARGET_BIT | - ISL_SURF_USAGE_TEXTURE_BIT, - alloc_flags, 0, NULL); - if (!mt) - return NULL; + if (mt->shadow_mt == NULL) { + intel_miptree_release(&mt); + return NULL; + } + } - mt->etc_format = etc_format; + if (needs_separate_stencil(brw, mt, format)) { + mt->stencil_mt = + make_surface(brw, target, MESA_FORMAT_S_UINT8, first_level, last_level, + width0, height0, depth0, num_samples, + ISL_TILING_W_BIT, mt_surf_usage(MESA_FORMAT_S_UINT8), + alloc_flags, 0, NULL); + if (mt->stencil_mt == NULL) { + intel_miptree_release(&mt); + return NULL; + } + } if (!(flags & MIPTREE_CREATE_NO_AUX)) intel_miptree_choose_aux_usage(brw, mt); @@ -792,9 +559,14 @@ intel_miptree_create(struct brw_context *brw, mt->offset = 0; - if (!intel_miptree_alloc_aux(brw, mt)) { - intel_miptree_release(&mt); - return NULL; + /* Create the auxiliary surface up-front. CCS_D, on the other hand, can only + * compress clear color so we wait until an actual fast-clear to allocate + * it. + */ + if (mt->aux_usage != ISL_AUX_USAGE_CCS_D && + !intel_miptree_alloc_aux(brw, mt)) { + mt->aux_usage = ISL_AUX_USAGE_NONE; + mt->supports_fast_clear = false; } return mt; @@ -809,23 +581,22 @@ intel_miptree_create_for_bo(struct brw_context *brw, uint32_t height, uint32_t depth, int pitch, + enum isl_tiling tiling, enum intel_miptree_create_flags flags) { const struct gen_device_info *devinfo = &brw->screen->devinfo; struct intel_mipmap_tree *mt; - uint32_t tiling, swizzle; const GLenum target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D; const GLenum base_format = _mesa_get_format_base_format(format); if ((base_format == GL_DEPTH_COMPONENT || base_format == GL_DEPTH_STENCIL)) { - const mesa_format depth_only_format = + const mesa_format mt_fmt = (devinfo->gen < 6) ? format : intel_depth_format_for_depthstencil_format(format); - mt = make_surface(brw, target, - devinfo->gen >= 6 ? depth_only_format : format, + mt = make_surface(brw, target, mt_fmt, 0, 0, width, height, depth, 1, ISL_TILING_Y0_BIT, - ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT, - BO_ALLOC_BUSY, pitch, bo); + mt_surf_usage(mt_fmt), + 0, pitch, bo); if (!mt) return NULL; @@ -839,24 +610,21 @@ intel_miptree_create_for_bo(struct brw_context *brw, mt = make_surface(brw, target, MESA_FORMAT_S_UINT8, 0, 0, width, height, depth, 1, ISL_TILING_W_BIT, - ISL_SURF_USAGE_STENCIL_BIT | - ISL_SURF_USAGE_TEXTURE_BIT, - BO_ALLOC_BUSY, pitch, bo); + mt_surf_usage(MESA_FORMAT_S_UINT8), + 0, pitch, bo); if (!mt) return NULL; - assert(bo->size >= mt->surf.size); + assert(bo->size >= mt->surf.size_B); brw_bo_reference(bo); return mt; } - brw_bo_get_tiling(bo, &tiling, &swizzle); - /* Nothing will be able to use this miptree with the BO if the offset isn't * aligned. */ - if (tiling != I915_TILING_NONE) + if (tiling != ISL_TILING_LINEAR) assert(offset % 4096 == 0); /* miptrees can't handle negative pitch. If you need flipping of images, @@ -864,16 +632,10 @@ intel_miptree_create_for_bo(struct brw_context *brw, */ assert(pitch >= 0); - /* The BO already has a tiling format and we shouldn't confuse the lower - * layers by making it try to find a tiling format again. - */ - assert((flags & MIPTREE_CREATE_LINEAR) == 0); - mt = make_surface(brw, target, format, 0, 0, width, height, depth, 1, - 1lu << isl_tiling_from_i915_tiling(tiling), - ISL_SURF_USAGE_RENDER_TARGET_BIT | - ISL_SURF_USAGE_TEXTURE_BIT, + 1lu << tiling, + mt_surf_usage(format), 0, pitch, bo); if (!mt) return NULL; @@ -885,9 +647,14 @@ intel_miptree_create_for_bo(struct brw_context *brw, if (!(flags & MIPTREE_CREATE_NO_AUX)) { intel_miptree_choose_aux_usage(brw, mt); - if (!intel_miptree_alloc_aux(brw, mt)) { - intel_miptree_release(&mt); - return NULL; + /* Create the auxiliary surface up-front. CCS_D, on the other hand, can + * only compress clear color so we wait until an actual fast-clear to + * allocate it. + */ + if (mt->aux_usage != ISL_AUX_USAGE_CCS_D && + !intel_miptree_alloc_aux(brw, mt)) { + mt->aux_usage = ISL_AUX_USAGE_NONE; + mt->supports_fast_clear = false; } } @@ -896,7 +663,8 @@ intel_miptree_create_for_bo(struct brw_context *brw, static struct intel_mipmap_tree * miptree_create_for_planar_image(struct brw_context *brw, - __DRIimage *image, GLenum target) + __DRIimage *image, GLenum target, + enum isl_tiling tiling) { const struct intel_image_format *f = image->planar_format; struct intel_mipmap_tree *planar_mt = NULL; @@ -918,9 +686,12 @@ miptree_create_for_planar_image(struct brw_context *brw, image->offsets[index], width, height, 1, image->strides[index], + tiling, MIPTREE_CREATE_NO_AUX); - if (mt == NULL) + if (mt == NULL) { + intel_miptree_release(&planar_mt); return NULL; + } mt->target = target; @@ -954,34 +725,49 @@ create_ccs_buf_for_image(struct brw_context *brw, assert(mt->last_level == 0); /* We shouldn't already have a CCS */ - assert(!mt->mcs_buf); + assert(!mt->aux_buf); if (!isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &temp_ccs_surf, image->aux_pitch)) return false; assert(image->aux_offset < image->bo->size); - assert(temp_ccs_surf.size <= image->bo->size - image->aux_offset); + assert(temp_ccs_surf.size_B <= image->bo->size - image->aux_offset); - mt->mcs_buf = calloc(sizeof(*mt->mcs_buf), 1); - if (mt->mcs_buf == NULL) + mt->aux_buf = calloc(sizeof(*mt->aux_buf), 1); + if (mt->aux_buf == NULL) return false; mt->aux_state = create_aux_state_map(mt, initial_state); if (!mt->aux_state) { - free(mt->mcs_buf); - mt->mcs_buf = NULL; + free(mt->aux_buf); + mt->aux_buf = NULL; return false; } - mt->mcs_buf->bo = image->bo; + /* On gen10+ we start using an extra space in the aux buffer to store the + * indirect clear color. However, if we imported an image from the window + * system with CCS, we don't have the extra space at the end of the aux + * buffer. So create a new bo here that will store that clear color. + */ + if (brw->isl_dev.ss.clear_color_state_size > 0) { + mt->aux_buf->clear_color_bo = + brw_bo_alloc_tiled(brw->bufmgr, "clear_color_bo", + brw->isl_dev.ss.clear_color_state_size, + BRW_MEMZONE_OTHER, I915_TILING_NONE, 0, + BO_ALLOC_ZEROED); + if (!mt->aux_buf->clear_color_bo) { + free(mt->aux_buf); + mt->aux_buf = NULL; + return false; + } + } + + mt->aux_buf->bo = image->bo; brw_bo_reference(image->bo); - mt->mcs_buf->offset = image->aux_offset; - mt->mcs_buf->size = image->bo->size - image->aux_offset; - mt->mcs_buf->pitch = image->aux_pitch; - mt->mcs_buf->qpitch = 0; - mt->mcs_buf->surf = temp_ccs_surf; + mt->aux_buf->offset = image->aux_offset; + mt->aux_buf->surf = temp_ccs_surf; return true; } @@ -990,10 +776,19 @@ struct intel_mipmap_tree * intel_miptree_create_for_dri_image(struct brw_context *brw, __DRIimage *image, GLenum target, mesa_format format, - bool is_winsys_image) + bool allow_internal_aux) { + uint32_t bo_tiling, bo_swizzle; + brw_bo_get_tiling(image->bo, &bo_tiling, &bo_swizzle); + + const struct isl_drm_modifier_info *mod_info = + isl_drm_modifier_get_info(image->modifier); + + const enum isl_tiling tiling = + mod_info ? mod_info->tiling : isl_tiling_from_i915_tiling(bo_tiling); + if (image->planar_format && image->planar_format->nplanes > 1) - return miptree_create_for_planar_image(brw, image, target); + return miptree_create_for_planar_image(brw, image, target, tiling); if (image->planar_format) assert(image->planar_format->planes[0].dri_format == image->dri_format); @@ -1014,9 +809,6 @@ intel_miptree_create_for_dri_image(struct brw_context *brw, if (!brw->ctx.TextureFormatSupported[format]) return NULL; - const struct isl_drm_modifier_info *mod_info = - isl_drm_modifier_get_info(image->modifier); - enum intel_miptree_create_flags mt_create_flags = 0; /* If this image comes in from a window system, we have different @@ -1027,7 +819,7 @@ intel_miptree_create_for_dri_image(struct brw_context *brw, * other hand, have no resolve point so we can't have aux without a * modifier. */ - if (!is_winsys_image) + if (!allow_internal_aux) mt_create_flags |= MIPTREE_CREATE_NO_AUX; /* If we have a modifier which specifies aux, don't create one yet */ @@ -1042,7 +834,7 @@ intel_miptree_create_for_dri_image(struct brw_context *brw, struct intel_mipmap_tree *mt = intel_miptree_create_for_bo(brw, image->bo, format, image->offset, image->width, image->height, 1, - image->pitch, mt_create_flags); + image->pitch, tiling, mt_create_flags); if (mt == NULL) return NULL; @@ -1076,7 +868,7 @@ intel_miptree_create_for_dri_image(struct brw_context *brw, * as part of the flush operation. */ mt->supports_fast_clear = - is_winsys_image || mod_info->supports_clear_color; + allow_internal_aux || mod_info->supports_clear_color; /* We don't know the actual state of the surface when we get it but we * can make a pretty good guess based on the modifier. What we do know @@ -1210,6 +1002,7 @@ intel_miptree_aux_buffer_free(struct intel_miptree_aux_buffer *aux_buf) return; brw_bo_unreference(aux_buf->bo); + brw_bo_unreference(aux_buf->clear_color_bo); free(aux_buf); } @@ -1228,9 +1021,8 @@ intel_miptree_release(struct intel_mipmap_tree **mt) brw_bo_unreference((*mt)->bo); intel_miptree_release(&(*mt)->stencil_mt); - intel_miptree_release(&(*mt)->r8stencil_mt); - intel_miptree_aux_buffer_free((*mt)->hiz_buf); - intel_miptree_aux_buffer_free((*mt)->mcs_buf); + intel_miptree_release(&(*mt)->shadow_mt); + intel_miptree_aux_buffer_free((*mt)->aux_buf); free_aux_state_map((*mt)->aux_state); intel_miptree_release(&(*mt)->plane[0]); @@ -1304,10 +1096,9 @@ intel_miptree_match_image(struct intel_mipmap_tree *mt, mt_format = MESA_FORMAT_Z24_UNORM_S8_UINT; if (mt->format == MESA_FORMAT_Z_FLOAT32 && mt->stencil_mt) mt_format = MESA_FORMAT_Z32_FLOAT_S8X24_UINT; - if (mt->etc_format != MESA_FORMAT_NONE) - mt_format = mt->etc_format; - if (image->TexFormat != mt_format) + if (_mesa_get_srgb_format_linear(image->TexFormat) != + _mesa_get_srgb_format_linear(mt_format)) return false; intel_get_image_dims(image, &width, &height, &depth); @@ -1420,7 +1211,7 @@ intel_miptree_get_aligned_offset(const struct intel_mipmap_tree *mt, uint32_t x, uint32_t y) { int cpp = mt->cpp; - uint32_t pitch = mt->surf.row_pitch; + uint32_t pitch = mt->surf.row_pitch_B; switch (mt->surf.tiling) { default: @@ -1537,6 +1328,7 @@ intel_miptree_copy_slice(struct brw_context *brw, unsigned dst_level, unsigned dst_layer) { + const struct gen_device_info *devinfo = &brw->screen->devinfo; mesa_format format = src_mt->format; unsigned width = minify(src_mt->surf.phys_level0_sa.width, src_level - src_mt->first_level); @@ -1546,7 +1338,34 @@ intel_miptree_copy_slice(struct brw_context *brw, assert(src_layer < get_num_phys_layers(&src_mt->surf, src_level - src_mt->first_level)); - assert(src_mt->format == dst_mt->format); + assert(_mesa_get_srgb_format_linear(src_mt->format) == + _mesa_get_srgb_format_linear(dst_mt->format)); + + DBG("validate blit mt %s %p %d,%d -> mt %s %p %d,%d (%dx%d)\n", + _mesa_get_format_name(src_mt->format), + src_mt, src_level, src_layer, + _mesa_get_format_name(dst_mt->format), + dst_mt, dst_level, dst_layer, + width, height); + + if (devinfo->gen >= 6) { + /* On gen6 and above, we just use blorp. It's faster than the blitter + * and can handle everything without software fallbacks. + */ + brw_blorp_copy_miptrees(brw, + src_mt, src_level, src_layer, + dst_mt, dst_level, dst_layer, + 0, 0, 0, 0, width, height); + + if (src_mt->stencil_mt) { + assert(dst_mt->stencil_mt); + brw_blorp_copy_miptrees(brw, + src_mt->stencil_mt, src_level, src_layer, + dst_mt->stencil_mt, dst_level, dst_layer, + 0, 0, 0, 0, width, height); + } + return; + } if (dst_mt->compressed) { unsigned int i, j; @@ -1555,17 +1374,8 @@ intel_miptree_copy_slice(struct brw_context *brw, width = ALIGN_NPOT(width, i) / i; } - /* If it's a packed depth/stencil buffer with separate stencil, the blit - * below won't apply since we can't do the depth's Y tiling or the - * stencil's W tiling in the blitter. - */ - if (src_mt->stencil_mt) { - intel_miptree_copy_slice_sw(brw, - src_mt, src_level, src_layer, - dst_mt, dst_level, dst_layer, - width, height); - return; - } + /* Gen4-5 doesn't support separate stencil */ + assert(!src_mt->stencil_mt); uint32_t dst_x, dst_y, src_x, src_y; intel_miptree_get_image_offset(dst_mt, dst_level, dst_layer, @@ -1575,15 +1385,15 @@ intel_miptree_copy_slice(struct brw_context *brw, DBG("validate blit mt %s %p %d,%d/%d -> mt %s %p %d,%d/%d (%dx%d)\n", _mesa_get_format_name(src_mt->format), - src_mt, src_x, src_y, src_mt->surf.row_pitch, + src_mt, src_x, src_y, src_mt->surf.row_pitch_B, _mesa_get_format_name(dst_mt->format), - dst_mt, dst_x, dst_y, dst_mt->surf.row_pitch, + dst_mt, dst_x, dst_y, dst_mt->surf.row_pitch_B, width, height); if (!intel_miptree_blit(brw, src_mt, src_level, src_layer, 0, 0, false, dst_mt, dst_level, dst_layer, 0, 0, false, - width, height, GL_COPY)) { + width, height, COLOR_LOGICOP_COPY)) { perf_debug("miptree validate blit for %s failed\n", _mesa_get_format_name(format)); @@ -1634,169 +1444,99 @@ intel_miptree_copy_teximage(struct brw_context *brw, intel_obj->needs_validate = true; } -static void -intel_miptree_init_mcs(struct brw_context *brw, - struct intel_mipmap_tree *mt, - int init_value) -{ - assert(mt->mcs_buf != NULL); - - /* From the Ivy Bridge PRM, Vol 2 Part 1 p326: - * - * When MCS buffer is enabled and bound to MSRT, it is required that it - * is cleared prior to any rendering. - * - * Since we don't use the MCS buffer for any purpose other than rendering, - * it makes sense to just clear it immediately upon allocation. - * - * Note: the clear value for MCS buffers is all 1's, so we memset to 0xff. - */ - void *map = brw_bo_map(brw, mt->mcs_buf->bo, MAP_WRITE); - if (unlikely(map == NULL)) { - fprintf(stderr, "Failed to map mcs buffer into GTT\n"); - brw_bo_unreference(mt->mcs_buf->bo); - free(mt->mcs_buf); - return; - } - void *data = map; - memset(data, init_value, mt->mcs_buf->size); - brw_bo_unmap(mt->mcs_buf->bo); -} - static struct intel_miptree_aux_buffer * intel_alloc_aux_buffer(struct brw_context *brw, - const char *name, const struct isl_surf *aux_surf, - uint32_t alloc_flags, - struct intel_mipmap_tree *mt) + bool wants_memset, + uint8_t memset_value) { struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); if (!buf) return false; - buf->size = aux_surf->size; - buf->pitch = aux_surf->row_pitch; - buf->qpitch = isl_surf_get_array_pitch_sa_rows(aux_surf); + uint64_t size = aux_surf->size_B; + + const bool has_indirect_clear = brw->isl_dev.ss.clear_color_state_size > 0; + if (has_indirect_clear) { + /* On CNL+, instead of setting the clear color in the SURFACE_STATE, we + * will set a pointer to a dword somewhere that contains the color. So, + * allocate the space for the clear color value here on the aux buffer. + */ + buf->clear_color_offset = size; + size += brw->isl_dev.ss.clear_color_state_size; + } + + /* If the buffer needs to be initialised (requiring the buffer to be + * immediately mapped to cpu space for writing), do not use the gpu access + * flag which can cause an unnecessary delay if the backing pages happened + * to be just used by the GPU. + */ + const bool alloc_zeroed = wants_memset && memset_value == 0; + const bool needs_memset = + !alloc_zeroed && (wants_memset || has_indirect_clear); + const uint32_t alloc_flags = + alloc_zeroed ? BO_ALLOC_ZEROED : (needs_memset ? 0 : BO_ALLOC_BUSY); /* ISL has stricter set of alignment rules then the drm allocator. * Therefore one can pass the ISL dimensions in terms of bytes instead of * trying to recalculate based on different format block sizes. */ - buf->bo = brw_bo_alloc_tiled(brw->bufmgr, name, buf->size, - I915_TILING_Y, buf->pitch, alloc_flags); + buf->bo = brw_bo_alloc_tiled(brw->bufmgr, "aux-miptree", size, + BRW_MEMZONE_OTHER, I915_TILING_Y, + aux_surf->row_pitch_B, alloc_flags); if (!buf->bo) { free(buf); return NULL; } - buf->surf = *aux_surf; - - return buf; -} - -static bool -intel_miptree_alloc_mcs(struct brw_context *brw, - struct intel_mipmap_tree *mt, - GLuint num_samples) -{ - assert(brw->screen->devinfo.gen >= 7); /* MCS only used on Gen7+ */ - assert(mt->mcs_buf == NULL); - assert(mt->aux_usage == ISL_AUX_USAGE_MCS); + /* Initialize the bo to the desired value */ + if (needs_memset) { + assert(!(alloc_flags & BO_ALLOC_BUSY)); - /* Multisampled miptrees are only supported for single level. */ - assert(mt->first_level == 0); - enum isl_aux_state **aux_state = - create_aux_state_map(mt, ISL_AUX_STATE_CLEAR); - if (!aux_state) - return false; + void *map = brw_bo_map(brw, buf->bo, MAP_WRITE | MAP_RAW); + if (map == NULL) { + intel_miptree_aux_buffer_free(buf); + return NULL; + } - struct isl_surf temp_mcs_surf; + /* Memset the aux_surf portion of the BO. */ + if (wants_memset) + memset(map, memset_value, aux_surf->size_B); - MAYBE_UNUSED bool ok = - isl_surf_get_mcs_surf(&brw->isl_dev, &mt->surf, &temp_mcs_surf); - assert(ok); + /* Zero the indirect clear color to match ::fast_clear_color. */ + if (has_indirect_clear) { + memset((char *)map + buf->clear_color_offset, 0, + brw->isl_dev.ss.clear_color_state_size); + } - /* Buffer needs to be initialised requiring the buffer to be immediately - * mapped to cpu space for writing. Therefore do not use the gpu access - * flag which can cause an unnecessary delay if the backing pages happened - * to be just used by the GPU. - */ - const uint32_t alloc_flags = 0; - mt->mcs_buf = intel_alloc_aux_buffer(brw, "mcs-miptree", - &temp_mcs_surf, alloc_flags, mt); - if (!mt->mcs_buf) { - free(aux_state); - return false; + brw_bo_unmap(buf->bo); } - mt->aux_state = aux_state; + if (has_indirect_clear) { + buf->clear_color_bo = buf->bo; + brw_bo_reference(buf->clear_color_bo); + } - intel_miptree_init_mcs(brw, mt, 0xFF); + buf->surf = *aux_surf; - return true; + return buf; } -bool -intel_miptree_alloc_ccs(struct brw_context *brw, - struct intel_mipmap_tree *mt) + +/** + * Helper for intel_miptree_alloc_aux() that sets + * \c mt->level[level].has_hiz. Return true if and only if + * \c has_hiz was set. + */ +static bool +intel_miptree_level_enable_hiz(struct brw_context *brw, + struct intel_mipmap_tree *mt, + uint32_t level) { - assert(mt->mcs_buf == NULL); - assert(mt->aux_usage == ISL_AUX_USAGE_CCS_E || - mt->aux_usage == ISL_AUX_USAGE_CCS_D); + const struct gen_device_info *devinfo = &brw->screen->devinfo; - struct isl_surf temp_ccs_surf; - - if (!isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &temp_ccs_surf, 0)) - return false; - - assert(temp_ccs_surf.size && - (temp_ccs_surf.size % temp_ccs_surf.row_pitch == 0)); - - enum isl_aux_state **aux_state = - create_aux_state_map(mt, ISL_AUX_STATE_PASS_THROUGH); - if (!aux_state) - return false; - - /* When CCS_E is used, we need to ensure that the CCS starts off in a valid - * state. From the Sky Lake PRM, "MCS Buffer for Render Target(s)": - * - * "If Software wants to enable Color Compression without Fast clear, - * Software needs to initialize MCS with zeros." - * - * A CCS value of 0 indicates that the corresponding block is in the - * pass-through state which is what we want. - * - * For CCS_D, on the other hand, we don't care as we're about to perform a - * fast-clear operation. In that case, being hot in caches more useful. - */ - const uint32_t alloc_flags = mt->aux_usage == ISL_AUX_USAGE_CCS_E ? - BO_ALLOC_ZEROED : BO_ALLOC_BUSY; - mt->mcs_buf = intel_alloc_aux_buffer(brw, "ccs-miptree", - &temp_ccs_surf, alloc_flags, mt); - if (!mt->mcs_buf) { - free(aux_state); - return false; - } - - mt->aux_state = aux_state; - - return true; -} - -/** - * Helper for intel_miptree_alloc_hiz() that sets - * \c mt->level[level].has_hiz. Return true if and only if - * \c has_hiz was set. - */ -static bool -intel_miptree_level_enable_hiz(struct brw_context *brw, - struct intel_mipmap_tree *mt, - uint32_t level) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - assert(mt->hiz_buf); - assert(mt->surf.size > 0); + assert(mt->aux_buf); + assert(mt->surf.size_B > 0); if (devinfo->gen >= 8 || devinfo->is_haswell) { uint32_t width = minify(mt->surf.phys_level0_sa.width, level); @@ -1819,41 +1559,6 @@ intel_miptree_level_enable_hiz(struct brw_context *brw, return true; } -bool -intel_miptree_alloc_hiz(struct brw_context *brw, - struct intel_mipmap_tree *mt) -{ - assert(mt->hiz_buf == NULL); - assert(mt->aux_usage == ISL_AUX_USAGE_HIZ); - - enum isl_aux_state **aux_state = - create_aux_state_map(mt, ISL_AUX_STATE_AUX_INVALID); - if (!aux_state) - return false; - - struct isl_surf temp_hiz_surf; - - MAYBE_UNUSED bool ok = - isl_surf_get_hiz_surf(&brw->isl_dev, &mt->surf, &temp_hiz_surf); - assert(ok); - - const uint32_t alloc_flags = BO_ALLOC_BUSY; - mt->hiz_buf = intel_alloc_aux_buffer(brw, "hiz-miptree", - &temp_hiz_surf, alloc_flags, mt); - - if (!mt->hiz_buf) { - free(aux_state); - return false; - } - - for (unsigned level = mt->first_level; level <= mt->last_level; ++level) - intel_miptree_level_enable_hiz(brw, mt, level); - - mt->aux_state = aux_state; - - return true; -} - /** * Allocate the initial aux surface for a miptree based on mt->aux_usage @@ -1862,42 +1567,97 @@ intel_miptree_alloc_hiz(struct brw_context *brw, * create the auxiliary surfaces up-front. CCS_D, on the other hand, can only * compress clear color so we wait until an actual fast-clear to allocate it. */ -static bool +bool intel_miptree_alloc_aux(struct brw_context *brw, struct intel_mipmap_tree *mt) { + assert(mt->aux_buf == NULL); + + /* Get the aux buf allocation parameters for this miptree. */ + enum isl_aux_state initial_state; + uint8_t memset_value; + struct isl_surf aux_surf; + MAYBE_UNUSED bool aux_surf_ok = false; + switch (mt->aux_usage) { case ISL_AUX_USAGE_NONE: - return true; - + aux_surf.size_B = 0; + aux_surf_ok = true; + break; case ISL_AUX_USAGE_HIZ: - assert(!_mesa_is_format_color_format(mt->format)); - if (!intel_miptree_alloc_hiz(brw, mt)) - return false; - return true; - + initial_state = ISL_AUX_STATE_AUX_INVALID; + memset_value = 0; + aux_surf_ok = isl_surf_get_hiz_surf(&brw->isl_dev, &mt->surf, &aux_surf); + break; case ISL_AUX_USAGE_MCS: - assert(_mesa_is_format_color_format(mt->format)); - assert(mt->surf.samples > 1); - if (!intel_miptree_alloc_mcs(brw, mt, mt->surf.samples)) - return false; - return true; - + /* From the Ivy Bridge PRM, Vol 2 Part 1 p326: + * + * When MCS buffer is enabled and bound to MSRT, it is required that + * it is cleared prior to any rendering. + * + * Since we don't use the MCS buffer for any purpose other than + * rendering, it makes sense to just clear it immediately upon + * allocation. + * + * Note: the clear value for MCS buffers is all 1's, so we memset to + * 0xff. + */ + initial_state = ISL_AUX_STATE_CLEAR; + memset_value = 0xFF; + aux_surf_ok = isl_surf_get_mcs_surf(&brw->isl_dev, &mt->surf, &aux_surf); + break; case ISL_AUX_USAGE_CCS_D: - /* Since CCS_D can only compress clear color so we wait until an actual - * fast-clear to allocate it. + case ISL_AUX_USAGE_CCS_E: + /* When CCS_E is used, we need to ensure that the CCS starts off in a + * valid state. From the Sky Lake PRM, "MCS Buffer for Render + * Target(s)": + * + * "If Software wants to enable Color Compression without Fast + * clear, Software needs to initialize MCS with zeros." + * + * A CCS value of 0 indicates that the corresponding block is in the + * pass-through state which is what we want. + * + * For CCS_D, do the same thing. On gen9+, this avoids having any + * undefined bits in the aux buffer. */ - return true; + initial_state = ISL_AUX_STATE_PASS_THROUGH; + memset_value = 0; + aux_surf_ok = + isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &aux_surf, 0); + break; + } - case ISL_AUX_USAGE_CCS_E: - assert(_mesa_is_format_color_format(mt->format)); - assert(mt->surf.samples == 1); - if (!intel_miptree_alloc_ccs(brw, mt)) - return false; + /* We should have a valid aux_surf. */ + if (!aux_surf_ok) + return false; + + /* No work is needed for a zero-sized auxiliary buffer. */ + if (aux_surf.size_B == 0) return true; + + /* Create the aux_state for the auxiliary buffer. */ + mt->aux_state = create_aux_state_map(mt, initial_state); + if (mt->aux_state == NULL) + return false; + + /* Allocate the auxiliary buffer. */ + const bool needs_memset = initial_state != ISL_AUX_STATE_AUX_INVALID; + mt->aux_buf = intel_alloc_aux_buffer(brw, &aux_surf, needs_memset, + memset_value); + if (mt->aux_buf == NULL) { + free_aux_state_map(mt->aux_state); + mt->aux_state = NULL; + return false; + } + + /* Perform aux_usage-specific initialization. */ + if (mt->aux_usage == ISL_AUX_USAGE_HIZ) { + for (unsigned level = mt->first_level; level <= mt->last_level; ++level) + intel_miptree_level_enable_hiz(brw, mt, level); } - unreachable("Invalid aux usage"); + return true; } @@ -1910,14 +1670,11 @@ intel_miptree_sample_with_hiz(struct brw_context *brw, { const struct gen_device_info *devinfo = &brw->screen->devinfo; - /* It's unclear how well supported sampling from the hiz buffer is on GEN8, - * so keep things conservative for now and never enable it unless we're SKL+. - */ - if (devinfo->gen < 9) { + if (!devinfo->has_sample_with_hiz) { return false; } - if (!mt->hiz_buf) { + if (!mt->aux_buf) { return false; } @@ -1997,7 +1754,7 @@ intel_miptree_has_color_unresolved(const struct intel_mipmap_tree *mt, { assert(_mesa_is_format_color_format(mt->format)); - if (!mt->mcs_buf) + if (!mt->aux_buf) return false; /* Clamp the level range to fit the miptree */ @@ -2024,7 +1781,7 @@ intel_miptree_check_color_resolve(const struct brw_context *brw, const struct intel_mipmap_tree *mt, unsigned level, unsigned layer) { - if (!mt->mcs_buf) + if (!mt->aux_buf) return; /* Fast color clear is supported for mipmapped surfaces only on Gen8+. */ @@ -2045,7 +1802,7 @@ intel_miptree_check_color_resolve(const struct brw_context *brw, (void)layer; } -static enum blorp_fast_clear_op +static enum isl_aux_op get_ccs_d_resolve_op(enum isl_aux_state aux_state, enum isl_aux_usage aux_usage, bool fast_clear_supported) @@ -2060,12 +1817,12 @@ get_ccs_d_resolve_op(enum isl_aux_state aux_state, case ISL_AUX_STATE_CLEAR: case ISL_AUX_STATE_PARTIAL_CLEAR: if (!ccs_supported) - return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + return ISL_AUX_OP_FULL_RESOLVE; else - return BLORP_FAST_CLEAR_OP_NONE; + return ISL_AUX_OP_NONE; case ISL_AUX_STATE_PASS_THROUGH: - return BLORP_FAST_CLEAR_OP_NONE; + return ISL_AUX_OP_NONE; case ISL_AUX_STATE_RESOLVED: case ISL_AUX_STATE_AUX_INVALID: @@ -2077,7 +1834,7 @@ get_ccs_d_resolve_op(enum isl_aux_state aux_state, unreachable("Invalid aux state for CCS_D"); } -static enum blorp_fast_clear_op +static enum isl_aux_op get_ccs_e_resolve_op(enum isl_aux_state aux_state, enum isl_aux_usage aux_usage, bool fast_clear_supported) @@ -2094,28 +1851,28 @@ get_ccs_e_resolve_op(enum isl_aux_state aux_state, case ISL_AUX_STATE_CLEAR: case ISL_AUX_STATE_PARTIAL_CLEAR: if (fast_clear_supported) - return BLORP_FAST_CLEAR_OP_NONE; + return ISL_AUX_OP_NONE; else if (aux_usage == ISL_AUX_USAGE_CCS_E) - return BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL; + return ISL_AUX_OP_PARTIAL_RESOLVE; else - return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + return ISL_AUX_OP_FULL_RESOLVE; case ISL_AUX_STATE_COMPRESSED_CLEAR: if (aux_usage != ISL_AUX_USAGE_CCS_E) - return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + return ISL_AUX_OP_FULL_RESOLVE; else if (!fast_clear_supported) - return BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL; + return ISL_AUX_OP_PARTIAL_RESOLVE; else - return BLORP_FAST_CLEAR_OP_NONE; + return ISL_AUX_OP_NONE; case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: if (aux_usage != ISL_AUX_USAGE_CCS_E) - return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + return ISL_AUX_OP_FULL_RESOLVE; else - return BLORP_FAST_CLEAR_OP_NONE; + return ISL_AUX_OP_NONE; case ISL_AUX_STATE_PASS_THROUGH: - return BLORP_FAST_CLEAR_OP_NONE; + return ISL_AUX_OP_NONE; case ISL_AUX_STATE_RESOLVED: case ISL_AUX_STATE_AUX_INVALID: @@ -2134,7 +1891,7 @@ intel_miptree_prepare_ccs_access(struct brw_context *brw, { enum isl_aux_state aux_state = intel_miptree_get_aux_state(mt, level, layer); - enum blorp_fast_clear_op resolve_op; + enum isl_aux_op resolve_op; if (mt->aux_usage == ISL_AUX_USAGE_CCS_E) { resolve_op = get_ccs_e_resolve_op(aux_state, aux_usage, fast_clear_supported); @@ -2144,12 +1901,12 @@ intel_miptree_prepare_ccs_access(struct brw_context *brw, fast_clear_supported); } - if (resolve_op != BLORP_FAST_CLEAR_OP_NONE) { + if (resolve_op != ISL_AUX_OP_NONE) { intel_miptree_check_color_resolve(brw, mt, level, layer); brw_blorp_resolve_color(brw, mt, level, layer, resolve_op); switch (resolve_op) { - case BLORP_FAST_CLEAR_OP_RESOLVE_FULL: + case ISL_AUX_OP_FULL_RESOLVE: /* The CCS full resolve operation destroys the CCS and sets it to the * pass-through state. (You can also think of this as being both a * resolve and an ambiguate in one operation.) @@ -2158,7 +1915,7 @@ intel_miptree_prepare_ccs_access(struct brw_context *brw, ISL_AUX_STATE_PASS_THROUGH); break; - case BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL: + case ISL_AUX_OP_PARTIAL_RESOLVE: intel_miptree_set_aux_state(brw, mt, level, layer, 1, ISL_AUX_STATE_COMPRESSED_NO_CLEAR); break; @@ -2307,17 +2064,17 @@ intel_miptree_prepare_hiz_access(struct brw_context *brw, { assert(aux_usage == ISL_AUX_USAGE_NONE || aux_usage == ISL_AUX_USAGE_HIZ); - enum blorp_hiz_op hiz_op = BLORP_HIZ_OP_NONE; + enum isl_aux_op hiz_op = ISL_AUX_OP_NONE; switch (intel_miptree_get_aux_state(mt, level, layer)) { case ISL_AUX_STATE_CLEAR: case ISL_AUX_STATE_COMPRESSED_CLEAR: if (aux_usage != ISL_AUX_USAGE_HIZ || !fast_clear_supported) - hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE; + hiz_op = ISL_AUX_OP_FULL_RESOLVE; break; case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: if (aux_usage != ISL_AUX_USAGE_HIZ) - hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE; + hiz_op = ISL_AUX_OP_FULL_RESOLVE; break; case ISL_AUX_STATE_PASS_THROUGH: @@ -2326,23 +2083,23 @@ intel_miptree_prepare_hiz_access(struct brw_context *brw, case ISL_AUX_STATE_AUX_INVALID: if (aux_usage == ISL_AUX_USAGE_HIZ) - hiz_op = BLORP_HIZ_OP_HIZ_RESOLVE; + hiz_op = ISL_AUX_OP_AMBIGUATE; break; case ISL_AUX_STATE_PARTIAL_CLEAR: unreachable("Invalid HiZ state"); } - if (hiz_op != BLORP_HIZ_OP_NONE) { + if (hiz_op != ISL_AUX_OP_NONE) { intel_hiz_exec(brw, mt, level, layer, 1, hiz_op); switch (hiz_op) { - case BLORP_HIZ_OP_DEPTH_RESOLVE: + case ISL_AUX_OP_FULL_RESOLVE: intel_miptree_set_aux_state(brw, mt, level, layer, 1, ISL_AUX_STATE_RESOLVED); break; - case BLORP_HIZ_OP_HIZ_RESOLVE: + case ISL_AUX_OP_AMBIGUATE: /* The HiZ resolve operation is actually an ambiguate */ intel_miptree_set_aux_state(brw, mt, level, layer, 1, ISL_AUX_STATE_PASS_THROUGH); @@ -2416,7 +2173,7 @@ intel_miptree_prepare_access(struct brw_context *brw, break; case ISL_AUX_USAGE_MCS: - assert(mt->mcs_buf); + assert(mt->aux_buf); assert(start_level == 0 && num_levels == 1); const uint32_t level_layers = miptree_layer_range_length(mt, 0, start_layer, num_layers); @@ -2428,7 +2185,7 @@ intel_miptree_prepare_access(struct brw_context *brw, case ISL_AUX_USAGE_CCS_D: case ISL_AUX_USAGE_CCS_E: - if (!mt->mcs_buf) + if (!mt->aux_buf) return; for (uint32_t l = 0; l < num_levels; l++) { @@ -2444,7 +2201,7 @@ intel_miptree_prepare_access(struct brw_context *brw, break; case ISL_AUX_USAGE_HIZ: - assert(mt->hiz_buf); + assert(mt->aux_buf); for (uint32_t l = 0; l < num_levels; l++) { const uint32_t level = start_level + l; if (!intel_miptree_level_has_hiz(mt, level)) @@ -2470,15 +2227,20 @@ intel_miptree_finish_write(struct brw_context *brw, uint32_t start_layer, uint32_t num_layers, enum isl_aux_usage aux_usage) { + const struct gen_device_info *devinfo = &brw->screen->devinfo; num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers); switch (mt->aux_usage) { case ISL_AUX_USAGE_NONE: - /* Nothing to do */ + if (mt->format == MESA_FORMAT_S_UINT8 && devinfo->gen <= 7) { + mt->shadow_needs_update = true; + } else if (intel_miptree_has_etc_shadow(brw, mt)) { + mt->shadow_needs_update = true; + } break; case ISL_AUX_USAGE_MCS: - assert(mt->mcs_buf); + assert(mt->aux_buf); for (uint32_t a = 0; a < num_layers; a++) { intel_miptree_finish_mcs_write(brw, mt, start_layer + a, aux_usage); @@ -2487,7 +2249,7 @@ intel_miptree_finish_write(struct brw_context *brw, case ISL_AUX_USAGE_CCS_D: case ISL_AUX_USAGE_CCS_E: - if (!mt->mcs_buf) + if (!mt->aux_buf) return; for (uint32_t a = 0; a < num_layers; a++) { @@ -2518,7 +2280,7 @@ intel_miptree_get_aux_state(const struct intel_mipmap_tree *mt, intel_miptree_check_level_layer(mt, level, layer); if (_mesa_is_format_color_format(mt->format)) { - assert(mt->mcs_buf != NULL); + assert(mt->aux_buf != NULL); assert(mt->surf.samples == 1 || mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY); } else if (mt->format == MESA_FORMAT_S_UINT8) { @@ -2539,7 +2301,7 @@ intel_miptree_set_aux_state(struct brw_context *brw, num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers); if (_mesa_is_format_color_format(mt->format)) { - assert(mt->mcs_buf != NULL); + assert(mt->aux_buf != NULL); assert(mt->surf.samples == 1 || mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY); } else if (mt->format == MESA_FORMAT_S_UINT8) { @@ -2575,9 +2337,8 @@ can_texture_with_ccs(struct brw_context *brw, if (mt->aux_usage != ISL_AUX_USAGE_CCS_E) return false; - /* TODO: Replace with format_ccs_e_compat_with_miptree for better perf. */ - if (!isl_formats_are_ccs_e_compatible(&brw->screen->devinfo, - mt->surf.format, view_format)) { + if (!format_ccs_e_compat_with_miptree(&brw->screen->devinfo, + mt, view_format)) { perf_debug("Incompatible sampling format (%s) for rbc (%s)\n", isl_format_get_layout(view_format)->name, _mesa_get_format_name(mt->format)); @@ -2590,8 +2351,19 @@ can_texture_with_ccs(struct brw_context *brw, enum isl_aux_usage intel_miptree_texture_aux_usage(struct brw_context *brw, struct intel_mipmap_tree *mt, - enum isl_format view_format) + enum isl_format view_format, + enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits) { + assert(brw->screen->devinfo.gen == 9 || astc5x5_wa_bits == 0); + + /* On gen9, ASTC 5x5 textures cannot live in the sampler cache along side + * CCS or HiZ compressed textures. See gen9_apply_astc5x5_wa_flush() for + * details. + */ + if ((astc5x5_wa_bits & GEN9_ASTC5X5_WA_TEX_TYPE_ASTC5x5) && + mt->aux_usage != ISL_AUX_USAGE_MCS) + return ISL_AUX_USAGE_NONE; + switch (mt->aux_usage) { case ISL_AUX_USAGE_HIZ: if (intel_miptree_sample_with_hiz(brw, mt)) @@ -2603,7 +2375,7 @@ intel_miptree_texture_aux_usage(struct brw_context *brw, case ISL_AUX_USAGE_CCS_D: case ISL_AUX_USAGE_CCS_E: - if (!mt->mcs_buf) { + if (!mt->aux_buf) { assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D); return ISL_AUX_USAGE_NONE; } @@ -2650,10 +2422,11 @@ intel_miptree_prepare_texture(struct brw_context *brw, enum isl_format view_format, uint32_t start_level, uint32_t num_levels, uint32_t start_layer, uint32_t num_layers, - bool disable_aux) + enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits) { - enum isl_aux_usage aux_usage = disable_aux ? ISL_AUX_USAGE_NONE : - intel_miptree_texture_aux_usage(brw, mt, view_format); + enum isl_aux_usage aux_usage = + intel_miptree_texture_aux_usage(brw, mt, view_format, astc5x5_wa_bits); + bool clear_supported = aux_usage != ISL_AUX_USAGE_NONE; /* Clear color is specified as ints or floats and the conversion is done by @@ -2682,38 +2455,42 @@ enum isl_aux_usage intel_miptree_render_aux_usage(struct brw_context *brw, struct intel_mipmap_tree *mt, enum isl_format render_format, - bool blend_enabled) + bool blend_enabled, + bool draw_aux_disabled) { + struct gen_device_info *devinfo = &brw->screen->devinfo; + + if (draw_aux_disabled) + return ISL_AUX_USAGE_NONE; + switch (mt->aux_usage) { case ISL_AUX_USAGE_MCS: - assert(mt->mcs_buf); + assert(mt->aux_buf); return ISL_AUX_USAGE_MCS; case ISL_AUX_USAGE_CCS_D: - /* If FRAMEBUFFER_SRGB is used on Gen9+ then we need to resolve any of - * the single-sampled color renderbuffers because the CCS buffer isn't - * supported for SRGB formats. This only matters if FRAMEBUFFER_SRGB is - * enabled because otherwise the surface state will be programmed with - * the linear equivalent format anyway. - */ - if (isl_format_is_srgb(render_format) && - _mesa_get_srgb_format_linear(mt->format) != mt->format) { - return ISL_AUX_USAGE_NONE; - } else if (!mt->mcs_buf) { + case ISL_AUX_USAGE_CCS_E: + if (!mt->aux_buf) { + assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D); return ISL_AUX_USAGE_NONE; - } else { - return ISL_AUX_USAGE_CCS_D; } - case ISL_AUX_USAGE_CCS_E: { - /* Lossless compression is not supported for SRGB formats, it - * should be impossible to get here with such surfaces. + /* gen9+ hardware technically supports non-0/1 clear colors with sRGB + * formats. However, there are issues with blending where it doesn't + * properly apply the sRGB curve to the clear color when blending. */ - assert(!isl_format_is_srgb(render_format) || - _mesa_get_srgb_format_linear(mt->format) == mt->format); + if (devinfo->gen >= 9 && blend_enabled && + isl_format_is_srgb(render_format) && + !isl_color_value_is_zero_one(mt->fast_clear_color, render_format)) + return ISL_AUX_USAGE_NONE; - return ISL_AUX_USAGE_CCS_E; - } + if (mt->aux_usage == ISL_AUX_USAGE_CCS_E && + format_ccs_e_compat_with_miptree(&brw->screen->devinfo, + mt, render_format)) + return ISL_AUX_USAGE_CCS_E; + + /* Otherwise, we have to fall back to CCS_D */ + return ISL_AUX_USAGE_CCS_D; default: return ISL_AUX_USAGE_NONE; @@ -2724,11 +2501,8 @@ void intel_miptree_prepare_render(struct brw_context *brw, struct intel_mipmap_tree *mt, uint32_t level, uint32_t start_layer, uint32_t layer_count, - enum isl_format render_format, - bool blend_enabled) + enum isl_aux_usage aux_usage) { - enum isl_aux_usage aux_usage = - intel_miptree_render_aux_usage(brw, mt, render_format, blend_enabled); intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count, aux_usage, aux_usage != ISL_AUX_USAGE_NONE); } @@ -2737,13 +2511,10 @@ void intel_miptree_finish_render(struct brw_context *brw, struct intel_mipmap_tree *mt, uint32_t level, uint32_t start_layer, uint32_t layer_count, - enum isl_format render_format, - bool blend_enabled) + enum isl_aux_usage aux_usage) { assert(_mesa_is_format_color_format(mt->format)); - enum isl_aux_usage aux_usage = - intel_miptree_render_aux_usage(brw, mt, render_format, blend_enabled); intel_miptree_finish_write(brw, mt, level, start_layer, layer_count, aux_usage); } @@ -2754,7 +2525,7 @@ intel_miptree_prepare_depth(struct brw_context *brw, uint32_t start_layer, uint32_t layer_count) { intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count, - mt->aux_usage, mt->hiz_buf != NULL); + mt->aux_usage, mt->aux_buf != NULL); } void @@ -2765,7 +2536,7 @@ intel_miptree_finish_depth(struct brw_context *brw, { if (depth_written) { intel_miptree_finish_write(brw, mt, level, start_layer, layer_count, - mt->hiz_buf != NULL); + mt->aux_usage); } } @@ -2789,7 +2560,7 @@ intel_miptree_prepare_external(struct brw_context *brw, assert(mt->surf.logical_level0_px.depth == 1); assert(mt->surf.logical_level0_px.array_len == 1); assert(mt->surf.samples == 1); - assert(mt->mcs_buf != NULL); + assert(mt->aux_buf != NULL); aux_usage = mod_info->aux_usage; supports_fast_clear = mod_info->supports_clear_color; @@ -2800,6 +2571,25 @@ intel_miptree_prepare_external(struct brw_context *brw, aux_usage, supports_fast_clear); } +void +intel_miptree_finish_external(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + if (!mt->aux_buf) + return; + + /* We don't know the actual aux state of the aux surface. The previous + * owner could have given it to us in a number of different states. + * Because we don't know the aux state, we reset the aux state to the + * least common denominator of possible valid states. + */ + enum isl_aux_state default_aux_state = + isl_drm_modifier_get_default_aux_state(mt->drm_modifier); + assert(mt->last_level == mt->first_level); + intel_miptree_set_aux_state(brw, mt, 0, 0, INTEL_REMAINING_LAYERS, + default_aux_state); +} + /** * Make it possible to share the BO backing the given miptree with another * process or another miptree. @@ -2826,32 +2616,15 @@ intel_miptree_make_shareable(struct brw_context *brw, 0, INTEL_REMAINING_LAYERS, ISL_AUX_USAGE_NONE, false); - if (mt->mcs_buf) { - brw_bo_unreference(mt->mcs_buf->bo); - free(mt->mcs_buf); - mt->mcs_buf = NULL; - - /* Any pending MCS/CCS operations are no longer needed. Trying to - * execute any will likely crash due to the missing aux buffer. So let's - * delete all pending ops. - */ - free(mt->aux_state); - mt->aux_state = NULL; - brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE; - } - - if (mt->hiz_buf) { - intel_miptree_aux_buffer_free(mt->hiz_buf); - mt->hiz_buf = NULL; + if (mt->aux_buf) { + intel_miptree_aux_buffer_free(mt->aux_buf); + mt->aux_buf = NULL; + /* Make future calls of intel_miptree_level_has_hiz() return false. */ for (uint32_t l = mt->first_level; l <= mt->last_level; ++l) { mt->level[l].has_hiz = false; } - /* Any pending HiZ operations are no longer needed. Trying to execute - * any will likely crash due to the missing aux buffer. So let's delete - * all pending ops. - */ free(mt->aux_state); mt->aux_state = NULL; brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE; @@ -2964,14 +2737,14 @@ intel_update_r8stencil(struct brw_context *brw, assert(devinfo->gen >= 7); struct intel_mipmap_tree *src = mt->format == MESA_FORMAT_S_UINT8 ? mt : mt->stencil_mt; - if (!src || devinfo->gen >= 8 || !src->r8stencil_needs_update) + if (!src || devinfo->gen >= 8) return; - assert(src->surf.size > 0); + assert(src->surf.size_B > 0); - if (!mt->r8stencil_mt) { + if (!mt->shadow_mt) { assert(devinfo->gen > 6); /* Handle MIPTREE_LAYOUT_GEN6_HIZ_STENCIL */ - mt->r8stencil_mt = make_surface( + mt->shadow_mt = make_surface( brw, src->target, MESA_FORMAT_R_UINT8, @@ -2985,10 +2758,13 @@ intel_update_r8stencil(struct brw_context *brw, ISL_TILING_Y0_BIT, ISL_SURF_USAGE_TEXTURE_BIT, BO_ALLOC_BUSY, 0, NULL); - assert(mt->r8stencil_mt); + assert(mt->shadow_mt); } - struct intel_mipmap_tree *dst = mt->r8stencil_mt; + if (src->shadow_needs_update == false) + return; + + struct intel_mipmap_tree *dst = mt->shadow_mt; for (int level = src->first_level; level <= src->last_level; level++) { const unsigned depth = src->surf.dim == ISL_SURF_DIM_3D ? @@ -3008,7 +2784,7 @@ intel_update_r8stencil(struct brw_context *brw, } brw_cache_flush_for_read(brw, dst->bo); - src->r8stencil_needs_update = false; + src->shadow_needs_update = false; } static void * @@ -3031,7 +2807,16 @@ intel_miptree_unmap_raw(struct intel_mipmap_tree *mt) } static void -intel_miptree_map_gtt(struct brw_context *brw, +intel_miptree_unmap_map(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, unsigned int slice) +{ + intel_miptree_unmap_raw(mt); +} + +static void +intel_miptree_map_map(struct brw_context *brw, struct intel_mipmap_tree *mt, struct intel_miptree_map *map, unsigned int level, unsigned int slice) @@ -3052,6 +2837,9 @@ intel_miptree_map_gtt(struct brw_context *brw, y /= bh; x /= bw; + intel_miptree_access_raw(brw, mt, level, slice, + map->mode & GL_MAP_WRITE_BIT); + base = intel_miptree_map_raw(brw, mt, map->mode); if (base == NULL) @@ -3066,7 +2854,7 @@ intel_miptree_map_gtt(struct brw_context *brw, x += image_x; y += image_y; - map->stride = mt->surf.row_pitch; + map->stride = mt->surf.row_pitch_B; map->ptr = base + y * map->stride + x * mt->cpp; } @@ -3075,12 +2863,187 @@ intel_miptree_map_gtt(struct brw_context *brw, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), x, y, map->ptr, map->stride); + + map->unmap = intel_miptree_unmap_map; } static void -intel_miptree_unmap_gtt(struct intel_mipmap_tree *mt) +intel_miptree_unmap_blit(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, + unsigned int slice) { - intel_miptree_unmap_raw(mt); + const struct gen_device_info *devinfo = &brw->screen->devinfo; + struct gl_context *ctx = &brw->ctx; + + intel_miptree_unmap_raw(map->linear_mt); + + if (map->mode & GL_MAP_WRITE_BIT) { + if (devinfo->gen >= 6) { + brw_blorp_copy_miptrees(brw, map->linear_mt, 0, 0, + mt, level, slice, + 0, 0, map->x, map->y, map->w, map->h); + } else { + bool ok = intel_miptree_copy(brw, + map->linear_mt, 0, 0, 0, 0, + mt, level, slice, map->x, map->y, + map->w, map->h); + WARN_ONCE(!ok, "Failed to blit from linear temporary mapping"); + } + } + + intel_miptree_release(&map->linear_mt); +} + +/* Compute extent parameters for use with tiled_memcpy functions. + * xs are in units of bytes and ys are in units of strides. + */ +static inline void +tile_extents(struct intel_mipmap_tree *mt, struct intel_miptree_map *map, + unsigned int level, unsigned int slice, unsigned int *x1_B, + unsigned int *x2_B, unsigned int *y1_el, unsigned int *y2_el) +{ + unsigned int block_width, block_height; + unsigned int x0_el, y0_el; + + _mesa_get_format_block_size(mt->format, &block_width, &block_height); + + assert(map->x % block_width == 0); + assert(map->y % block_height == 0); + + intel_miptree_get_image_offset(mt, level, slice, &x0_el, &y0_el); + *x1_B = (map->x / block_width + x0_el) * mt->cpp; + *y1_el = map->y / block_height + y0_el; + *x2_B = (DIV_ROUND_UP(map->x + map->w, block_width) + x0_el) * mt->cpp; + *y2_el = DIV_ROUND_UP(map->y + map->h, block_height) + y0_el; +} + +static void +intel_miptree_unmap_tiled_memcpy(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, + unsigned int slice) +{ + if (map->mode & GL_MAP_WRITE_BIT) { + unsigned int x1, x2, y1, y2; + tile_extents(mt, map, level, slice, &x1, &x2, &y1, &y2); + + char *dst = intel_miptree_map_raw(brw, mt, map->mode | MAP_RAW); + dst += mt->offset; + + isl_memcpy_linear_to_tiled( + x1, x2, y1, y2, dst, map->ptr, mt->surf.row_pitch_B, map->stride, + brw->has_swizzling, mt->surf.tiling, ISL_MEMCPY); + + intel_miptree_unmap_raw(mt); + } + _mesa_align_free(map->buffer); + map->buffer = map->ptr = NULL; +} + +/** + * Determine which copy function to use for the given format combination + * + * The only two possible copy functions which are ever returned are a + * direct memcpy and a RGBA <-> BGRA copy function. Since RGBA -> BGRA and + * BGRA -> RGBA are exactly the same operation (and memcpy is obviously + * symmetric), it doesn't matter whether the copy is from the tiled image + * to the untiled or vice versa. The copy function required is the same in + * either case so this function can be used. + * + * \param[in] tiledFormat The format of the tiled image + * \param[in] format The GL format of the client data + * \param[in] type The GL type of the client data + * \param[out] mem_copy Will be set to one of either the standard + * library's memcpy or a different copy function + * that performs an RGBA to BGRA conversion + * \param[out] cpp Number of bytes per channel + * + * \return true if the format and type combination are valid + */ +MAYBE_UNUSED isl_memcpy_type +intel_miptree_get_memcpy_type(mesa_format tiledFormat, GLenum format, GLenum type, + uint32_t *cpp) +{ + if (type == GL_UNSIGNED_INT_8_8_8_8_REV && + !(format == GL_RGBA || format == GL_BGRA)) + return ISL_MEMCPY_INVALID; /* Invalid type/format combination */ + + if ((tiledFormat == MESA_FORMAT_L_UNORM8 && format == GL_LUMINANCE) || + (tiledFormat == MESA_FORMAT_A_UNORM8 && format == GL_ALPHA)) { + *cpp = 1; + return ISL_MEMCPY; + } else if ((tiledFormat == MESA_FORMAT_B8G8R8A8_UNORM) || + (tiledFormat == MESA_FORMAT_B8G8R8X8_UNORM) || + (tiledFormat == MESA_FORMAT_B8G8R8A8_SRGB) || + (tiledFormat == MESA_FORMAT_B8G8R8X8_SRGB)) { + *cpp = 4; + if (format == GL_BGRA) { + return ISL_MEMCPY; + } else if (format == GL_RGBA) { + return ISL_MEMCPY_BGRA8; + } + } else if ((tiledFormat == MESA_FORMAT_R8G8B8A8_UNORM) || + (tiledFormat == MESA_FORMAT_R8G8B8X8_UNORM) || + (tiledFormat == MESA_FORMAT_R8G8B8A8_SRGB) || + (tiledFormat == MESA_FORMAT_R8G8B8X8_SRGB)) { + *cpp = 4; + if (format == GL_BGRA) { + /* Copying from RGBA to BGRA is the same as BGRA to RGBA so we can + * use the same function. + */ + return ISL_MEMCPY_BGRA8; + } else if (format == GL_RGBA) { + return ISL_MEMCPY; + } + } + + return ISL_MEMCPY_INVALID; +} + +static void +intel_miptree_map_tiled_memcpy(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, unsigned int slice) +{ + intel_miptree_access_raw(brw, mt, level, slice, + map->mode & GL_MAP_WRITE_BIT); + + unsigned int x1, x2, y1, y2; + tile_extents(mt, map, level, slice, &x1, &x2, &y1, &y2); + map->stride = ALIGN(_mesa_format_row_stride(mt->format, map->w), 16); + + /* The tiling and detiling functions require that the linear buffer + * has proper 16-byte alignment (that is, its `x0` is 16-byte + * aligned). Here we over-allocate the linear buffer by enough + * bytes to get the proper alignment. + */ + map->buffer = _mesa_align_malloc(map->stride * (y2 - y1) + (x1 & 0xf), 16); + map->ptr = (char *)map->buffer + (x1 & 0xf); + assert(map->buffer); + + if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { + char *src = intel_miptree_map_raw(brw, mt, map->mode | MAP_RAW); + src += mt->offset; + + const isl_memcpy_type copy_type = +#if defined(USE_SSE41) + cpu_has_sse4_1 ? ISL_MEMCPY_STREAMING_LOAD : +#endif + ISL_MEMCPY; + + isl_memcpy_tiled_to_linear( + x1, x2, y1, y2, map->ptr, src, map->stride, + mt->surf.row_pitch_B, brw->has_swizzling, mt->surf.tiling, + copy_type); + + intel_miptree_unmap_raw(mt); + } + + map->unmap = intel_miptree_unmap_tiled_memcpy; } static void @@ -3089,18 +3052,19 @@ intel_miptree_map_blit(struct brw_context *brw, struct intel_miptree_map *map, unsigned int level, unsigned int slice) { - map->linear_mt = intel_miptree_create(brw, GL_TEXTURE_2D, mt->format, - /* first_level */ 0, - /* last_level */ 0, - map->w, map->h, 1, - /* samples */ 1, - MIPTREE_CREATE_LINEAR); + const struct gen_device_info *devinfo = &brw->screen->devinfo; + map->linear_mt = make_surface(brw, GL_TEXTURE_2D, mt->format, + 0, 0, map->w, map->h, 1, 1, + ISL_TILING_LINEAR_BIT, + ISL_SURF_USAGE_RENDER_TARGET_BIT | + ISL_SURF_USAGE_TEXTURE_BIT, + 0, 0, NULL); if (!map->linear_mt) { fprintf(stderr, "Failed to allocate blit temporary\n"); goto fail; } - map->stride = map->linear_mt->surf.row_pitch; + map->stride = map->linear_mt->surf.row_pitch_B; /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless @@ -3108,12 +3072,18 @@ intel_miptree_map_blit(struct brw_context *brw, * temporary buffer back out. */ if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { - if (!intel_miptree_copy(brw, - mt, level, slice, map->x, map->y, - map->linear_mt, 0, 0, 0, 0, - map->w, map->h)) { - fprintf(stderr, "Failed to blit\n"); - goto fail; + if (devinfo->gen >= 6) { + brw_blorp_copy_miptrees(brw, mt, level, slice, + map->linear_mt, 0, 0, + map->x, map->y, 0, 0, map->w, map->h); + } else { + if (!intel_miptree_copy(brw, + mt, level, slice, map->x, map->y, + map->linear_mt, 0, 0, 0, 0, + map->w, map->h)) { + fprintf(stderr, "Failed to blit\n"); + goto fail; + } } } @@ -3124,6 +3094,7 @@ intel_miptree_map_blit(struct brw_context *brw, mt, _mesa_get_format_name(mt->format), level, slice, map->ptr, map->stride); + map->unmap = intel_miptree_unmap_blit; return; fail: @@ -3132,32 +3103,22 @@ fail: map->stride = 0; } -static void -intel_miptree_unmap_blit(struct brw_context *brw, - struct intel_mipmap_tree *mt, - struct intel_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - struct gl_context *ctx = &brw->ctx; - - intel_miptree_unmap_raw(map->linear_mt); - - if (map->mode & GL_MAP_WRITE_BIT) { - bool ok = intel_miptree_copy(brw, - map->linear_mt, 0, 0, 0, 0, - mt, level, slice, map->x, map->y, - map->w, map->h); - WARN_ONCE(!ok, "Failed to blit from linear temporary mapping"); - } - - intel_miptree_release(&map->linear_mt); -} - /** * "Map" a buffer by copying it to an untiled temporary using MOVNTDQA. */ #if defined(USE_SSE41) +static void +intel_miptree_unmap_movntdqa(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, + unsigned int slice) +{ + _mesa_align_free(map->buffer); + map->buffer = NULL; + map->ptr = NULL; +} + static void intel_miptree_map_movntdqa(struct brw_context *brw, struct intel_mipmap_tree *mt, @@ -3167,6 +3128,8 @@ intel_miptree_map_movntdqa(struct brw_context *brw, assert(map->mode & GL_MAP_READ_BIT); assert(!(map->mode & GL_MAP_WRITE_BIT)); + intel_miptree_access_raw(brw, mt, level, slice, false); + DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), @@ -3185,7 +3148,7 @@ intel_miptree_map_movntdqa(struct brw_context *brw, src += mt->offset; - src += image_y * mt->surf.row_pitch; + src += image_y * mt->surf.row_pitch_B; src += image_x * mt->cpp; /* Due to the pixel offsets for the particular image being mapped, our @@ -3193,7 +3156,7 @@ intel_miptree_map_movntdqa(struct brw_context *brw, * divisible by 16, then the amount by which it's misaligned will remain * consistent from row to row. */ - assert((mt->surf.row_pitch % 16) == 0); + assert((mt->surf.row_pitch_B % 16) == 0); const int misalignment = ((uintptr_t) src) & 15; /* Create an untiled temporary buffer for the mapping. */ @@ -3209,26 +3172,46 @@ intel_miptree_map_movntdqa(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { void *dst_ptr = map->ptr + y * map->stride; - void *src_ptr = src + y * mt->surf.row_pitch; + void *src_ptr = src + y * mt->surf.row_pitch_B; _mesa_streaming_load_memcpy(dst_ptr, src_ptr, width_bytes); } intel_miptree_unmap_raw(mt); + + map->unmap = intel_miptree_unmap_movntdqa; } +#endif static void -intel_miptree_unmap_movntdqa(struct brw_context *brw, - struct intel_mipmap_tree *mt, - struct intel_miptree_map *map, - unsigned int level, - unsigned int slice) +intel_miptree_unmap_s8(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, + unsigned int slice) { - _mesa_align_free(map->buffer); - map->buffer = NULL; - map->ptr = NULL; + if (map->mode & GL_MAP_WRITE_BIT) { + unsigned int image_x, image_y; + uint8_t *untiled_s8_map = map->ptr; + uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT); + + intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); + + for (uint32_t y = 0; y < map->h; y++) { + for (uint32_t x = 0; x < map->w; x++) { + ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch_B, + image_x + x + map->x, + image_y + y + map->y, + brw->has_swizzling); + tiled_s8_map[offset] = untiled_s8_map[y * map->w + x]; + } + } + + intel_miptree_unmap_raw(mt); + } + + free(map->buffer); } -#endif static void intel_miptree_map_s8(struct brw_context *brw, @@ -3241,6 +3224,9 @@ intel_miptree_map_s8(struct brw_context *brw, if (!map->buffer) return; + intel_miptree_access_raw(brw, mt, level, slice, + map->mode & GL_MAP_WRITE_BIT); + /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless * invalidate is set, since we'll be writing the whole rectangle from our @@ -3255,7 +3241,7 @@ intel_miptree_map_s8(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch, + ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch_B, x + image_x + map->x, y + image_y + map->y, brw->has_swizzling); @@ -3273,101 +3259,80 @@ intel_miptree_map_s8(struct brw_context *brw, map->x, map->y, map->w, map->h, mt, map->ptr, map->stride); } + + map->unmap = intel_miptree_unmap_s8; } +/** + * Mapping functions for packed depth/stencil miptrees backed by real separate + * miptrees for depth and stencil. + * + * On gen7, and to support HiZ pre-gen7, we have to have the stencil buffer + * separate from the depth buffer. Yet at the GL API level, we have to expose + * packed depth/stencil textures and FBO attachments, and Mesa core expects to + * be able to map that memory for texture storage and glReadPixels-type + * operations. We give Mesa core that access by mallocing a temporary and + * copying the data between the actual backing store and the temporary. + */ static void -intel_miptree_unmap_s8(struct brw_context *brw, - struct intel_mipmap_tree *mt, - struct intel_miptree_map *map, - unsigned int level, - unsigned int slice) +intel_miptree_unmap_depthstencil(struct brw_context *brw, + struct intel_mipmap_tree *mt, + struct intel_miptree_map *map, + unsigned int level, + unsigned int slice) { + struct intel_mipmap_tree *z_mt = mt; + struct intel_mipmap_tree *s_mt = mt->stencil_mt; + bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32; + if (map->mode & GL_MAP_WRITE_BIT) { - unsigned int image_x, image_y; - uint8_t *untiled_s8_map = map->ptr; - uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT); + uint32_t *packed_map = map->ptr; + uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_WRITE_BIT); + uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_WRITE_BIT); + unsigned int s_image_x, s_image_y; + unsigned int z_image_x, z_image_y; - intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); + intel_miptree_get_image_offset(s_mt, level, slice, + &s_image_x, &s_image_y); + intel_miptree_get_image_offset(z_mt, level, slice, + &z_image_x, &z_image_y); for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch, - image_x + x + map->x, - image_y + y + map->y, - brw->has_swizzling); - tiled_s8_map[offset] = untiled_s8_map[y * map->w + x]; + ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch_B, + x + s_image_x + map->x, + y + s_image_y + map->y, + brw->has_swizzling); + ptrdiff_t z_offset = ((y + z_image_y + map->y) * + (z_mt->surf.row_pitch_B / 4) + + (x + z_image_x + map->x)); + + if (map_z32f_x24s8) { + z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0]; + s_map[s_offset] = packed_map[(y * map->w + x) * 2 + 1]; + } else { + uint32_t packed = packed_map[y * map->w + x]; + s_map[s_offset] = packed >> 24; + z_map[z_offset] = packed; + } } } - intel_miptree_unmap_raw(mt); - } - - free(map->buffer); -} + intel_miptree_unmap_raw(s_mt); + intel_miptree_unmap_raw(z_mt); -static void -intel_miptree_map_etc(struct brw_context *brw, - struct intel_mipmap_tree *mt, - struct intel_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - assert(mt->etc_format != MESA_FORMAT_NONE); - if (mt->etc_format == MESA_FORMAT_ETC1_RGB8) { - assert(mt->format == MESA_FORMAT_R8G8B8X8_UNORM); + DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n", + __func__, + map->x, map->y, map->w, map->h, + z_mt, _mesa_get_format_name(z_mt->format), + map->x + z_image_x, map->y + z_image_y, + s_mt, map->x + s_image_x, map->y + s_image_y, + map->ptr, map->stride); } - assert(map->mode & GL_MAP_WRITE_BIT); - assert(map->mode & GL_MAP_INVALIDATE_RANGE_BIT); - - map->stride = _mesa_format_row_stride(mt->etc_format, map->w); - map->buffer = malloc(_mesa_format_image_size(mt->etc_format, - map->w, map->h, 1)); - map->ptr = map->buffer; -} - -static void -intel_miptree_unmap_etc(struct brw_context *brw, - struct intel_mipmap_tree *mt, - struct intel_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - uint32_t image_x; - uint32_t image_y; - intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); - - image_x += map->x; - image_y += map->y; - - uint8_t *dst = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT) - + image_y * mt->surf.row_pitch - + image_x * mt->cpp; - - if (mt->etc_format == MESA_FORMAT_ETC1_RGB8) - _mesa_etc1_unpack_rgba8888(dst, mt->surf.row_pitch, - map->ptr, map->stride, - map->w, map->h); - else - _mesa_unpack_etc2_format(dst, mt->surf.row_pitch, - map->ptr, map->stride, - map->w, map->h, mt->etc_format); - - intel_miptree_unmap_raw(mt); free(map->buffer); } -/** - * Mapping function for packed depth/stencil miptrees backed by real separate - * miptrees for depth and stencil. - * - * On gen7, and to support HiZ pre-gen7, we have to have the stencil buffer - * separate from the depth buffer. Yet at the GL API level, we have to expose - * packed depth/stencil textures and FBO attachments, and Mesa core expects to - * be able to map that memory for texture storage and glReadPixels-type - * operations. We give Mesa core that access by mallocing a temporary and - * copying the data between the actual backing store and the temporary. - */ static void intel_miptree_map_depthstencil(struct brw_context *brw, struct intel_mipmap_tree *mt, @@ -3384,6 +3349,11 @@ intel_miptree_map_depthstencil(struct brw_context *brw, if (!map->buffer) return; + intel_miptree_access_raw(brw, z_mt, level, slice, + map->mode & GL_MAP_WRITE_BIT); + intel_miptree_access_raw(brw, s_mt, level, slice, + map->mode & GL_MAP_WRITE_BIT); + /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless * invalidate is set, since we'll be writing the whole rectangle from our @@ -3404,12 +3374,12 @@ intel_miptree_map_depthstencil(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { int map_x = map->x + x, map_y = map->y + y; - ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch, + ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch_B, map_x + s_image_x, map_y + s_image_y, brw->has_swizzling); ptrdiff_t z_offset = ((map_y + z_image_y) * - (z_mt->surf.row_pitch / 4) + + (z_mt->surf.row_pitch_B / 4) + (map_x + z_image_x)); uint8_t s = s_map[s_offset]; uint32_t z = z_map[z_offset]; @@ -3437,65 +3407,8 @@ intel_miptree_map_depthstencil(struct brw_context *brw, map->x, map->y, map->w, map->h, mt, map->ptr, map->stride); } -} - -static void -intel_miptree_unmap_depthstencil(struct brw_context *brw, - struct intel_mipmap_tree *mt, - struct intel_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - struct intel_mipmap_tree *z_mt = mt; - struct intel_mipmap_tree *s_mt = mt->stencil_mt; - bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32; - if (map->mode & GL_MAP_WRITE_BIT) { - uint32_t *packed_map = map->ptr; - uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_WRITE_BIT); - uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_WRITE_BIT); - unsigned int s_image_x, s_image_y; - unsigned int z_image_x, z_image_y; - - intel_miptree_get_image_offset(s_mt, level, slice, - &s_image_x, &s_image_y); - intel_miptree_get_image_offset(z_mt, level, slice, - &z_image_x, &z_image_y); - - for (uint32_t y = 0; y < map->h; y++) { - for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch, - x + s_image_x + map->x, - y + s_image_y + map->y, - brw->has_swizzling); - ptrdiff_t z_offset = ((y + z_image_y + map->y) * - (z_mt->surf.row_pitch / 4) + - (x + z_image_x + map->x)); - - if (map_z32f_x24s8) { - z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0]; - s_map[s_offset] = packed_map[(y * map->w + x) * 2 + 1]; - } else { - uint32_t packed = packed_map[y * map->w + x]; - s_map[s_offset] = packed >> 24; - z_map[z_offset] = packed; - } - } - } - - intel_miptree_unmap_raw(s_mt); - intel_miptree_unmap_raw(z_mt); - - DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n", - __func__, - map->x, map->y, map->w, map->h, - z_mt, _mesa_get_format_name(z_mt->format), - map->x + z_image_x, map->y + z_image_y, - s_mt, map->x + s_image_x, map->y + s_image_y, - map->ptr, map->stride); - } - - free(map->buffer); + map->unmap = intel_miptree_unmap_depthstencil; } /** @@ -3546,21 +3459,18 @@ intel_miptree_release_map(struct intel_mipmap_tree *mt, static bool can_blit_slice(struct intel_mipmap_tree *mt, - unsigned int level, unsigned int slice) + const struct intel_miptree_map *map) { /* See intel_miptree_blit() for details on the 32k pitch limit. */ - if (mt->surf.row_pitch >= 32768) - return false; - - return true; + const unsigned src_blt_pitch = intel_miptree_blt_pitch(mt); + const unsigned dst_blt_pitch = ALIGN(map->w * mt->cpp, 64); + return src_blt_pitch < 32768 && dst_blt_pitch < 32768; } static bool use_intel_mipree_map_blit(struct brw_context *brw, struct intel_mipmap_tree *mt, - GLbitfield mode, - unsigned int level, - unsigned int slice) + const struct intel_miptree_map *map) { const struct gen_device_info *devinfo = &brw->screen->devinfo; @@ -3568,19 +3478,19 @@ use_intel_mipree_map_blit(struct brw_context *brw, /* It's probably not worth swapping to the blit ring because of * all the overhead involved. */ - !(mode & GL_MAP_WRITE_BIT) && + !(map->mode & GL_MAP_WRITE_BIT) && !mt->compressed && (mt->surf.tiling == ISL_TILING_X || /* Prior to Sandybridge, the blitter can't handle Y tiling */ (devinfo->gen >= 6 && mt->surf.tiling == ISL_TILING_Y0) || /* Fast copy blit on skl+ supports all tiling formats. */ devinfo->gen >= 9) && - can_blit_slice(mt, level, slice)) + can_blit_slice(mt, map)) return true; if (mt->surf.tiling != ISL_TILING_LINEAR && mt->bo->size >= brw->max_gtt_map_object_size) { - assert(can_blit_slice(mt, level, slice)); + assert(can_blit_slice(mt, map)); return true; } @@ -3611,6 +3521,7 @@ intel_miptree_map(struct brw_context *brw, void **out_ptr, ptrdiff_t *out_stride) { + const struct gen_device_info *devinfo = &brw->screen->devinfo; struct intel_miptree_map *map; assert(mt->surf.samples == 1); @@ -3622,26 +3533,24 @@ intel_miptree_map(struct brw_context *brw, return; } - intel_miptree_access_raw(brw, mt, level, slice, - map->mode & GL_MAP_WRITE_BIT); - if (mt->format == MESA_FORMAT_S_UINT8) { intel_miptree_map_s8(brw, mt, map, level, slice); - } else if (mt->etc_format != MESA_FORMAT_NONE && - !(mode & BRW_MAP_DIRECT_BIT)) { - intel_miptree_map_etc(brw, mt, map, level, slice); } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) { intel_miptree_map_depthstencil(brw, mt, map, level, slice); - } else if (use_intel_mipree_map_blit(brw, mt, mode, level, slice)) { + } else if (use_intel_mipree_map_blit(brw, mt, map)) { intel_miptree_map_blit(brw, mt, map, level, slice); + } else if (mt->surf.tiling != ISL_TILING_LINEAR && devinfo->gen > 4) { + intel_miptree_map_tiled_memcpy(brw, mt, map, level, slice); #if defined(USE_SSE41) } else if (!(mode & GL_MAP_WRITE_BIT) && !mt->compressed && cpu_has_sse4_1 && - (mt->surf.row_pitch % 16 == 0)) { + (mt->surf.row_pitch_B % 16 == 0)) { intel_miptree_map_movntdqa(brw, mt, map, level, slice); #endif } else { - intel_miptree_map_gtt(brw, mt, map, level, slice); + if (mt->surf.tiling != ISL_TILING_LINEAR) + perf_debug("intel_miptree_map: mapping via gtt"); + intel_miptree_map_map(brw, mt, map, level, slice); } *out_ptr = map->ptr; @@ -3667,22 +3576,8 @@ intel_miptree_unmap(struct brw_context *brw, DBG("%s: mt %p (%s) level %d slice %d\n", __func__, mt, _mesa_get_format_name(mt->format), level, slice); - if (mt->format == MESA_FORMAT_S_UINT8) { - intel_miptree_unmap_s8(brw, mt, map, level, slice); - } else if (mt->etc_format != MESA_FORMAT_NONE && - !(map->mode & BRW_MAP_DIRECT_BIT)) { - intel_miptree_unmap_etc(brw, mt, map, level, slice); - } else if (mt->stencil_mt && !(map->mode & BRW_MAP_DIRECT_BIT)) { - intel_miptree_unmap_depthstencil(brw, mt, map, level, slice); - } else if (map->linear_mt) { - intel_miptree_unmap_blit(brw, mt, map, level, slice); -#if defined(USE_SSE41) - } else if (map->buffer && cpu_has_sse4_1) { - intel_miptree_unmap_movntdqa(brw, mt, map, level, slice); -#endif - } else { - intel_miptree_unmap_gtt(mt); - } + if (map->unmap) + map->unmap(brw, mt, map, level, slice); intel_miptree_release_map(mt, level, slice); } @@ -3743,15 +3638,122 @@ get_isl_dim_layout(const struct gen_device_info *devinfo, unreachable("Invalid texture target"); } -enum isl_aux_usage -intel_miptree_get_aux_isl_usage(const struct brw_context *brw, - const struct intel_mipmap_tree *mt) +bool +intel_miptree_set_clear_color(struct brw_context *brw, + struct intel_mipmap_tree *mt, + union isl_color_value clear_color) +{ + if (memcmp(&mt->fast_clear_color, &clear_color, sizeof(clear_color)) != 0) { + mt->fast_clear_color = clear_color; + if (mt->aux_buf->clear_color_bo) { + /* We can't update the clear color while the hardware is still using + * the previous one for a resolve or sampling from it. Make sure that + * there are no pending commands at this point. + */ + brw_emit_pipe_control_flush(brw, PIPE_CONTROL_CS_STALL); + for (int i = 0; i < 4; i++) { + brw_store_data_imm32(brw, mt->aux_buf->clear_color_bo, + mt->aux_buf->clear_color_offset + i * 4, + mt->fast_clear_color.u32[i]); + } + brw_emit_pipe_control_flush(brw, PIPE_CONTROL_STATE_CACHE_INVALIDATE); + } + brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE; + return true; + } + return false; +} + +union isl_color_value +intel_miptree_get_clear_color(const struct gen_device_info *devinfo, + const struct intel_mipmap_tree *mt, + enum isl_format view_format, bool sampling, + struct brw_bo **clear_color_bo, + uint64_t *clear_color_offset) { - if (mt->hiz_buf) - return ISL_AUX_USAGE_HIZ; + assert(mt->aux_buf); - if (!mt->mcs_buf) - return ISL_AUX_USAGE_NONE; + if (devinfo->gen == 10 && isl_format_is_srgb(view_format) && sampling) { + /* The gen10 sampler doesn't gamma-correct the clear color. In this case, + * we switch to using the inline clear color and do the sRGB color + * conversion process defined in the OpenGL spec. The red, green, and + * blue channels take part in gamma correction, while the alpha channel + * is unchanged. + */ + union isl_color_value srgb_decoded_value = mt->fast_clear_color; + for (unsigned i = 0; i < 3; i++) { + srgb_decoded_value.f32[i] = + util_format_srgb_to_linear_float(mt->fast_clear_color.f32[i]); + } + *clear_color_bo = 0; + *clear_color_offset = 0; + return srgb_decoded_value; + } else { + *clear_color_bo = mt->aux_buf->clear_color_bo; + *clear_color_offset = mt->aux_buf->clear_color_offset; + return mt->fast_clear_color; + } +} + +static void +intel_miptree_update_etc_shadow(struct brw_context *brw, + struct intel_mipmap_tree *mt, + unsigned int level, + unsigned int slice, + int level_w, + int level_h) +{ + ptrdiff_t etc_stride, shadow_stride; + void *mptr, *sptr; + struct intel_mipmap_tree *smt = mt->shadow_mt; + + assert(intel_miptree_has_etc_shadow(brw, mt)); + + intel_miptree_map(brw, mt, level, slice, 0, 0, level_w, level_h, + GL_MAP_READ_BIT, &mptr, &etc_stride); + intel_miptree_map(brw, smt, level, slice, 0, 0, level_w, level_h, + GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT, + &sptr, &shadow_stride); + + if (mt->format == MESA_FORMAT_ETC1_RGB8) { + _mesa_etc1_unpack_rgba8888(sptr, shadow_stride, mptr, etc_stride, + level_w, level_h); + } else { + /* destination and source images must have the same swizzle */ + bool is_bgra = (smt->format == MESA_FORMAT_B8G8R8A8_SRGB); + _mesa_unpack_etc2_format(sptr, shadow_stride, mptr, etc_stride, + level_w, level_h, mt->format, is_bgra); + } + + intel_miptree_unmap(brw, mt, level, slice); + intel_miptree_unmap(brw, smt, level, slice); +} + +void +intel_miptree_update_etc_shadow_levels(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + struct intel_mipmap_tree *smt; + int num_slices; + + assert(mt); + assert(mt->surf.size_B > 0); + assert(intel_miptree_has_etc_shadow(brw, mt)); + + smt = mt->shadow_mt; + num_slices = smt->surf.logical_level0_px.array_len; + + for (int level = smt->first_level; level <= smt->last_level; level++) { + int level_w = minify(smt->surf.logical_level0_px.width, + level - smt->first_level); + int level_h = minify(smt->surf.logical_level0_px.height, + level - smt->first_level); + + for (unsigned int slice = 0; slice < num_slices; slice++) { + intel_miptree_update_etc_shadow(brw, mt, level, slice, level_w, + level_h); + } + } - return mt->aux_usage; + mt->shadow_needs_update = false; }