X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fintel_mipmap_tree.c;h=593bb9da0d5e9eeb06bdcafc1b9b58c07f94a598;hb=278460279b4e089d51a24fb01dc56dc1e88dcb72;hp=f815fbe1e5d3370613d433aecfd226748abbabd5;hpb=aebcf26d8219cee79da89313124c2147595a660c;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c index f815fbe1e5d..593bb9da0d5 100644 --- a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c +++ b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c @@ -49,12 +49,18 @@ #define FILE_DEBUG_FLAG DEBUG_MIPTREE +static bool +intel_miptree_alloc_mcs(struct brw_context *brw, + struct intel_mipmap_tree *mt, + GLuint num_samples); + /** * Determine which MSAA layout should be used by the MSAA surface being * created, based on the chip generation and the surface type. */ static enum intel_msaa_layout -compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target) +compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target, + bool disable_aux_buffers) { /* Prior to Gen7, all MSAA surfaces used IMS layout. */ if (brw->gen < 7) @@ -80,6 +86,11 @@ compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target) */ if (brw->gen == 7 && _mesa_get_format_datatype(format) == GL_INT) { return INTEL_MSAA_LAYOUT_UMS; + } else if (disable_aux_buffers) { + /* We can't use the CMS layout because it uses an aux buffer, the MCS + * buffer. So fallback to UMS, which is identical to CMS without the + * MCS. */ + return INTEL_MSAA_LAYOUT_UMS; } else { return INTEL_MSAA_LAYOUT_CMS; } @@ -147,15 +158,32 @@ intel_get_non_msrt_mcs_alignment(struct brw_context *brw, } } +bool +intel_tiling_supports_non_msrt_mcs(struct brw_context *brw, unsigned tiling) +{ + /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render + * Target(s)", beneath the "Fast Color Clear" bullet (p326): + * + * - Support is limited to tiled render targets. + * + * Gen9 changes the restriction to Y-tile only. + */ + if (brw->gen >= 9) + return tiling == I915_TILING_Y; + else if (brw->gen >= 7) + return tiling != I915_TILING_NONE; + else + return false; +} /** * For a single-sampled render target ("non-MSRT"), determine if an MCS buffer - * can be used. + * can be used. This doesn't (and should not) inspect any of the properties of + * the miptree's BO. * * From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render Target(s)", * beneath the "Fast Color Clear" bullet (p326): * - * - Support is limited to tiled render targets. * - Support is for non-mip-mapped and non-array surface types only. * * And then later, on p327: @@ -164,13 +192,16 @@ intel_get_non_msrt_mcs_alignment(struct brw_context *brw, * 64bpp, and 128bpp. */ bool -intel_is_non_msrt_mcs_buffer_supported(struct brw_context *brw, - struct intel_mipmap_tree *mt) +intel_miptree_is_fast_clear_capable(struct brw_context *brw, + struct intel_mipmap_tree *mt) { /* MCS support does not exist prior to Gen7 */ if (brw->gen < 7) return false; + if (mt->disable_aux_buffers) + return false; + /* MCS is only supported for color buffers */ switch (_mesa_get_format_base_format(mt->format)) { case GL_DEPTH_COMPONENT: @@ -179,15 +210,25 @@ intel_is_non_msrt_mcs_buffer_supported(struct brw_context *brw, return false; } - if (mt->tiling != I915_TILING_X && - mt->tiling != I915_TILING_Y) - return false; if (mt->cpp != 4 && mt->cpp != 8 && mt->cpp != 16) return false; - if (mt->first_level != 0 || mt->last_level != 0) + if (mt->first_level != 0 || mt->last_level != 0) { + if (brw->gen >= 8) { + perf_debug("Multi-LOD fast clear - giving up (%dx%dx%d).\n", + mt->logical_width0, mt->logical_height0, mt->last_level); + } + return false; - if (mt->physical_depth0 != 1) + } + if (mt->physical_depth0 != 1) { + if (brw->gen >= 8) { + perf_debug("Layered fast clear - giving up. (%dx%d%d)\n", + mt->logical_width0, mt->logical_height0, + mt->physical_depth0); + } + return false; + } /* There's no point in using an MCS buffer if the surface isn't in a * renderable format. @@ -221,7 +262,7 @@ intel_depth_format_for_depthstencil_format(mesa_format format) { * intel_miptree_create_for_bo(). If true, then do not create * \c stencil_mt. */ -struct intel_mipmap_tree * +static struct intel_mipmap_tree * intel_miptree_create_layout(struct brw_context *brw, GLenum target, mesa_format format, @@ -230,15 +271,15 @@ intel_miptree_create_layout(struct brw_context *brw, GLuint width0, GLuint height0, GLuint depth0, - bool for_bo, GLuint num_samples, - bool force_all_slices_at_each_lod) + enum intel_miptree_tiling_mode requested, + uint32_t layout_flags) { struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1); if (!mt) return NULL; - DBG("%s target %s format %s level %d..%d slices %d <-- %p\n", __FUNCTION__, + DBG("%s target %s format %s level %d..%d slices %d <-- %p\n", __func__, _mesa_lookup_enum_by_nr(target), _mesa_get_format_name(format), first_level, last_level, depth0, mt); @@ -271,6 +312,7 @@ intel_miptree_create_layout(struct brw_context *brw, mt->logical_height0 = height0; mt->logical_depth0 = depth0; mt->fast_clear_state = INTEL_FAST_CLEAR_STATE_NO_MCS; + mt->disable_aux_buffers = (layout_flags & MIPTREE_LAYOUT_DISABLE_AUX) != 0; exec_list_make_empty(&mt->hiz_map); /* The cpp is bytes per (1, blockheight)-sized block for compressed @@ -288,9 +330,31 @@ intel_miptree_create_layout(struct brw_context *brw, if (num_samples > 1) { /* Adjust width/height/depth for MSAA */ - mt->msaa_layout = compute_msaa_layout(brw, format, mt->target); + mt->msaa_layout = compute_msaa_layout(brw, format, + mt->target, mt->disable_aux_buffers); if (mt->msaa_layout == INTEL_MSAA_LAYOUT_IMS) { - /* In the Sandy Bridge PRM, volume 4, part 1, page 31, it says: + /* From the Ivybridge PRM, Volume 1, Part 1, page 108: + * "If the surface is multisampled and it is a depth or stencil + * surface or Multisampled Surface StorageFormat in SURFACE_STATE is + * MSFMT_DEPTH_STENCIL, WL and HL must be adjusted as follows before + * proceeding: + * + * +----------------------------------------------------------------+ + * | Num Multisamples | W_l = | H_l = | + * +----------------------------------------------------------------+ + * | 2 | ceiling(W_l / 2) * 4 | H_l (no adjustment) | + * | 4 | ceiling(W_l / 2) * 4 | ceiling(H_l / 2) * 4 | + * | 8 | ceiling(W_l / 2) * 8 | ceiling(H_l / 2) * 4 | + * | 16 | ceiling(W_l / 2) * 8 | ceiling(H_l / 2) * 8 | + * +----------------------------------------------------------------+ + * " + * + * Note that MSFMT_DEPTH_STENCIL just means the IMS (interleaved) + * format rather than UMS/CMS (array slices). The Sandybridge PRM, + * Volume 1, Part 1, Page 111 has the same formula for 4x MSAA. + * + * Another more complicated explanation for these adjustments comes + * from the Sandybridge PRM, volume 4, part 1, page 31: * * "Any of the other messages (sample*, LOD, load4) used with a * (4x) multisampled surface will in-effect sample a surface with @@ -350,19 +414,29 @@ intel_miptree_create_layout(struct brw_context *brw, } } - /* Set array_layout to ALL_SLICES_AT_EACH_LOD when gen7+ array_spacing_lod0 - * can be used. array_spacing_lod0 is only used for non-IMS MSAA surfaces. + /* Set array_layout to ALL_SLICES_AT_EACH_LOD when array_spacing_lod0 can + * be used. array_spacing_lod0 is only used for non-IMS MSAA surfaces on + * Gen 7 and 8. On Gen 8 and 9 this layout is not available but it is still + * used on Gen8 to make it pick a qpitch value which doesn't include space + * for the mipmaps. On Gen9 this is not necessary because it will + * automatically pick a packed qpitch value whenever mt->first_level == + * mt->last_level. * TODO: can we use it elsewhere? + * TODO: also disable this on Gen8 and pick the qpitch value like Gen9 */ - switch (mt->msaa_layout) { - case INTEL_MSAA_LAYOUT_NONE: - case INTEL_MSAA_LAYOUT_IMS: + if (brw->gen >= 9) { mt->array_layout = ALL_LOD_IN_EACH_SLICE; - break; - case INTEL_MSAA_LAYOUT_UMS: - case INTEL_MSAA_LAYOUT_CMS: - mt->array_layout = ALL_SLICES_AT_EACH_LOD; - break; + } else { + switch (mt->msaa_layout) { + case INTEL_MSAA_LAYOUT_NONE: + case INTEL_MSAA_LAYOUT_IMS: + mt->array_layout = ALL_LOD_IN_EACH_SLICE; + break; + case INTEL_MSAA_LAYOUT_UMS: + case INTEL_MSAA_LAYOUT_CMS: + mt->array_layout = ALL_SLICES_AT_EACH_LOD; + break; + } } if (target == GL_TEXTURE_CUBE_MAP) { @@ -374,11 +448,15 @@ intel_miptree_create_layout(struct brw_context *brw, mt->physical_height0 = height0; mt->physical_depth0 = depth0; - if (!for_bo && + if (!(layout_flags & MIPTREE_LAYOUT_FOR_BO) && _mesa_get_format_base_format(format) == GL_DEPTH_STENCIL && (brw->must_use_separate_stencil || - (brw->has_separate_stencil && brw_is_hiz_depth_format(brw, format)))) { - const bool force_all_slices_at_each_lod = brw->gen == 6; + (brw->has_separate_stencil && + intel_miptree_wants_hiz_buffer(brw, mt)))) { + uint32_t stencil_flags = MIPTREE_LAYOUT_ACCELERATED_UPLOAD; + if (brw->gen == 6) + stencil_flags |= MIPTREE_LAYOUT_FORCE_ALL_SLICE_AT_LOD; + mt->stencil_mt = intel_miptree_create(brw, mt->target, MESA_FORMAT_S_UINT8, @@ -387,10 +465,10 @@ intel_miptree_create_layout(struct brw_context *brw, mt->logical_width0, mt->logical_height0, mt->logical_depth0, - true, num_samples, INTEL_MIPTREE_TILING_ANY, - force_all_slices_at_each_lod); + stencil_flags); + if (!mt->stencil_mt) { intel_miptree_release(&mt); return NULL; @@ -408,109 +486,36 @@ intel_miptree_create_layout(struct brw_context *brw, } } - if (force_all_slices_at_each_lod) + if (layout_flags & MIPTREE_LAYOUT_FORCE_ALL_SLICE_AT_LOD) mt->array_layout = ALL_SLICES_AT_EACH_LOD; - brw_miptree_layout(brw, mt); - - return mt; -} - -/** - * \brief Helper function for intel_miptree_create(). - */ -static uint32_t -intel_miptree_choose_tiling(struct brw_context *brw, - mesa_format format, - uint32_t width0, - uint32_t num_samples, - enum intel_miptree_tiling_mode requested, - struct intel_mipmap_tree *mt) -{ - if (format == MESA_FORMAT_S_UINT8) { - /* The stencil buffer is W tiled. However, we request from the kernel a - * non-tiled buffer because the GTT is incapable of W fencing. - */ - return I915_TILING_NONE; - } - - /* Some usages may want only one type of tiling, like depth miptrees (Y - * tiled), or temporary BOs for uploading data once (linear). + /* + * Obey HALIGN_16 constraints for Gen8 and Gen9 buffers which are + * multisampled or have an AUX buffer attached to it. + * + * GEN | MSRT | AUX_CCS_* or AUX_MCS + * ------------------------------------------- + * 9 | HALIGN_16 | HALIGN_16 + * 8 | HALIGN_ANY | HALIGN_16 + * 7 | ? | ? + * 6 | ? | ? */ - switch (requested) { - case INTEL_MIPTREE_TILING_ANY: - break; - case INTEL_MIPTREE_TILING_Y: - return I915_TILING_Y; - case INTEL_MIPTREE_TILING_NONE: - return I915_TILING_NONE; - } - - if (num_samples > 1) { - /* From p82 of the Sandy Bridge PRM, dw3[1] of SURFACE_STATE ("Tiled - * Surface"): - * - * [DevSNB+]: For multi-sample render targets, this field must be - * 1. MSRTs can only be tiled. - * - * Our usual reason for preferring X tiling (fast blits using the - * blitting engine) doesn't apply to MSAA, since we'll generally be - * downsampling or upsampling when blitting between the MSAA buffer - * and another buffer, and the blitting engine doesn't support that. - * So use Y tiling, since it makes better use of the cache. - */ - return I915_TILING_Y; - } - - GLenum base_format = _mesa_get_format_base_format(format); - if (base_format == GL_DEPTH_COMPONENT || - base_format == GL_DEPTH_STENCIL_EXT) - return I915_TILING_Y; - - int minimum_pitch = mt->total_width * mt->cpp; - - /* If the width is much smaller than a tile, don't bother tiling. */ - if (minimum_pitch < 64) - return I915_TILING_NONE; - - if (ALIGN(minimum_pitch, 512) >= 32768 || - mt->total_width >= 32768 || mt->total_height >= 32768) { - perf_debug("%dx%d miptree too large to blit, falling back to untiled", - mt->total_width, mt->total_height); - return I915_TILING_NONE; + if (intel_miptree_is_fast_clear_capable(brw, mt)) { + if (brw->gen >= 9 || (brw->gen == 8 && num_samples == 1)) + layout_flags |= MIPTREE_LAYOUT_FORCE_HALIGN16; + } else if (brw->gen >= 9 && num_samples > 1) { + layout_flags |= MIPTREE_LAYOUT_FORCE_HALIGN16; + } else { + /* For now, nothing else has this requirement */ + assert((layout_flags & MIPTREE_LAYOUT_FORCE_HALIGN16) == 0); } - /* Pre-gen6 doesn't have BLORP to handle Y-tiling, so use X-tiling. */ - if (brw->gen < 6) - return I915_TILING_X; + brw_miptree_layout(brw, mt, requested, layout_flags); - /* From the Sandybridge PRM, Volume 1, Part 2, page 32: - * "NOTE: 128BPE Format Color Buffer ( render target ) MUST be either TileX - * or Linear." - * 128 bits per pixel translates to 16 bytes per pixel. This is necessary - * all the way back to 965, but is explicitly permitted on Gen7. - */ - if (brw->gen != 7 && mt->cpp >= 16) - return I915_TILING_X; - - /* From the Ivy Bridge PRM, Vol4 Part1 2.12.2.1 (SURFACE_STATE for most - * messages), on p64, under the heading "Surface Vertical Alignment": - * - * This field must be set to VALIGN_4 for all tiled Y Render Target - * surfaces. - * - * So if the surface is renderable and uses a vertical alignment of 2, - * force it to be X tiled. This is somewhat conservative (it's possible - * that the client won't ever render to this surface), but it's difficult - * to know that ahead of time. And besides, since we use a vertical - * alignment of 4 as often as we can, this shouldn't happen very often. - */ - if (brw->gen == 7 && mt->align_h == 2 && - brw->format_supported_as_render_target[format]) { - return I915_TILING_X; - } + if (mt->disable_aux_buffers) + assert(mt->msaa_layout != INTEL_MSAA_LAYOUT_CMS); - return I915_TILING_Y | I915_TILING_X; + return mt; } @@ -556,32 +561,33 @@ intel_lower_compressed_format(struct brw_context *brw, mesa_format format) struct intel_mipmap_tree * intel_miptree_create(struct brw_context *brw, - GLenum target, - mesa_format format, - GLuint first_level, - GLuint last_level, - GLuint width0, - GLuint height0, - GLuint depth0, - bool expect_accelerated_upload, + GLenum target, + mesa_format format, + GLuint first_level, + GLuint last_level, + GLuint width0, + GLuint height0, + GLuint depth0, GLuint num_samples, enum intel_miptree_tiling_mode requested_tiling, - bool force_all_slices_at_each_lod) + uint32_t layout_flags) { struct intel_mipmap_tree *mt; mesa_format tex_format = format; mesa_format etc_format = MESA_FORMAT_NONE; GLuint total_width, total_height; + uint32_t alloc_flags = 0; format = intel_lower_compressed_format(brw, format); etc_format = (format != tex_format) ? tex_format : MESA_FORMAT_NONE; + assert((layout_flags & MIPTREE_LAYOUT_DISABLE_AUX) == 0); + assert((layout_flags & MIPTREE_LAYOUT_FOR_BO) == 0); mt = intel_miptree_create_layout(brw, target, format, - first_level, last_level, width0, - height0, depth0, - false, num_samples, - force_all_slices_at_each_lod); + first_level, last_level, width0, + height0, depth0, num_samples, + requested_tiling, layout_flags); /* * pitch == 0 || height == 0 indicates the null texture */ @@ -599,42 +605,36 @@ intel_miptree_create(struct brw_context *brw, total_height = ALIGN(total_height, 64); } - uint32_t tiling = intel_miptree_choose_tiling(brw, format, width0, - num_samples, requested_tiling, - mt); bool y_or_x = false; - if (tiling == (I915_TILING_Y | I915_TILING_X)) { + if (mt->tiling == (I915_TILING_Y | I915_TILING_X)) { y_or_x = true; mt->tiling = I915_TILING_Y; - } else { - mt->tiling = tiling; } + if (layout_flags & MIPTREE_LAYOUT_ACCELERATED_UPLOAD) + alloc_flags |= BO_ALLOC_FOR_RENDER; + unsigned long pitch; + mt->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "miptree", total_width, + total_height, mt->cpp, &mt->tiling, + &pitch, alloc_flags); mt->etc_format = etc_format; - mt->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "miptree", - total_width, total_height, mt->cpp, - &mt->tiling, &pitch, - (expect_accelerated_upload ? - BO_ALLOC_FOR_RENDER : 0)); mt->pitch = pitch; /* If the BO is too large to fit in the aperture, we need to use the - * BLT engine to support it. The BLT paths can't currently handle Y-tiling, - * so we need to fall back to X. + * BLT engine to support it. Prior to Sandybridge, the BLT paths can't + * handle Y-tiling, so we need to fall back to X. */ - if (y_or_x && mt->bo->size >= brw->max_gtt_map_object_size) { + if (brw->gen < 6 && y_or_x && mt->bo->size >= brw->max_gtt_map_object_size) { perf_debug("%dx%d miptree larger than aperture; falling back to X-tiled\n", mt->total_width, mt->total_height); mt->tiling = I915_TILING_X; drm_intel_bo_unreference(mt->bo); mt->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "miptree", - total_width, total_height, mt->cpp, - &mt->tiling, &pitch, - (expect_accelerated_upload ? - BO_ALLOC_FOR_RENDER : 0)); + total_width, total_height, mt->cpp, + &mt->tiling, &pitch, alloc_flags); mt->pitch = pitch; } @@ -647,6 +647,7 @@ intel_miptree_create(struct brw_context *brw, if (mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS) { + assert(mt->num_samples > 1); if (!intel_miptree_alloc_mcs(brw, mt, num_samples)) { intel_miptree_release(&mt); return NULL; @@ -658,8 +659,11 @@ intel_miptree_create(struct brw_context *brw, * Allocation of the MCS miptree will be deferred until the first fast * clear actually occurs. */ - if (intel_is_non_msrt_mcs_buffer_supported(brw, mt)) + if (intel_tiling_supports_non_msrt_mcs(brw, mt->tiling) && + intel_miptree_is_fast_clear_capable(brw, mt)) { mt->fast_clear_state = INTEL_FAST_CLEAR_STATE_RESOLVED; + assert(brw->gen < 8 || mt->align_w == 16 || num_samples <= 1); + } return mt; } @@ -671,10 +675,13 @@ intel_miptree_create_for_bo(struct brw_context *brw, uint32_t offset, uint32_t width, uint32_t height, - int pitch) + uint32_t depth, + int pitch, + uint32_t layout_flags) { struct intel_mipmap_tree *mt; uint32_t tiling, swizzle; + GLenum target; drm_intel_bo_get_tiling(bo, &tiling, &swizzle); @@ -689,14 +696,22 @@ intel_miptree_create_for_bo(struct brw_context *brw, */ assert(pitch >= 0); - mt = intel_miptree_create_layout(brw, GL_TEXTURE_2D, format, + target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D; + + /* 'requested' parameter of intel_miptree_create_layout() is relevant + * only for non bo miptree. Tiling for bo is already computed above. + * So, the tiling requested (INTEL_MIPTREE_TILING_ANY) below is + * just a place holder and will not make any change to the miptree + * tiling format. + */ + layout_flags |= MIPTREE_LAYOUT_FOR_BO; + mt = intel_miptree_create_layout(brw, target, format, 0, 0, - width, height, 1, - true, 0, false); - if (!mt) { - free(mt); - return mt; - } + width, height, depth, 0, + INTEL_MIPTREE_TILING_ANY, + layout_flags); + if (!mt) + return NULL; drm_intel_bo_reference(bo); mt->bo = bo; @@ -742,7 +757,9 @@ intel_update_winsys_renderbuffer_miptree(struct brw_context *intel, 0, width, height, - pitch); + 1, + pitch, + 0); if (!singlesample_mt) goto fail; @@ -751,7 +768,8 @@ intel_update_winsys_renderbuffer_miptree(struct brw_context *intel, * Allocation of the MCS miptree will be deferred until the first fast * clear actually occurs. */ - if (intel_is_non_msrt_mcs_buffer_supported(intel, singlesample_mt)) + if (intel_tiling_supports_non_msrt_mcs(intel, singlesample_mt->tiling) && + intel_miptree_is_fast_clear_capable(intel, singlesample_mt)) singlesample_mt->fast_clear_state = INTEL_FAST_CLEAR_STATE_RESOLVED; if (num_samples == 0) { @@ -800,12 +818,13 @@ intel_miptree_create_for_renderbuffer(struct brw_context *brw, GLenum target = num_samples > 1 ? GL_TEXTURE_2D_MULTISAMPLE : GL_TEXTURE_2D; mt = intel_miptree_create(brw, target, format, 0, 0, - width, height, depth, true, num_samples, - INTEL_MIPTREE_TILING_ANY, false); + width, height, depth, num_samples, + INTEL_MIPTREE_TILING_ANY, + MIPTREE_LAYOUT_ACCELERATED_UPLOAD); if (!mt) goto fail; - if (brw_is_hiz_depth_format(brw, format)) { + if (intel_miptree_wants_hiz_buffer(brw, mt)) { ok = intel_miptree_alloc_hiz(brw, mt); if (!ok) goto fail; @@ -829,7 +848,7 @@ intel_miptree_reference(struct intel_mipmap_tree **dst, if (src) { src->refcount++; - DBG("%s %p refcount now %d\n", __FUNCTION__, src, src->refcount); + DBG("%s %p refcount now %d\n", __func__, src, src->refcount); } *dst = src; @@ -842,15 +861,21 @@ intel_miptree_release(struct intel_mipmap_tree **mt) if (!*mt) return; - DBG("%s %p refcount will be %d\n", __FUNCTION__, *mt, (*mt)->refcount - 1); + DBG("%s %p refcount will be %d\n", __func__, *mt, (*mt)->refcount - 1); if (--(*mt)->refcount <= 0) { GLuint i; - DBG("%s deleting %p\n", __FUNCTION__, *mt); + DBG("%s deleting %p\n", __func__, *mt); drm_intel_bo_unreference((*mt)->bo); intel_miptree_release(&(*mt)->stencil_mt); - intel_miptree_release(&(*mt)->hiz_mt); + if ((*mt)->hiz_buf) { + if ((*mt)->hiz_buf->mt) + intel_miptree_release(&(*mt)->hiz_buf->mt); + else + drm_intel_bo_unreference((*mt)->hiz_buf->bo); + free((*mt)->hiz_buf); + } intel_miptree_release(&(*mt)->mcs_mt); intel_resolve_map_clear(&(*mt)->hiz_map); @@ -956,7 +981,7 @@ intel_miptree_set_level_info(struct intel_mipmap_tree *mt, mt->level[level].level_x = x; mt->level[level].level_y = y; - DBG("%s level %d, depth %d, offset %d,%d\n", __FUNCTION__, + DBG("%s level %d, depth %d, offset %d,%d\n", __func__, level, d, x, y); assert(mt->level[level].slice == NULL); @@ -981,7 +1006,7 @@ intel_miptree_set_image_offset(struct intel_mipmap_tree *mt, mt->level[level].slice[img].y_offset = mt->level[level].level_y + y; DBG("%s level %d img %d pos %d,%d\n", - __FUNCTION__, level, img, + __func__, level, img, mt->level[level].slice[img].x_offset, mt->level[level].slice[img].y_offset); } @@ -1244,7 +1269,12 @@ intel_miptree_copy_teximage(struct brw_context *brw, intel_texture_object(intelImage->base.Base.TexObject); int level = intelImage->base.Base.Level; int face = intelImage->base.Base.Face; - GLuint depth = intelImage->base.Base.Depth; + + GLuint depth; + if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) + depth = intelImage->base.Base.Height; + else + depth = intelImage->base.Base.Depth; if (!invalidate) { for (int slice = 0; slice < depth; slice++) { @@ -1256,13 +1286,14 @@ intel_miptree_copy_teximage(struct brw_context *brw, intel_obj->needs_validate = true; } -bool +static bool intel_miptree_alloc_mcs(struct brw_context *brw, struct intel_mipmap_tree *mt, GLuint num_samples) { assert(brw->gen >= 7); /* MCS only used on Gen7+ */ assert(mt->mcs_mt == NULL); + assert(!mt->disable_aux_buffers); /* Choose the correct format for the MCS buffer. All that really matters * is that we allocate the right buffer size, since we'll always be @@ -1300,10 +1331,9 @@ intel_miptree_alloc_mcs(struct brw_context *brw, mt->logical_width0, mt->logical_height0, mt->logical_depth0, - true, 0 /* num_samples */, INTEL_MIPTREE_TILING_Y, - false); + MIPTREE_LAYOUT_ACCELERATED_UPLOAD); /* From the Ivy Bridge PRM, Vol 2 Part 1 p326: * @@ -1329,6 +1359,7 @@ intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, struct intel_mipmap_tree *mt) { assert(mt->mcs_mt == NULL); + assert(!mt->disable_aux_buffers); /* The format of the MCS buffer is opaque to the driver; all that matters * is that we get its size and pitch right. We'll pretend that the format @@ -1350,6 +1381,9 @@ intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, unsigned mcs_height = ALIGN(mt->logical_height0, height_divisor) / height_divisor; assert(mt->logical_depth0 == 1); + uint32_t layout_flags = MIPTREE_LAYOUT_ACCELERATED_UPLOAD; + if (brw->gen >= 8) + layout_flags |= MIPTREE_LAYOUT_FORCE_HALIGN16; mt->mcs_mt = intel_miptree_create(brw, mt->target, format, @@ -1358,10 +1392,9 @@ intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, mcs_width, mcs_height, mt->logical_depth0, - true, 0 /* num_samples */, INTEL_MIPTREE_TILING_Y, - false); + layout_flags); return mt->mcs_mt; } @@ -1377,7 +1410,7 @@ intel_miptree_level_enable_hiz(struct brw_context *brw, struct intel_mipmap_tree *mt, uint32_t level) { - assert(mt->hiz_mt); + assert(mt->hiz_buf); if (brw->gen >= 8 || brw->is_haswell) { uint32_t width = minify(mt->physical_width0, level); @@ -1401,27 +1434,280 @@ intel_miptree_level_enable_hiz(struct brw_context *brw, } +/** + * Helper for intel_miptree_alloc_hiz() that determines the required hiz + * buffer dimensions and allocates a bo for the hiz buffer. + */ +static struct intel_miptree_aux_buffer * +intel_gen7_hiz_buf_create(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + unsigned z_width = mt->logical_width0; + unsigned z_height = mt->logical_height0; + const unsigned z_depth = MAX2(mt->logical_depth0, 1); + unsigned hz_width, hz_height; + struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); + + if (!buf) + return NULL; + + /* Gen7 PRM Volume 2, Part 1, 11.5.3 "Hierarchical Depth Buffer" documents + * adjustments required for Z_Height and Z_Width based on multisampling. + */ + switch (mt->num_samples) { + case 0: + case 1: + break; + case 2: + case 4: + z_width *= 2; + z_height *= 2; + break; + case 8: + z_width *= 4; + z_height *= 2; + break; + default: + unreachable("unsupported sample count"); + } + + const unsigned vertical_align = 8; /* 'j' in the docs */ + const unsigned H0 = z_height; + const unsigned h0 = ALIGN(H0, vertical_align); + const unsigned h1 = ALIGN(minify(H0, 1), vertical_align); + const unsigned Z0 = z_depth; + + /* HZ_Width (bytes) = ceiling(Z_Width / 16) * 16 */ + hz_width = ALIGN(z_width, 16); + + if (mt->target == GL_TEXTURE_3D) { + unsigned H_i = H0; + unsigned Z_i = Z0; + hz_height = 0; + for (int level = mt->first_level; level <= mt->last_level; ++level) { + unsigned h_i = ALIGN(H_i, vertical_align); + /* sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ + hz_height += h_i * Z_i; + H_i = minify(H_i, 1); + Z_i = minify(Z_i, 1); + } + /* HZ_Height = + * (1/2) * sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) + */ + hz_height = DIV_ROUND_UP(hz_height, 2); + } else { + const unsigned hz_qpitch = h0 + h1 + (12 * vertical_align); + if (mt->target == GL_TEXTURE_CUBE_MAP_ARRAY || + mt->target == GL_TEXTURE_CUBE_MAP) { + /* HZ_Height (rows) = Ceiling ( ( Q_pitch * Z_depth * 6/2) /8 ) * 8 */ + hz_height = DIV_ROUND_UP(hz_qpitch * Z0 * 6, 2 * 8) * 8; + } else { + /* HZ_Height (rows) = Ceiling ( ( Q_pitch * Z_depth/2) /8 ) * 8 */ + hz_height = DIV_ROUND_UP(hz_qpitch * Z0, 2 * 8) * 8; + } + } + + unsigned long pitch; + uint32_t tiling = I915_TILING_Y; + buf->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "hiz", + hz_width, hz_height, 1, + &tiling, &pitch, + BO_ALLOC_FOR_RENDER); + if (!buf->bo) { + free(buf); + return NULL; + } else if (tiling != I915_TILING_Y) { + drm_intel_bo_unreference(buf->bo); + free(buf); + return NULL; + } + + buf->pitch = pitch; + + return buf; +} + + +/** + * Helper for intel_miptree_alloc_hiz() that determines the required hiz + * buffer dimensions and allocates a bo for the hiz buffer. + */ +static struct intel_miptree_aux_buffer * +intel_gen8_hiz_buf_create(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + unsigned z_width = mt->logical_width0; + unsigned z_height = mt->logical_height0; + const unsigned z_depth = MAX2(mt->logical_depth0, 1); + unsigned hz_width, hz_height; + struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); + + if (!buf) + return NULL; + + /* Gen7 PRM Volume 2, Part 1, 11.5.3 "Hierarchical Depth Buffer" documents + * adjustments required for Z_Height and Z_Width based on multisampling. + */ + switch (mt->num_samples) { + case 0: + case 1: + break; + case 2: + case 4: + z_width *= 2; + z_height *= 2; + break; + case 8: + z_width *= 4; + z_height *= 2; + break; + default: + unreachable("unsupported sample count"); + } + + const unsigned vertical_align = 8; /* 'j' in the docs */ + const unsigned H0 = z_height; + const unsigned h0 = ALIGN(H0, vertical_align); + const unsigned h1 = ALIGN(minify(H0, 1), vertical_align); + const unsigned Z0 = z_depth; + + /* HZ_Width (bytes) = ceiling(Z_Width / 16) * 16 */ + hz_width = ALIGN(z_width, 16); + + unsigned H_i = H0; + unsigned Z_i = Z0; + unsigned sum_h_i = 0; + unsigned hz_height_3d_sum = 0; + for (int level = mt->first_level; level <= mt->last_level; ++level) { + unsigned i = level - mt->first_level; + unsigned h_i = ALIGN(H_i, vertical_align); + /* sum(i=2 to m; h_i) */ + if (i >= 2) { + sum_h_i += h_i; + } + /* sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ + hz_height_3d_sum += h_i * Z_i; + H_i = minify(H_i, 1); + Z_i = minify(Z_i, 1); + } + /* HZ_QPitch = h0 + max(h1, sum(i=2 to m; h_i)) */ + buf->qpitch = h0 + MAX2(h1, sum_h_i); + + if (mt->target == GL_TEXTURE_3D) { + /* (1/2) * sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ + hz_height = DIV_ROUND_UP(hz_height_3d_sum, 2); + } else { + /* HZ_Height (rows) = ceiling( (HZ_QPitch/2)/8) *8 * Z_Depth */ + hz_height = DIV_ROUND_UP(buf->qpitch, 2 * 8) * 8 * Z0; + if (mt->target == GL_TEXTURE_CUBE_MAP_ARRAY || + mt->target == GL_TEXTURE_CUBE_MAP) { + /* HZ_Height (rows) = ceiling( (HZ_QPitch/2)/8) *8 * 6 * Z_Depth + * + * We can can just take our hz_height calculation from above, and + * multiply by 6 for the cube map and cube map array types. + */ + hz_height *= 6; + } + } + + unsigned long pitch; + uint32_t tiling = I915_TILING_Y; + buf->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "hiz", + hz_width, hz_height, 1, + &tiling, &pitch, + BO_ALLOC_FOR_RENDER); + if (!buf->bo) { + free(buf); + return NULL; + } else if (tiling != I915_TILING_Y) { + drm_intel_bo_unreference(buf->bo); + free(buf); + return NULL; + } + + buf->pitch = pitch; + + return buf; +} + + +static struct intel_miptree_aux_buffer * +intel_hiz_miptree_buf_create(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); + uint32_t layout_flags = MIPTREE_LAYOUT_ACCELERATED_UPLOAD; + + if (brw->gen == 6) + layout_flags |= MIPTREE_LAYOUT_FORCE_ALL_SLICE_AT_LOD; + + if (!buf) + return NULL; + + buf->mt = intel_miptree_create(brw, + mt->target, + mt->format, + mt->first_level, + mt->last_level, + mt->logical_width0, + mt->logical_height0, + mt->logical_depth0, + mt->num_samples, + INTEL_MIPTREE_TILING_ANY, + layout_flags); + if (!buf->mt) { + free(buf); + return NULL; + } + + buf->bo = buf->mt->bo; + buf->pitch = buf->mt->pitch; + buf->qpitch = buf->mt->qpitch; + + return buf; +} + +bool +intel_miptree_wants_hiz_buffer(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + if (!brw->has_hiz) + return false; + + if (mt->hiz_buf != NULL) + return false; + + if (mt->disable_aux_buffers) + return false; + + switch (mt->format) { + case MESA_FORMAT_Z_FLOAT32: + case MESA_FORMAT_Z32_FLOAT_S8X24_UINT: + case MESA_FORMAT_Z24_UNORM_X8_UINT: + case MESA_FORMAT_Z24_UNORM_S8_UINT: + case MESA_FORMAT_Z_UNORM16: + return true; + default: + return false; + } +} bool intel_miptree_alloc_hiz(struct brw_context *brw, struct intel_mipmap_tree *mt) { - assert(mt->hiz_mt == NULL); - const bool force_all_slices_at_each_lod = brw->gen == 6; - mt->hiz_mt = intel_miptree_create(brw, - mt->target, - mt->format, - mt->first_level, - mt->last_level, - mt->logical_width0, - mt->logical_height0, - mt->logical_depth0, - true, - mt->num_samples, - INTEL_MIPTREE_TILING_ANY, - force_all_slices_at_each_lod); + assert(mt->hiz_buf == NULL); + assert(!mt->disable_aux_buffers); - if (!mt->hiz_mt) + if (brw->gen == 7) { + mt->hiz_buf = intel_gen7_hiz_buf_create(brw, mt); + } else if (brw->gen >= 8) { + mt->hiz_buf = intel_gen8_hiz_buf_create(brw, mt); + } else { + mt->hiz_buf = intel_hiz_miptree_buf_create(brw, mt); + } + + if (!mt->hiz_buf) return false; /* Mark that all slices need a HiZ resolve. */ @@ -1773,7 +2059,7 @@ intel_miptree_map_gtt(struct brw_context *brw, } DBG("%s: %d,%d %dx%d from mt %p (%s) " - "%"PRIiPTR",%"PRIiPTR" = %p/%d\n", __FUNCTION__, + "%"PRIiPTR",%"PRIiPTR" = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), x, y, map->ptr, map->stride); @@ -1798,28 +2084,34 @@ intel_miptree_map_blit(struct brw_context *brw, map->mt = intel_miptree_create(brw, GL_TEXTURE_2D, mt->format, 0, 0, map->w, map->h, 1, - false, 0, - INTEL_MIPTREE_TILING_NONE, - false); + 0, INTEL_MIPTREE_TILING_NONE, 0); + if (!map->mt) { fprintf(stderr, "Failed to allocate blit temporary\n"); goto fail; } map->stride = map->mt->pitch; - if (!intel_miptree_blit(brw, - mt, level, slice, - map->x, map->y, false, - map->mt, 0, 0, - 0, 0, false, - map->w, map->h, GL_COPY)) { - fprintf(stderr, "Failed to blit\n"); - goto fail; + /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no + * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless + * invalidate is set, since we'll be writing the whole rectangle from our + * temporary buffer back out. + */ + if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { + if (!intel_miptree_blit(brw, + mt, level, slice, + map->x, map->y, false, + map->mt, 0, 0, + 0, 0, false, + map->w, map->h, GL_COPY)) { + fprintf(stderr, "Failed to blit\n"); + goto fail; + } } map->ptr = intel_miptree_map_raw(brw, map->mt); - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), level, slice, map->ptr, map->stride); @@ -1869,7 +2161,7 @@ intel_miptree_map_movntdqa(struct brw_context *brw, assert(map->mode & GL_MAP_READ_BIT); assert(!(map->mode & GL_MAP_WRITE_BIT)); - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), level, slice, map->ptr, map->stride); @@ -1964,11 +2256,11 @@ intel_miptree_map_s8(struct brw_context *brw, intel_miptree_unmap_raw(brw, mt); - DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, map->x + image_x, map->y + image_y, map->ptr, map->stride); } else { - DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, map->ptr, map->stride); } @@ -2126,13 +2418,13 @@ intel_miptree_map_depthstencil(struct brw_context *brw, intel_miptree_unmap_raw(brw, z_mt); DBG("%s: %d,%d %dx%d from z mt %p %d,%d, s mt %p %d,%d = %p/%d\n", - __FUNCTION__, + __func__, map->x, map->y, map->w, map->h, z_mt, map->x + z_image_x, map->y + z_image_y, s_mt, map->x + s_image_x, map->y + s_image_y, map->ptr, map->stride); } else { - DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, map->ptr, map->stride); } @@ -2186,7 +2478,7 @@ intel_miptree_unmap_depthstencil(struct brw_context *brw, intel_miptree_unmap_raw(brw, z_mt); DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n", - __FUNCTION__, + __func__, map->x, map->y, map->w, map->h, z_mt, _mesa_get_format_name(z_mt->format), map->x + z_image_x, map->y + z_image_y, @@ -2253,12 +2545,41 @@ can_blit_slice(struct intel_mipmap_tree *mt, if (image_x >= 32768 || image_y >= 32768) return false; + /* See intel_miptree_blit() for details on the 32k pitch limit. */ if (mt->pitch >= 32768) return false; return true; } +static bool +use_intel_mipree_map_blit(struct brw_context *brw, + struct intel_mipmap_tree *mt, + GLbitfield mode, + unsigned int level, + unsigned int slice) +{ + if (brw->has_llc && + /* It's probably not worth swapping to the blit ring because of + * all the overhead involved. + */ + !(mode & GL_MAP_WRITE_BIT) && + !mt->compressed && + (mt->tiling == I915_TILING_X || + /* Prior to Sandybridge, the blitter can't handle Y tiling */ + (brw->gen >= 6 && mt->tiling == I915_TILING_Y)) && + can_blit_slice(mt, level, slice)) + return true; + + if (mt->tiling != I915_TILING_NONE && + mt->bo->size >= brw->max_gtt_map_object_size) { + assert(can_blit_slice(mt, level, slice)); + return true; + } + + return false; +} + /** * Parameter \a out_stride has type ptrdiff_t not because the buffer stride may * exceed 32 bits but to diminish the likelihood subtle bugs in pointer @@ -2306,21 +2627,12 @@ intel_miptree_map(struct brw_context *brw, intel_miptree_map_etc(brw, mt, map, level, slice); } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) { intel_miptree_map_depthstencil(brw, mt, map, level, slice); - } - /* See intel_miptree_blit() for details on the 32k pitch limit. */ - else if (brw->has_llc && - !(mode & GL_MAP_WRITE_BIT) && - !mt->compressed && - (mt->tiling == I915_TILING_X || - (brw->gen >= 6 && mt->tiling == I915_TILING_Y)) && - can_blit_slice(mt, level, slice)) { - intel_miptree_map_blit(brw, mt, map, level, slice); - } else if (mt->tiling != I915_TILING_NONE && - mt->bo->size >= brw->max_gtt_map_object_size) { - assert(can_blit_slice(mt, level, slice)); + } else if (use_intel_mipree_map_blit(brw, mt, mode, level, slice)) { intel_miptree_map_blit(brw, mt, map, level, slice); #if defined(USE_SSE41) - } else if (!(mode & GL_MAP_WRITE_BIT) && !mt->compressed && cpu_has_sse4_1) { + } else if (!(mode & GL_MAP_WRITE_BIT) && + !mt->compressed && cpu_has_sse4_1 && + (mt->pitch % 16 == 0)) { intel_miptree_map_movntdqa(brw, mt, map, level, slice); #endif } else { @@ -2347,7 +2659,7 @@ intel_miptree_unmap(struct brw_context *brw, if (!map) return; - DBG("%s: mt %p (%s) level %d slice %d\n", __FUNCTION__, + DBG("%s: mt %p (%s) level %d slice %d\n", __func__, mt, _mesa_get_format_name(mt->format), level, slice); if (mt->format == MESA_FORMAT_S_UINT8) {