X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fintel_mipmap_tree.c;h=8616c0193c8a0addb7d61a16a820cea6133f60df;hb=b639ed2f1b170d1184c6d94c88c826c51ffc8726;hp=dd7e57aea84f77a7c9ad143eda80b6c92d4c45cd;hpb=cf29913aa156accbe60cb35f9a0bd2c21726cfa3;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c index dd7e57aea84..8616c0193c8 100644 --- a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c +++ b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c @@ -29,7 +29,6 @@ #include #include "intel_batchbuffer.h" -#include "intel_chipset.h" #include "intel_mipmap_tree.h" #include "intel_resolve_map.h" #include "intel_tex.h" @@ -50,12 +49,18 @@ #define FILE_DEBUG_FLAG DEBUG_MIPTREE +static bool +intel_miptree_alloc_mcs(struct brw_context *brw, + struct intel_mipmap_tree *mt, + GLuint num_samples); + /** * Determine which MSAA layout should be used by the MSAA surface being * created, based on the chip generation and the surface type. */ static enum intel_msaa_layout -compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target) +compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target, + bool disable_aux_buffers) { /* Prior to Gen7, all MSAA surfaces used IMS layout. */ if (brw->gen < 7) @@ -68,14 +73,6 @@ compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target) case GL_DEPTH_STENCIL: return INTEL_MSAA_LAYOUT_IMS; default: - /* Disable MCS on Broadwell for now. We can enable it once things - * are working without it. - */ - if (brw->gen >= 8) { - perf_debug("Missing CMS support on Broadwell.\n"); - return INTEL_MSAA_LAYOUT_UMS; - } - /* From the Ivy Bridge PRM, Vol4 Part1 p77 ("MCS Enable"): * * This field must be set to 0 for all SINT MSRTs when all RT channels @@ -87,9 +84,12 @@ compute_msaa_layout(struct brw_context *brw, mesa_format format, GLenum target) * would require converting between CMS and UMS MSAA layouts on the fly, * which is expensive. */ - if (_mesa_get_format_datatype(format) == GL_INT) { - /* TODO: is this workaround needed for future chipsets? */ - assert(brw->gen == 7); + if (brw->gen == 7 && _mesa_get_format_datatype(format) == GL_INT) { + return INTEL_MSAA_LAYOUT_UMS; + } else if (disable_aux_buffers) { + /* We can't use the CMS layout because it uses an aux buffer, the MCS + * buffer. So fallback to UMS, which is identical to CMS without the + * MCS. */ return INTEL_MSAA_LAYOUT_UMS; } else { return INTEL_MSAA_LAYOUT_CMS; @@ -146,7 +146,7 @@ intel_get_non_msrt_mcs_alignment(struct brw_context *brw, { switch (mt->tiling) { default: - assert(!"Non-MSRT MCS requires X or Y tiling"); + unreachable("Non-MSRT MCS requires X or Y tiling"); /* In release builds, fall through */ case I915_TILING_Y: *width_px = 32 / mt->cpp; @@ -179,7 +179,10 @@ intel_is_non_msrt_mcs_buffer_supported(struct brw_context *brw, struct intel_mipmap_tree *mt) { /* MCS support does not exist prior to Gen7 */ - if (brw->gen < 7 || brw->gen >= 8) + if (brw->gen < 7) + return false; + + if (mt->disable_aux_buffers) return false; /* MCS is only supported for color buffers */ @@ -195,10 +198,23 @@ intel_is_non_msrt_mcs_buffer_supported(struct brw_context *brw, return false; if (mt->cpp != 4 && mt->cpp != 8 && mt->cpp != 16) return false; - if (mt->first_level != 0 || mt->last_level != 0) + if (mt->first_level != 0 || mt->last_level != 0) { + if (brw->gen >= 8) { + perf_debug("Multi-LOD fast clear - giving up (%dx%dx%d).\n", + mt->logical_width0, mt->logical_height0, mt->last_level); + } + return false; - if (mt->physical_depth0 != 1) + } + if (mt->physical_depth0 != 1) { + if (brw->gen >= 8) { + perf_debug("Layered fast clear - giving up. (%dx%d%d)\n", + mt->logical_width0, mt->logical_height0, + mt->physical_depth0); + } + return false; + } /* There's no point in using an MCS buffer if the surface isn't in a * renderable format. @@ -232,7 +248,7 @@ intel_depth_format_for_depthstencil_format(mesa_format format) { * intel_miptree_create_for_bo(). If true, then do not create * \c stencil_mt. */ -struct intel_mipmap_tree * +static struct intel_mipmap_tree * intel_miptree_create_layout(struct brw_context *brw, GLenum target, mesa_format format, @@ -242,17 +258,39 @@ intel_miptree_create_layout(struct brw_context *brw, GLuint height0, GLuint depth0, bool for_bo, - GLuint num_samples) + GLuint num_samples, + bool force_all_slices_at_each_lod, + bool disable_aux_buffers) { struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1); if (!mt) return NULL; - DBG("%s target %s format %s level %d..%d slices %d <-- %p\n", __FUNCTION__, + DBG("%s target %s format %s level %d..%d slices %d <-- %p\n", __func__, _mesa_lookup_enum_by_nr(target), _mesa_get_format_name(format), first_level, last_level, depth0, mt); + if (target == GL_TEXTURE_1D_ARRAY) { + /* For a 1D Array texture the OpenGL API will treat the height0 + * parameter as the number of array slices. For Intel hardware, we treat + * the 1D array as a 2D Array with a height of 1. + * + * So, when we first come through this path to create a 1D Array + * texture, height0 stores the number of slices, and depth0 is 1. In + * this case, we want to swap height0 and depth0. + * + * Since some miptrees will be created based on the base miptree, we may + * come through this path and see height0 as 1 and depth0 being the + * number of slices. In this case we don't need to do the swap. + */ + assert(height0 == 1 || depth0 == 1); + if (height0 > 1) { + depth0 = height0; + height0 = 1; + } + } + mt->target = target; mt->format = format; mt->first_level = first_level; @@ -261,6 +299,8 @@ intel_miptree_create_layout(struct brw_context *brw, mt->logical_height0 = height0; mt->logical_depth0 = depth0; mt->fast_clear_state = INTEL_FAST_CLEAR_STATE_NO_MCS; + mt->disable_aux_buffers = disable_aux_buffers; + exec_list_make_empty(&mt->hiz_map); /* The cpp is bytes per (1, blockheight)-sized block for compressed * textures. This is why you'll see divides by blockheight all over @@ -277,9 +317,31 @@ intel_miptree_create_layout(struct brw_context *brw, if (num_samples > 1) { /* Adjust width/height/depth for MSAA */ - mt->msaa_layout = compute_msaa_layout(brw, format, mt->target); + mt->msaa_layout = compute_msaa_layout(brw, format, + mt->target, mt->disable_aux_buffers); if (mt->msaa_layout == INTEL_MSAA_LAYOUT_IMS) { - /* In the Sandy Bridge PRM, volume 4, part 1, page 31, it says: + /* From the Ivybridge PRM, Volume 1, Part 1, page 108: + * "If the surface is multisampled and it is a depth or stencil + * surface or Multisampled Surface StorageFormat in SURFACE_STATE is + * MSFMT_DEPTH_STENCIL, WL and HL must be adjusted as follows before + * proceeding: + * + * +----------------------------------------------------------------+ + * | Num Multisamples | W_l = | H_l = | + * +----------------------------------------------------------------+ + * | 2 | ceiling(W_l / 2) * 4 | H_l (no adjustment) | + * | 4 | ceiling(W_l / 2) * 4 | ceiling(H_l / 2) * 4 | + * | 8 | ceiling(W_l / 2) * 8 | ceiling(H_l / 2) * 4 | + * | 16 | ceiling(W_l / 2) * 8 | ceiling(H_l / 2) * 8 | + * +----------------------------------------------------------------+ + * " + * + * Note that MSFMT_DEPTH_STENCIL just means the IMS (interleaved) + * format rather than UMS/CMS (array slices). The Sandybridge PRM, + * Volume 1, Part 1, Page 111 has the same formula for 4x MSAA. + * + * Another more complicated explanation for these adjustments comes + * from the Sandybridge PRM, volume 4, part 1, page 31: * * "Any of the other messages (sample*, LOD, load4) used with a * (4x) multisampled surface will in-effect sample a surface with @@ -331,7 +393,7 @@ intel_miptree_create_layout(struct brw_context *brw, /* num_samples should already have been quantized to 0, 1, 2, 4, or * 8. */ - assert(false); + unreachable("not reached"); } } else { /* Non-interleaved */ @@ -339,18 +401,29 @@ intel_miptree_create_layout(struct brw_context *brw, } } - /* array_spacing_lod0 is only used for non-IMS MSAA surfaces. TODO: can we - * use it elsewhere? + /* Set array_layout to ALL_SLICES_AT_EACH_LOD when array_spacing_lod0 can + * be used. array_spacing_lod0 is only used for non-IMS MSAA surfaces on + * Gen 7 and 8. On Gen 8 and 9 this layout is not available but it is still + * used on Gen8 to make it pick a qpitch value which doesn't include space + * for the mipmaps. On Gen9 this is not necessary because it will + * automatically pick a packed qpitch value whenever mt->first_level == + * mt->last_level. + * TODO: can we use it elsewhere? + * TODO: also disable this on Gen8 and pick the qpitch value like Gen9 */ - switch (mt->msaa_layout) { - case INTEL_MSAA_LAYOUT_NONE: - case INTEL_MSAA_LAYOUT_IMS: - mt->array_spacing_lod0 = false; - break; - case INTEL_MSAA_LAYOUT_UMS: - case INTEL_MSAA_LAYOUT_CMS: - mt->array_spacing_lod0 = true; - break; + if (brw->gen >= 9) { + mt->array_layout = ALL_LOD_IN_EACH_SLICE; + } else { + switch (mt->msaa_layout) { + case INTEL_MSAA_LAYOUT_NONE: + case INTEL_MSAA_LAYOUT_IMS: + mt->array_layout = ALL_LOD_IN_EACH_SLICE; + break; + case INTEL_MSAA_LAYOUT_UMS: + case INTEL_MSAA_LAYOUT_CMS: + mt->array_layout = ALL_SLICES_AT_EACH_LOD; + break; + } } if (target == GL_TEXTURE_CUBE_MAP) { @@ -365,7 +438,9 @@ intel_miptree_create_layout(struct brw_context *brw, if (!for_bo && _mesa_get_format_base_format(format) == GL_DEPTH_STENCIL && (brw->must_use_separate_stencil || - (brw->has_separate_stencil && brw_is_hiz_depth_format(brw, format)))) { + (brw->has_separate_stencil && + intel_miptree_wants_hiz_buffer(brw, mt)))) { + const bool force_all_slices_at_each_lod = brw->gen == 6; mt->stencil_mt = intel_miptree_create(brw, mt->target, MESA_FORMAT_S_UINT8, @@ -376,7 +451,8 @@ intel_miptree_create_layout(struct brw_context *brw, mt->logical_depth0, true, num_samples, - INTEL_MIPTREE_TILING_ANY); + INTEL_MIPTREE_TILING_ANY, + force_all_slices_at_each_lod); if (!mt->stencil_mt) { intel_miptree_release(&mt); return NULL; @@ -394,8 +470,14 @@ intel_miptree_create_layout(struct brw_context *brw, } } + if (force_all_slices_at_each_lod) + mt->array_layout = ALL_SLICES_AT_EACH_LOD; + brw_miptree_layout(brw, mt); + if (mt->disable_aux_buffers) + assert(mt->msaa_layout != INTEL_MSAA_LAYOUT_CMS); + return mt; } @@ -450,6 +532,13 @@ intel_miptree_choose_tiling(struct brw_context *brw, base_format == GL_DEPTH_STENCIL_EXT) return I915_TILING_Y; + /* 1D textures (and 1D array textures) don't get any benefit from tiling, + * in fact it leads to a less efficient use of memory space and bandwidth + * due to tile alignment. + */ + if (mt->logical_height0 == 1) + return I915_TILING_NONE; + int minimum_pitch = mt->total_width * mt->cpp; /* If the width is much smaller than a tile, don't bother tiling. */ @@ -470,10 +559,10 @@ intel_miptree_choose_tiling(struct brw_context *brw, /* From the Sandybridge PRM, Volume 1, Part 2, page 32: * "NOTE: 128BPE Format Color Buffer ( render target ) MUST be either TileX * or Linear." - * 128 bits per pixel translates to 16 bytes per pixel. This is necessary - * all the way back to 965, but is explicitly permitted on Gen7. + * 128 bits per pixel translates to 16 bytes per pixel. This is necessary + * all the way back to 965, but is permitted on Gen7+. */ - if (brw->gen != 7 && mt->cpp >= 16) + if (brw->gen < 7 && mt->cpp >= 16) return I915_TILING_X; /* From the Ivy Bridge PRM, Vol4 Part1 2.12.2.1 (SURFACE_STATE for most @@ -548,7 +637,8 @@ intel_miptree_create(struct brw_context *brw, GLuint depth0, bool expect_accelerated_upload, GLuint num_samples, - enum intel_miptree_tiling_mode requested_tiling) + enum intel_miptree_tiling_mode requested_tiling, + bool force_all_slices_at_each_lod) { struct intel_mipmap_tree *mt; mesa_format tex_format = format; @@ -562,7 +652,9 @@ intel_miptree_create(struct brw_context *brw, mt = intel_miptree_create_layout(brw, target, format, first_level, last_level, width0, height0, depth0, - false, num_samples); + false, num_samples, + force_all_slices_at_each_lod, + false /*disable_aux_buffers*/); /* * pitch == 0 || height == 0 indicates the null texture */ @@ -602,10 +694,10 @@ intel_miptree_create(struct brw_context *brw, mt->pitch = pitch; /* If the BO is too large to fit in the aperture, we need to use the - * BLT engine to support it. The BLT paths can't currently handle Y-tiling, - * so we need to fall back to X. + * BLT engine to support it. Prior to Sandybridge, the BLT paths can't + * handle Y-tiling, so we need to fall back to X. */ - if (y_or_x && mt->bo->size >= brw->max_gtt_map_object_size) { + if (brw->gen < 6 && y_or_x && mt->bo->size >= brw->max_gtt_map_object_size) { perf_debug("%dx%d miptree larger than aperture; falling back to X-tiled\n", mt->total_width, mt->total_height); @@ -652,10 +744,13 @@ intel_miptree_create_for_bo(struct brw_context *brw, uint32_t offset, uint32_t width, uint32_t height, - int pitch) + uint32_t depth, + int pitch, + bool disable_aux_buffers) { struct intel_mipmap_tree *mt; uint32_t tiling, swizzle; + GLenum target; drm_intel_bo_get_tiling(bo, &tiling, &swizzle); @@ -670,14 +765,15 @@ intel_miptree_create_for_bo(struct brw_context *brw, */ assert(pitch >= 0); - mt = intel_miptree_create_layout(brw, GL_TEXTURE_2D, format, + target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D; + + mt = intel_miptree_create_layout(brw, target, format, 0, 0, - width, height, 1, - true, 0 /* num_samples */); - if (!mt) { - free(mt); - return mt; - } + width, height, depth, + true, 0, false, + disable_aux_buffers); + if (!mt) + return NULL; drm_intel_bo_reference(bo); mt->bo = bo; @@ -723,7 +819,9 @@ intel_update_winsys_renderbuffer_miptree(struct brw_context *intel, 0, width, height, - pitch); + 1, + pitch, + false); if (!singlesample_mt) goto fail; @@ -782,11 +880,11 @@ intel_miptree_create_for_renderbuffer(struct brw_context *brw, mt = intel_miptree_create(brw, target, format, 0, 0, width, height, depth, true, num_samples, - INTEL_MIPTREE_TILING_ANY); + INTEL_MIPTREE_TILING_ANY, false); if (!mt) goto fail; - if (brw_is_hiz_depth_format(brw, format)) { + if (intel_miptree_wants_hiz_buffer(brw, mt)) { ok = intel_miptree_alloc_hiz(brw, mt); if (!ok) goto fail; @@ -810,7 +908,7 @@ intel_miptree_reference(struct intel_mipmap_tree **dst, if (src) { src->refcount++; - DBG("%s %p refcount now %d\n", __FUNCTION__, src, src->refcount); + DBG("%s %p refcount now %d\n", __func__, src, src->refcount); } *dst = src; @@ -823,15 +921,21 @@ intel_miptree_release(struct intel_mipmap_tree **mt) if (!*mt) return; - DBG("%s %p refcount will be %d\n", __FUNCTION__, *mt, (*mt)->refcount - 1); + DBG("%s %p refcount will be %d\n", __func__, *mt, (*mt)->refcount - 1); if (--(*mt)->refcount <= 0) { GLuint i; - DBG("%s deleting %p\n", __FUNCTION__, *mt); + DBG("%s deleting %p\n", __func__, *mt); drm_intel_bo_unreference((*mt)->bo); intel_miptree_release(&(*mt)->stencil_mt); - intel_miptree_release(&(*mt)->hiz_mt); + if ((*mt)->hiz_buf) { + if ((*mt)->hiz_buf->mt) + intel_miptree_release(&(*mt)->hiz_buf->mt); + else + drm_intel_bo_unreference((*mt)->hiz_buf->bo); + free((*mt)->hiz_buf); + } intel_miptree_release(&(*mt)->mcs_mt); intel_resolve_map_clear(&(*mt)->hiz_map); @@ -937,7 +1041,7 @@ intel_miptree_set_level_info(struct intel_mipmap_tree *mt, mt->level[level].level_x = x; mt->level[level].level_y = y; - DBG("%s level %d, depth %d, offset %d,%d\n", __FUNCTION__, + DBG("%s level %d, depth %d, offset %d,%d\n", __func__, level, d, x, y); assert(mt->level[level].slice == NULL); @@ -962,7 +1066,7 @@ intel_miptree_set_image_offset(struct intel_mipmap_tree *mt, mt->level[level].slice[img].y_offset = mt->level[level].level_y + y; DBG("%s level %d img %d pos %d,%d\n", - __FUNCTION__, level, img, + __func__, level, img, mt->level[level].slice[img].x_offset, mt->level[level].slice[img].y_offset); } @@ -996,7 +1100,7 @@ intel_miptree_get_tile_masks(const struct intel_mipmap_tree *mt, switch (tiling) { default: - assert(false); + unreachable("not reached"); case I915_TILING_NONE: *mask_x = *mask_y = 0; break; @@ -1040,7 +1144,7 @@ intel_miptree_get_aligned_offset(const struct intel_mipmap_tree *mt, switch (tiling) { default: - assert(false); + unreachable("not reached"); case I915_TILING_NONE: return y * pitch + x * cpp; case I915_TILING_X: @@ -1092,7 +1196,7 @@ intel_miptree_copy_slice_sw(struct brw_context *brw, int height) { void *src, *dst; - int src_stride, dst_stride; + ptrdiff_t src_stride, dst_stride; int cpp = dst_mt->cpp; intel_miptree_map(brw, src_mt, @@ -1110,7 +1214,7 @@ intel_miptree_copy_slice_sw(struct brw_context *brw, BRW_MAP_DIRECT_BIT, &dst, &dst_stride); - DBG("sw blit %s mt %p %p/%d -> %s mt %p %p/%d (%dx%d)\n", + DBG("sw blit %s mt %p %p/%"PRIdPTR" -> %s mt %p %p/%"PRIdPTR" (%dx%d)\n", _mesa_get_format_name(src_mt->format), src_mt, src, src_stride, _mesa_get_format_name(dst_mt->format), @@ -1225,7 +1329,12 @@ intel_miptree_copy_teximage(struct brw_context *brw, intel_texture_object(intelImage->base.Base.TexObject); int level = intelImage->base.Base.Level; int face = intelImage->base.Base.Face; - GLuint depth = intelImage->base.Base.Depth; + + GLuint depth; + if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) + depth = intelImage->base.Base.Height; + else + depth = intelImage->base.Base.Depth; if (!invalidate) { for (int slice = 0; slice < depth; slice++) { @@ -1237,13 +1346,14 @@ intel_miptree_copy_teximage(struct brw_context *brw, intel_obj->needs_validate = true; } -bool +static bool intel_miptree_alloc_mcs(struct brw_context *brw, struct intel_mipmap_tree *mt, GLuint num_samples) { assert(brw->gen >= 7); /* MCS only used on Gen7+ */ assert(mt->mcs_mt == NULL); + assert(!mt->disable_aux_buffers); /* Choose the correct format for the MCS buffer. All that really matters * is that we allocate the right buffer size, since we'll always be @@ -1252,6 +1362,7 @@ intel_miptree_alloc_mcs(struct brw_context *brw, */ mesa_format format; switch (num_samples) { + case 2: case 4: /* 8 bits/pixel are required for MCS data when using 4x MSAA (2 bits for * each sample). @@ -1265,8 +1376,7 @@ intel_miptree_alloc_mcs(struct brw_context *brw, format = MESA_FORMAT_R_UINT32; break; default: - assert(!"Unrecognized sample count in intel_miptree_alloc_mcs"); - return false; + unreachable("Unrecognized sample count in intel_miptree_alloc_mcs"); }; /* From the Ivy Bridge PRM, Vol4 Part1 p76, "MCS Base Address": @@ -1283,7 +1393,8 @@ intel_miptree_alloc_mcs(struct brw_context *brw, mt->logical_depth0, true, 0 /* num_samples */, - INTEL_MIPTREE_TILING_Y); + INTEL_MIPTREE_TILING_Y, + false); /* From the Ivy Bridge PRM, Vol 2 Part 1 p326: * @@ -1309,6 +1420,7 @@ intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, struct intel_mipmap_tree *mt) { assert(mt->mcs_mt == NULL); + assert(!mt->disable_aux_buffers); /* The format of the MCS buffer is opaque to the driver; all that matters * is that we get its size and pitch right. We'll pretend that the format @@ -1340,7 +1452,8 @@ intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, mt->logical_depth0, true, 0 /* num_samples */, - INTEL_MIPTREE_TILING_Y); + INTEL_MIPTREE_TILING_Y, + false); return mt->mcs_mt; } @@ -1356,7 +1469,7 @@ intel_miptree_level_enable_hiz(struct brw_context *brw, struct intel_mipmap_tree *mt, uint32_t level) { - assert(mt->hiz_mt); + assert(mt->hiz_buf); if (brw->gen >= 8 || brw->is_haswell) { uint32_t width = minify(mt->physical_width0, level); @@ -1369,51 +1482,304 @@ intel_miptree_level_enable_hiz(struct brw_context *brw, * force the proper size alignments. */ if (level > 0 && ((width & 7) || (height & 3))) { + DBG("mt %p level %d: HiZ DISABLED\n", mt, level); return false; } } + DBG("mt %p level %d: HiZ enabled\n", mt, level); mt->level[level].has_hiz = true; return true; } +/** + * Helper for intel_miptree_alloc_hiz() that determines the required hiz + * buffer dimensions and allocates a bo for the hiz buffer. + */ +static struct intel_miptree_aux_buffer * +intel_gen7_hiz_buf_create(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + unsigned z_width = mt->logical_width0; + unsigned z_height = mt->logical_height0; + const unsigned z_depth = MAX2(mt->logical_depth0, 1); + unsigned hz_width, hz_height; + struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); + + if (!buf) + return NULL; + + /* Gen7 PRM Volume 2, Part 1, 11.5.3 "Hierarchical Depth Buffer" documents + * adjustments required for Z_Height and Z_Width based on multisampling. + */ + switch (mt->num_samples) { + case 0: + case 1: + break; + case 2: + case 4: + z_width *= 2; + z_height *= 2; + break; + case 8: + z_width *= 4; + z_height *= 2; + break; + default: + unreachable("unsupported sample count"); + } + + const unsigned vertical_align = 8; /* 'j' in the docs */ + const unsigned H0 = z_height; + const unsigned h0 = ALIGN(H0, vertical_align); + const unsigned h1 = ALIGN(minify(H0, 1), vertical_align); + const unsigned Z0 = z_depth; + + /* HZ_Width (bytes) = ceiling(Z_Width / 16) * 16 */ + hz_width = ALIGN(z_width, 16); + + if (mt->target == GL_TEXTURE_3D) { + unsigned H_i = H0; + unsigned Z_i = Z0; + hz_height = 0; + for (int level = mt->first_level; level <= mt->last_level; ++level) { + unsigned h_i = ALIGN(H_i, vertical_align); + /* sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ + hz_height += h_i * Z_i; + H_i = minify(H_i, 1); + Z_i = minify(Z_i, 1); + } + /* HZ_Height = + * (1/2) * sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) + */ + hz_height = DIV_ROUND_UP(hz_height, 2); + } else { + const unsigned hz_qpitch = h0 + h1 + (12 * vertical_align); + if (mt->target == GL_TEXTURE_CUBE_MAP_ARRAY || + mt->target == GL_TEXTURE_CUBE_MAP) { + /* HZ_Height (rows) = Ceiling ( ( Q_pitch * Z_depth * 6/2) /8 ) * 8 */ + hz_height = DIV_ROUND_UP(hz_qpitch * Z0 * 6, 2 * 8) * 8; + } else { + /* HZ_Height (rows) = Ceiling ( ( Q_pitch * Z_depth/2) /8 ) * 8 */ + hz_height = DIV_ROUND_UP(hz_qpitch * Z0, 2 * 8) * 8; + } + } + + unsigned long pitch; + uint32_t tiling = I915_TILING_Y; + buf->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "hiz", + hz_width, hz_height, 1, + &tiling, &pitch, + BO_ALLOC_FOR_RENDER); + if (!buf->bo) { + free(buf); + return NULL; + } else if (tiling != I915_TILING_Y) { + drm_intel_bo_unreference(buf->bo); + free(buf); + return NULL; + } + + buf->pitch = pitch; + + return buf; +} + + +/** + * Helper for intel_miptree_alloc_hiz() that determines the required hiz + * buffer dimensions and allocates a bo for the hiz buffer. + */ +static struct intel_miptree_aux_buffer * +intel_gen8_hiz_buf_create(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + unsigned z_width = mt->logical_width0; + unsigned z_height = mt->logical_height0; + const unsigned z_depth = MAX2(mt->logical_depth0, 1); + unsigned hz_width, hz_height; + struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); + + if (!buf) + return NULL; + + /* Gen7 PRM Volume 2, Part 1, 11.5.3 "Hierarchical Depth Buffer" documents + * adjustments required for Z_Height and Z_Width based on multisampling. + */ + switch (mt->num_samples) { + case 0: + case 1: + break; + case 2: + case 4: + z_width *= 2; + z_height *= 2; + break; + case 8: + z_width *= 4; + z_height *= 2; + break; + default: + unreachable("unsupported sample count"); + } + + const unsigned vertical_align = 8; /* 'j' in the docs */ + const unsigned H0 = z_height; + const unsigned h0 = ALIGN(H0, vertical_align); + const unsigned h1 = ALIGN(minify(H0, 1), vertical_align); + const unsigned Z0 = z_depth; + + /* HZ_Width (bytes) = ceiling(Z_Width / 16) * 16 */ + hz_width = ALIGN(z_width, 16); + + unsigned H_i = H0; + unsigned Z_i = Z0; + unsigned sum_h_i = 0; + unsigned hz_height_3d_sum = 0; + for (int level = mt->first_level; level <= mt->last_level; ++level) { + unsigned i = level - mt->first_level; + unsigned h_i = ALIGN(H_i, vertical_align); + /* sum(i=2 to m; h_i) */ + if (i >= 2) { + sum_h_i += h_i; + } + /* sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ + hz_height_3d_sum += h_i * Z_i; + H_i = minify(H_i, 1); + Z_i = minify(Z_i, 1); + } + /* HZ_QPitch = h0 + max(h1, sum(i=2 to m; h_i)) */ + buf->qpitch = h0 + MAX2(h1, sum_h_i); + + if (mt->target == GL_TEXTURE_3D) { + /* (1/2) * sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ + hz_height = DIV_ROUND_UP(hz_height_3d_sum, 2); + } else { + /* HZ_Height (rows) = ceiling( (HZ_QPitch/2)/8) *8 * Z_Depth */ + hz_height = DIV_ROUND_UP(buf->qpitch, 2 * 8) * 8 * Z0; + if (mt->target == GL_TEXTURE_CUBE_MAP_ARRAY || + mt->target == GL_TEXTURE_CUBE_MAP) { + /* HZ_Height (rows) = ceiling( (HZ_QPitch/2)/8) *8 * 6 * Z_Depth + * + * We can can just take our hz_height calculation from above, and + * multiply by 6 for the cube map and cube map array types. + */ + hz_height *= 6; + } + } + + unsigned long pitch; + uint32_t tiling = I915_TILING_Y; + buf->bo = drm_intel_bo_alloc_tiled(brw->bufmgr, "hiz", + hz_width, hz_height, 1, + &tiling, &pitch, + BO_ALLOC_FOR_RENDER); + if (!buf->bo) { + free(buf); + return NULL; + } else if (tiling != I915_TILING_Y) { + drm_intel_bo_unreference(buf->bo); + free(buf); + return NULL; + } + + buf->pitch = pitch; + + return buf; +} + + +static struct intel_miptree_aux_buffer * +intel_hiz_miptree_buf_create(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); + const bool force_all_slices_at_each_lod = brw->gen == 6; + + if (!buf) + return NULL; + + buf->mt = intel_miptree_create(brw, + mt->target, + mt->format, + mt->first_level, + mt->last_level, + mt->logical_width0, + mt->logical_height0, + mt->logical_depth0, + true, + mt->num_samples, + INTEL_MIPTREE_TILING_ANY, + force_all_slices_at_each_lod); + if (!buf->mt) { + free(buf); + return NULL; + } + + buf->bo = buf->mt->bo; + buf->pitch = buf->mt->pitch; + buf->qpitch = buf->mt->qpitch; + + return buf; +} + +bool +intel_miptree_wants_hiz_buffer(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + if (!brw->has_hiz) + return false; + + if (mt->hiz_buf != NULL) + return false; + + if (mt->disable_aux_buffers) + return false; + + switch (mt->format) { + case MESA_FORMAT_Z_FLOAT32: + case MESA_FORMAT_Z32_FLOAT_S8X24_UINT: + case MESA_FORMAT_Z24_UNORM_X8_UINT: + case MESA_FORMAT_Z24_UNORM_S8_UINT: + case MESA_FORMAT_Z_UNORM16: + return true; + default: + return false; + } +} bool intel_miptree_alloc_hiz(struct brw_context *brw, struct intel_mipmap_tree *mt) { - assert(mt->hiz_mt == NULL); - mt->hiz_mt = intel_miptree_create(brw, - mt->target, - mt->format, - mt->first_level, - mt->last_level, - mt->logical_width0, - mt->logical_height0, - mt->logical_depth0, - true, - mt->num_samples, - INTEL_MIPTREE_TILING_ANY); + assert(mt->hiz_buf == NULL); + assert(!mt->disable_aux_buffers); - if (!mt->hiz_mt) + if (brw->gen == 7) { + mt->hiz_buf = intel_gen7_hiz_buf_create(brw, mt); + } else if (brw->gen >= 8) { + mt->hiz_buf = intel_gen8_hiz_buf_create(brw, mt); + } else { + mt->hiz_buf = intel_hiz_miptree_buf_create(brw, mt); + } + + if (!mt->hiz_buf) return false; /* Mark that all slices need a HiZ resolve. */ - struct intel_resolve_map *head = &mt->hiz_map; for (int level = mt->first_level; level <= mt->last_level; ++level) { if (!intel_miptree_level_enable_hiz(brw, mt, level)) continue; for (int layer = 0; layer < mt->level[level].depth; ++layer) { - head->next = malloc(sizeof(*head->next)); - head->next->prev = head; - head->next->next = NULL; - head = head->next; - - head->level = level; - head->layer = layer; - head->need = GEN6_HIZ_OP_HIZ_RESOLVE; + struct intel_resolve_map *m = malloc(sizeof(struct intel_resolve_map)); + exec_node_init(&m->link); + m->level = level; + m->layer = layer; + m->need = GEN6_HIZ_OP_HIZ_RESOLVE; + + exec_list_push_tail(&mt->hiz_map, &m->link); } } @@ -1513,15 +1879,13 @@ intel_miptree_all_slices_resolve(struct brw_context *brw, enum gen6_hiz_op need) { bool did_resolve = false; - struct intel_resolve_map *i, *next; - for (i = mt->hiz_map.next; i; i = next) { - next = i->next; - if (i->need != need) + foreach_list_typed_safe(struct intel_resolve_map, map, link, &mt->hiz_map) { + if (map->need != need) continue; - intel_hiz_exec(brw, mt, i->level, i->layer, need); - intel_resolve_map_remove(i); + intel_hiz_exec(brw, mt, map->level, map->layer, need); + intel_resolve_map_remove(map); did_resolve = true; } @@ -1558,7 +1922,7 @@ intel_miptree_resolve_color(struct brw_context *brw, case INTEL_FAST_CLEAR_STATE_CLEAR: /* Fast color clear resolves only make sense for non-MSAA buffers. */ if (mt->msaa_layout == INTEL_MSAA_LAYOUT_NONE) - brw_blorp_resolve_color(brw, mt); + brw_meta_resolve_color(brw, mt); break; } } @@ -1655,8 +2019,8 @@ intel_miptree_updownsample(struct brw_context *brw, { if (brw->gen < 8) { brw_blorp_blit_miptrees(brw, - src, 0 /* level */, 0 /* layer */, - dst, 0 /* level */, 0 /* layer */, + src, 0 /* level */, 0 /* layer */, src->format, + dst, 0 /* level */, 0 /* layer */, dst->format, 0, 0, src->logical_width0, src->logical_height0, 0, 0, @@ -1676,7 +2040,9 @@ intel_miptree_updownsample(struct brw_context *brw, brw_blorp_blit_miptrees(brw, src->stencil_mt, 0 /* level */, 0 /* layer */, + src->stencil_mt->format, dst->stencil_mt, 0 /* level */, 0 /* layer */, + dst->stencil_mt->format, 0, 0, src->logical_width0, src->logical_height0, 0, 0, @@ -1722,8 +2088,8 @@ intel_miptree_map_gtt(struct brw_context *brw, unsigned int bw, bh; void *base; unsigned int image_x, image_y; - int x = map->x; - int y = map->y; + intptr_t x = map->x; + intptr_t y = map->y; /* For compressed formats, the stride is the number of bytes per * row of blocks. intel_miptree_get_image_offset() already does @@ -1749,7 +2115,8 @@ intel_miptree_map_gtt(struct brw_context *brw, map->ptr = base + y * map->stride + x * mt->cpp; } - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p (%s) " + "%"PRIiPTR",%"PRIiPTR" = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), x, y, map->ptr, map->stride); @@ -1775,26 +2142,34 @@ intel_miptree_map_blit(struct brw_context *brw, 0, 0, map->w, map->h, 1, false, 0, - INTEL_MIPTREE_TILING_NONE); + INTEL_MIPTREE_TILING_NONE, + false); if (!map->mt) { fprintf(stderr, "Failed to allocate blit temporary\n"); goto fail; } map->stride = map->mt->pitch; - if (!intel_miptree_blit(brw, - mt, level, slice, - map->x, map->y, false, - map->mt, 0, 0, - 0, 0, false, - map->w, map->h, GL_COPY)) { - fprintf(stderr, "Failed to blit\n"); - goto fail; + /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no + * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless + * invalidate is set, since we'll be writing the whole rectangle from our + * temporary buffer back out. + */ + if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { + if (!intel_miptree_blit(brw, + mt, level, slice, + map->x, map->y, false, + map->mt, 0, 0, + 0, 0, false, + map->w, map->h, GL_COPY)) { + fprintf(stderr, "Failed to blit\n"); + goto fail; + } } map->ptr = intel_miptree_map_raw(brw, map->mt); - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), level, slice, map->ptr, map->stride); @@ -1834,6 +2209,7 @@ intel_miptree_unmap_blit(struct brw_context *brw, /** * "Map" a buffer by copying it to an untiled temporary using MOVNTDQA. */ +#if defined(USE_SSE41) static void intel_miptree_map_movntdqa(struct brw_context *brw, struct intel_mipmap_tree *mt, @@ -1843,7 +2219,7 @@ intel_miptree_map_movntdqa(struct brw_context *brw, assert(map->mode & GL_MAP_READ_BIT); assert(!(map->mode & GL_MAP_WRITE_BIT)); - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, _mesa_get_format_name(mt->format), level, slice, map->ptr, map->stride); @@ -1901,6 +2277,7 @@ intel_miptree_unmap_movntdqa(struct brw_context *brw, map->buffer = NULL; map->ptr = NULL; } +#endif static void intel_miptree_map_s8(struct brw_context *brw, @@ -1937,11 +2314,11 @@ intel_miptree_map_s8(struct brw_context *brw, intel_miptree_unmap_raw(brw, mt); - DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, map->x + image_x, map->y + image_y, map->ptr, map->stride); } else { - DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, map->ptr, map->stride); } @@ -2099,13 +2476,13 @@ intel_miptree_map_depthstencil(struct brw_context *brw, intel_miptree_unmap_raw(brw, z_mt); DBG("%s: %d,%d %dx%d from z mt %p %d,%d, s mt %p %d,%d = %p/%d\n", - __FUNCTION__, + __func__, map->x, map->y, map->w, map->h, z_mt, map->x + z_image_x, map->y + z_image_y, s_mt, map->x + s_image_x, map->y + s_image_y, map->ptr, map->stride); } else { - DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __FUNCTION__, + DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__, map->x, map->y, map->w, map->h, mt, map->ptr, map->stride); } @@ -2140,9 +2517,9 @@ intel_miptree_unmap_depthstencil(struct brw_context *brw, x + s_image_x + map->x, y + s_image_y + map->y, brw->has_swizzling); - ptrdiff_t z_offset = ((y + z_image_y) * + ptrdiff_t z_offset = ((y + z_image_y + map->y) * (z_mt->pitch / 4) + - (x + z_image_x)); + (x + z_image_x + map->x)); if (map_z32f_x24s8) { z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0]; @@ -2159,7 +2536,7 @@ intel_miptree_unmap_depthstencil(struct brw_context *brw, intel_miptree_unmap_raw(brw, z_mt); DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n", - __FUNCTION__, + __func__, map->x, map->y, map->w, map->h, z_mt, _mesa_get_format_name(z_mt->format), map->x + z_image_x, map->y + z_image_y, @@ -2226,12 +2603,52 @@ can_blit_slice(struct intel_mipmap_tree *mt, if (image_x >= 32768 || image_y >= 32768) return false; + /* See intel_miptree_blit() for details on the 32k pitch limit. */ if (mt->pitch >= 32768) return false; return true; } +static bool +use_intel_mipree_map_blit(struct brw_context *brw, + struct intel_mipmap_tree *mt, + GLbitfield mode, + unsigned int level, + unsigned int slice) +{ + if (brw->has_llc && + /* It's probably not worth swapping to the blit ring because of + * all the overhead involved. + */ + !(mode & GL_MAP_WRITE_BIT) && + !mt->compressed && + (mt->tiling == I915_TILING_X || + /* Prior to Sandybridge, the blitter can't handle Y tiling */ + (brw->gen >= 6 && mt->tiling == I915_TILING_Y)) && + can_blit_slice(mt, level, slice)) + return true; + + if (mt->tiling != I915_TILING_NONE && + mt->bo->size >= brw->max_gtt_map_object_size) { + assert(can_blit_slice(mt, level, slice)); + return true; + } + + return false; +} + +/** + * Parameter \a out_stride has type ptrdiff_t not because the buffer stride may + * exceed 32 bits but to diminish the likelihood subtle bugs in pointer + * arithmetic overflow. + * + * If you call this function and use \a out_stride, then you're doing pointer + * arithmetic on \a out_ptr. The type of \a out_stride doesn't prevent all + * bugs. The caller must still take care to avoid 32-bit overflow errors in + * all arithmetic expressions that contain buffer offsets and pixel sizes, + * which usually have type uint32_t or GLuint. + */ void intel_miptree_map(struct brw_context *brw, struct intel_mipmap_tree *mt, @@ -2243,7 +2660,7 @@ intel_miptree_map(struct brw_context *brw, unsigned int h, GLbitfield mode, void **out_ptr, - int *out_stride) + ptrdiff_t *out_stride) { struct intel_miptree_map *map; @@ -2268,21 +2685,12 @@ intel_miptree_map(struct brw_context *brw, intel_miptree_map_etc(brw, mt, map, level, slice); } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) { intel_miptree_map_depthstencil(brw, mt, map, level, slice); - } - /* See intel_miptree_blit() for details on the 32k pitch limit. */ - else if (brw->has_llc && - !(mode & GL_MAP_WRITE_BIT) && - !mt->compressed && - (mt->tiling == I915_TILING_X || - (brw->gen >= 6 && mt->tiling == I915_TILING_Y)) && - can_blit_slice(mt, level, slice)) { - intel_miptree_map_blit(brw, mt, map, level, slice); - } else if (mt->tiling != I915_TILING_NONE && - mt->bo->size >= brw->max_gtt_map_object_size) { - assert(can_blit_slice(mt, level, slice)); + } else if (use_intel_mipree_map_blit(brw, mt, mode, level, slice)) { intel_miptree_map_blit(brw, mt, map, level, slice); +#if defined(USE_SSE41) } else if (!(mode & GL_MAP_WRITE_BIT) && !mt->compressed && cpu_has_sse4_1) { intel_miptree_map_movntdqa(brw, mt, map, level, slice); +#endif } else { intel_miptree_map_gtt(brw, mt, map, level, slice); } @@ -2307,7 +2715,7 @@ intel_miptree_unmap(struct brw_context *brw, if (!map) return; - DBG("%s: mt %p (%s) level %d slice %d\n", __FUNCTION__, + DBG("%s: mt %p (%s) level %d slice %d\n", __func__, mt, _mesa_get_format_name(mt->format), level, slice); if (mt->format == MESA_FORMAT_S_UINT8) { @@ -2319,8 +2727,10 @@ intel_miptree_unmap(struct brw_context *brw, intel_miptree_unmap_depthstencil(brw, mt, map, level, slice); } else if (map->mt) { intel_miptree_unmap_blit(brw, mt, map, level, slice); +#if defined(USE_SSE41) } else if (map->buffer && cpu_has_sse4_1) { intel_miptree_unmap_movntdqa(brw, mt, map, level, slice); +#endif } else { intel_miptree_unmap_gtt(brw, mt, map, level, slice); }