X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fintel_mipmap_tree.c;h=e7ebc29b59d690235b64769589c53a4a8d03a9bd;hb=b02d136b5e14f94e5f3ab8489f0fbfaa88e76820;hp=800b1626964cbd082c3a572a61f3f28ad8dd8597;hpb=421d713eec705cb0af551f719fc1d505933dae50;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c index 800b1626964..e7ebc29b59d 100644 --- a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c +++ b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c @@ -27,8 +27,8 @@ #include #include "intel_batchbuffer.h" +#include "intel_image.h" #include "intel_mipmap_tree.h" -#include "intel_resolve_map.h" #include "intel_tex.h" #include "intel_blit.h" #include "intel_fbo.h" @@ -65,7 +65,7 @@ intel_miptree_alloc_mcs(struct brw_context *brw, */ static enum intel_msaa_layout compute_msaa_layout(struct brw_context *brw, mesa_format format, - enum intel_aux_disable aux_disable) + uint32_t layout_flags) { /* Prior to Gen7, all MSAA surfaces used IMS layout. */ if (brw->gen < 7) @@ -91,7 +91,7 @@ compute_msaa_layout(struct brw_context *brw, mesa_format format, */ if (brw->gen == 7 && _mesa_get_format_datatype(format) == GL_INT) { return INTEL_MSAA_LAYOUT_UMS; - } else if (aux_disable & INTEL_AUX_DISABLE_MCS) { + } else if (layout_flags & MIPTREE_LAYOUT_DISABLE_AUX) { /* We can't use the CMS layout because it uses an aux buffer, the MCS * buffer. So fallback to UMS, which is identical to CMS without the * MCS. */ @@ -102,9 +102,8 @@ compute_msaa_layout(struct brw_context *brw, mesa_format format, } } -bool -intel_tiling_supports_non_msrt_mcs(const struct brw_context *brw, - unsigned tiling) +static bool +intel_tiling_supports_ccs(const struct brw_context *brw, unsigned tiling) { /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render * Target(s)", beneath the "Fast Color Clear" bullet (p326): @@ -142,17 +141,14 @@ intel_tiling_supports_non_msrt_mcs(const struct brw_context *brw, * - MCS and Lossless compression is supported for TiledY/TileYs/TileYf * non-MSRTs only. */ -bool -intel_miptree_supports_non_msrt_fast_clear(struct brw_context *brw, - const struct intel_mipmap_tree *mt) +static bool +intel_miptree_supports_ccs(struct brw_context *brw, + const struct intel_mipmap_tree *mt) { /* MCS support does not exist prior to Gen7 */ if (brw->gen < 7) return false; - if (mt->aux_disable & INTEL_AUX_DISABLE_MCS) - return false; - /* This function applies only to non-multisampled render targets. */ if (mt->num_samples > 1) return false; @@ -205,7 +201,7 @@ intel_miptree_supports_non_msrt_fast_clear(struct brw_context *brw, /* There's no point in using an MCS buffer if the surface isn't in a * renderable format. */ - if (!brw->format_supported_as_render_target[mt->format]) + if (!brw->mesa_format_supports_render[mt->format]) return false; if (brw->gen >= 9) { @@ -217,6 +213,26 @@ intel_miptree_supports_non_msrt_fast_clear(struct brw_context *brw, return true; } +static bool +intel_miptree_supports_hiz(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + if (!brw->has_hiz) + return false; + + switch (mt->format) { + case MESA_FORMAT_Z_FLOAT32: + case MESA_FORMAT_Z32_FLOAT_S8X24_UINT: + case MESA_FORMAT_Z24_UNORM_X8_UINT: + case MESA_FORMAT_Z24_UNORM_S8_UINT: + case MESA_FORMAT_Z_UNORM16: + return true; + default: + return false; + } +} + + /* On Gen9 support for color buffer compression was extended to single * sampled surfaces. This is a helper considering both auxiliary buffer * type and number of samples telling if the given miptree represents @@ -244,9 +260,9 @@ intel_miptree_is_lossless_compressed(const struct brw_context *brw, return mt->num_samples <= 1; } -bool -intel_miptree_supports_lossless_compressed(struct brw_context *brw, - const struct intel_mipmap_tree *mt) +static bool +intel_miptree_supports_ccs_e(struct brw_context *brw, + const struct intel_mipmap_tree *mt) { /* For now compression is only enabled for integer formats even though * there exist supported floating point formats also. This is a heuristic @@ -258,8 +274,7 @@ intel_miptree_supports_lossless_compressed(struct brw_context *brw, if (_mesa_get_format_datatype(mt->format) == GL_FLOAT) return false; - /* Fast clear mechanism and lossless compression go hand in hand. */ - if (!intel_miptree_supports_non_msrt_fast_clear(brw, mt)) + if (!intel_miptree_supports_ccs(brw, mt)) return false; /* Fast clear can be also used to clear srgb surfaces by using equivalent @@ -286,6 +301,27 @@ intel_depth_format_for_depthstencil_format(mesa_format format) { } } +static bool +create_mapping_table(GLenum target, unsigned first_level, unsigned last_level, + unsigned depth0, struct intel_mipmap_level *table) +{ + for (unsigned level = first_level; level <= last_level; level++) { + const unsigned d = + target == GL_TEXTURE_3D ? minify(depth0, level) : depth0; + + table[level].slice = calloc(d, sizeof(*table[0].slice)); + if (!table[level].slice) + goto unwind; + } + + return true; + +unwind: + for (unsigned level = first_level; level <= last_level; level++) + free(table[level].slice); + + return false; +} /** * @param for_bo Indicates that the caller is @@ -323,22 +359,23 @@ intel_miptree_create_layout(struct brw_context *brw, mt->logical_width0 = width0; mt->logical_height0 = height0; mt->logical_depth0 = depth0; - mt->aux_disable = (layout_flags & MIPTREE_LAYOUT_DISABLE_AUX) != 0 ? - INTEL_AUX_DISABLE_ALL : INTEL_AUX_DISABLE_NONE; - mt->aux_disable |= INTEL_AUX_DISABLE_CCS; mt->is_scanout = (layout_flags & MIPTREE_LAYOUT_FOR_SCANOUT) != 0; - exec_list_make_empty(&mt->hiz_map); - exec_list_make_empty(&mt->color_resolve_map); + mt->aux_usage = ISL_AUX_USAGE_NONE; + mt->supports_fast_clear = false; + mt->aux_state = NULL; mt->cpp = _mesa_get_format_bytes(format); mt->num_samples = num_samples; mt->compressed = _mesa_is_format_compressed(format); mt->msaa_layout = INTEL_MSAA_LAYOUT_NONE; mt->refcount = 1; + if (brw->gen == 6 && format == MESA_FORMAT_S_UINT8) + layout_flags |= MIPTREE_LAYOUT_GEN6_HIZ_STENCIL; + int depth_multiply = 1; if (num_samples > 1) { /* Adjust width/height/depth for MSAA */ - mt->msaa_layout = compute_msaa_layout(brw, format, mt->aux_disable); + mt->msaa_layout = compute_msaa_layout(brw, format, layout_flags); if (mt->msaa_layout == INTEL_MSAA_LAYOUT_IMS) { /* From the Ivybridge PRM, Volume 1, Part 1, page 108: * "If the surface is multisampled and it is a depth or stencil @@ -426,6 +463,12 @@ intel_miptree_create_layout(struct brw_context *brw, } } + if (!create_mapping_table(target, first_level, last_level, depth0, + mt->level)) { + free(mt); + return NULL; + } + /* Set array_layout to ALL_SLICES_AT_EACH_LOD when array_spacing_lod0 can * be used. array_spacing_lod0 is only used for non-IMS MSAA surfaces on * Gen 7 and 8. On Gen 8 and 9 this layout is not available but it is still @@ -461,12 +504,10 @@ intel_miptree_create_layout(struct brw_context *brw, if (!(layout_flags & MIPTREE_LAYOUT_FOR_BO) && _mesa_get_format_base_format(format) == GL_DEPTH_STENCIL && (brw->must_use_separate_stencil || - (brw->has_separate_stencil && - intel_miptree_wants_hiz_buffer(brw, mt)))) { + (brw->has_separate_stencil && intel_miptree_supports_hiz(brw, mt)))) { uint32_t stencil_flags = MIPTREE_LAYOUT_ACCELERATED_UPLOAD; if (brw->gen == 6) { - stencil_flags |= MIPTREE_LAYOUT_GEN6_HIZ_STENCIL | - MIPTREE_LAYOUT_TILING_ANY; + stencil_flags |= MIPTREE_LAYOUT_TILING_ANY; } mt->stencil_mt = intel_miptree_create(brw, @@ -512,7 +553,7 @@ intel_miptree_create_layout(struct brw_context *brw, * 7 | ? | ? * 6 | ? | ? */ - if (intel_miptree_supports_non_msrt_fast_clear(brw, mt)) { + if (intel_miptree_supports_ccs(brw, mt)) { if (brw->gen >= 9 || (brw->gen == 8 && num_samples <= 1)) layout_flags |= MIPTREE_LAYOUT_FORCE_HALIGN16; } else if (brw->gen >= 9 && num_samples > 1) { @@ -532,13 +573,43 @@ intel_miptree_create_layout(struct brw_context *brw, return NULL; } - if (mt->aux_disable & INTEL_AUX_DISABLE_MCS) - assert(mt->msaa_layout != INTEL_MSAA_LAYOUT_CMS); - return mt; } +/** + * Choose the aux usage for this miptree. This function must be called fairly + * late in the miptree create process after we have a tiling. + */ +static void +intel_miptree_choose_aux_usage(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + assert(mt->aux_usage == ISL_AUX_USAGE_NONE); + + if (mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS) { + mt->aux_usage = ISL_AUX_USAGE_MCS; + } else if (intel_tiling_supports_ccs(brw, mt->tiling) && + intel_miptree_supports_ccs(brw, mt)) { + if (!unlikely(INTEL_DEBUG & DEBUG_NO_RBC) && + brw->gen >= 9 && !mt->is_scanout && + intel_miptree_supports_ccs_e(brw, mt)) { + mt->aux_usage = ISL_AUX_USAGE_CCS_E; + } else { + mt->aux_usage = ISL_AUX_USAGE_CCS_D; + } + } else if (intel_miptree_supports_hiz(brw, mt)) { + mt->aux_usage = ISL_AUX_USAGE_HIZ; + } + + /* We can do fast-clear on all auxiliary surface types that are + * allocated through the normal texture creation paths. + */ + if (mt->aux_usage != ISL_AUX_USAGE_NONE) + mt->supports_fast_clear = true; +} + + /** * Choose an appropriate uncompressed format for a requested * compressed format, if unsupported. @@ -578,6 +649,139 @@ intel_lower_compressed_format(struct brw_context *brw, mesa_format format) } } +/** \brief Assert that the level and layer are valid for the miptree. */ +void +intel_miptree_check_level_layer(const struct intel_mipmap_tree *mt, + uint32_t level, + uint32_t layer) +{ + (void) mt; + (void) level; + (void) layer; + + assert(level >= mt->first_level); + assert(level <= mt->last_level); + + if (mt->surf.size > 0) + assert(layer < (mt->surf.dim == ISL_SURF_DIM_3D ? + minify(mt->surf.phys_level0_sa.depth, level) : + mt->surf.phys_level0_sa.array_len)); + else + assert(layer < mt->level[level].depth); +} + +static enum isl_aux_state ** +create_aux_state_map(struct intel_mipmap_tree *mt, + enum isl_aux_state initial) +{ + const uint32_t levels = mt->last_level + 1; + + uint32_t total_slices = 0; + for (uint32_t level = 0; level < levels; level++) + total_slices += mt->level[level].depth; + + const size_t per_level_array_size = levels * sizeof(enum isl_aux_state *); + + /* We're going to allocate a single chunk of data for both the per-level + * reference array and the arrays of aux_state. This makes cleanup + * significantly easier. + */ + const size_t total_size = per_level_array_size + + total_slices * sizeof(enum isl_aux_state); + void *data = malloc(total_size); + if (data == NULL) + return NULL; + + enum isl_aux_state **per_level_arr = data; + enum isl_aux_state *s = data + per_level_array_size; + for (uint32_t level = 0; level < levels; level++) { + per_level_arr[level] = s; + for (uint32_t a = 0; a < mt->level[level].depth; a++) + *(s++) = initial; + } + assert((void *)s == data + total_size); + + return per_level_arr; +} + +static void +free_aux_state_map(enum isl_aux_state **state) +{ + free(state); +} + +static struct intel_mipmap_tree * +make_surface(struct brw_context *brw, GLenum target, mesa_format format, + unsigned first_level, unsigned last_level, + unsigned width0, unsigned height0, unsigned depth0, + unsigned num_samples, enum isl_tiling isl_tiling, + isl_surf_usage_flags_t isl_usage_flags, uint32_t alloc_flags, + struct brw_bo *bo) +{ + struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1); + if (!mt) + return NULL; + + if (!create_mapping_table(target, first_level, last_level, depth0, + mt->level)) { + free(mt); + return NULL; + } + + if (target == GL_TEXTURE_CUBE_MAP || + target == GL_TEXTURE_CUBE_MAP_ARRAY) + isl_usage_flags |= ISL_SURF_USAGE_CUBE_BIT; + + DBG("%s: %s %s %ux %u:%u:%u %d..%d <-- %p\n", + __func__, + _mesa_enum_to_string(target), + _mesa_get_format_name(format), + num_samples, width0, height0, depth0, + first_level, last_level, mt); + + struct isl_surf_init_info init_info = { + .dim = get_isl_surf_dim(target), + .format = translate_tex_format(brw, format, false), + .width = width0, + .height = height0, + .depth = target == GL_TEXTURE_3D ? depth0 : 1, + .levels = last_level - first_level + 1, + .array_len = target == GL_TEXTURE_3D ? 1 : depth0, + .samples = MAX2(num_samples, 1), + .usage = isl_usage_flags, + .tiling_flags = 1u << isl_tiling + }; + + if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info)) + goto fail; + + assert(mt->surf.size % mt->surf.row_pitch == 0); + + if (!bo) { + mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "isl-miptree", + mt->surf.size, + isl_tiling_to_bufmgr_tiling(isl_tiling), + mt->surf.row_pitch, alloc_flags); + if (!mt->bo) + goto fail; + } else { + mt->bo = bo; + } + + mt->first_level = first_level; + mt->last_level = last_level; + mt->target = target; + mt->format = format; + mt->refcount = 1; + mt->aux_state = NULL; + + return mt; + +fail: + intel_miptree_release(&mt); + return NULL; +} + static struct intel_mipmap_tree * miptree_create(struct brw_context *brw, GLenum target, @@ -590,6 +794,13 @@ miptree_create(struct brw_context *brw, GLuint num_samples, uint32_t layout_flags) { + if (brw->gen == 6 && format == MESA_FORMAT_S_UINT8) + return make_surface(brw, target, format, first_level, last_level, + width0, height0, depth0, num_samples, ISL_TILING_W, + ISL_SURF_USAGE_STENCIL_BIT | + ISL_SURF_USAGE_TEXTURE_BIT, + BO_ALLOC_FOR_RENDER, NULL); + struct intel_mipmap_tree *mt; mesa_format tex_format = format; mesa_format etc_format = MESA_FORMAT_NONE; @@ -617,21 +828,24 @@ miptree_create(struct brw_context *brw, if (format == MESA_FORMAT_S_UINT8) { /* Align to size of W tile, 64x64. */ - mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "miptree", - ALIGN(mt->total_width, 64), - ALIGN(mt->total_height, 64), - mt->cpp, mt->tiling, &mt->pitch, - alloc_flags); + mt->bo = brw_bo_alloc_tiled_2d(brw->bufmgr, "miptree", + ALIGN(mt->total_width, 64), + ALIGN(mt->total_height, 64), + mt->cpp, mt->tiling, &mt->pitch, + alloc_flags); } else { - mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "miptree", - mt->total_width, mt->total_height, - mt->cpp, mt->tiling, &mt->pitch, - alloc_flags); + mt->bo = brw_bo_alloc_tiled_2d(brw->bufmgr, "miptree", + mt->total_width, mt->total_height, + mt->cpp, mt->tiling, &mt->pitch, + alloc_flags); } if (layout_flags & MIPTREE_LAYOUT_FOR_SCANOUT) mt->bo->cache_coherent = false; + if (!(layout_flags & MIPTREE_LAYOUT_DISABLE_AUX)) + intel_miptree_choose_aux_usage(brw, mt); + return mt; } @@ -667,9 +881,9 @@ intel_miptree_create(struct brw_context *brw, mt->tiling = I915_TILING_X; brw_bo_unreference(mt->bo); - mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "miptree", - mt->total_width, mt->total_height, mt->cpp, - mt->tiling, &mt->pitch, alloc_flags); + mt->bo = brw_bo_alloc_tiled_2d(brw->bufmgr, "miptree", + mt->total_width, mt->total_height, mt->cpp, + mt->tiling, &mt->pitch, alloc_flags); } mt->offset = 0; @@ -688,29 +902,14 @@ intel_miptree_create(struct brw_context *brw, } } - /* If this miptree is capable of supporting fast color clears, set - * fast_clear_state appropriately to ensure that fast clears will occur. - * Allocation of the MCS miptree will be deferred until the first fast - * clear actually occurs or when compressed single sampled buffer is - * written by the GPU for the first time. + /* Since CCS_E can compress more than just clear color, we create the CCS + * for it up-front. For CCS_D which only compresses clears, we create the + * CCS on-demand when a clear occurs that wants one. */ - if (intel_tiling_supports_non_msrt_mcs(brw, mt->tiling) && - intel_miptree_supports_non_msrt_fast_clear(brw, mt)) { - mt->aux_disable &= ~INTEL_AUX_DISABLE_CCS; - assert(brw->gen < 8 || mt->halign == 16 || num_samples <= 1); - - /* On Gen9+ clients are not currently capable of consuming compressed - * single-sampled buffers. Disabling compression allows us to skip - * resolves. - */ - const bool lossless_compression_disabled = INTEL_DEBUG & DEBUG_NO_RBC; - const bool is_lossless_compressed = - unlikely(!lossless_compression_disabled) && - brw->gen >= 9 && !mt->is_scanout && - intel_miptree_supports_lossless_compressed(brw, mt); - - if (is_lossless_compressed) { - intel_miptree_alloc_non_msrt_mcs(brw, mt, is_lossless_compressed); + if (mt->aux_usage == ISL_AUX_USAGE_CCS_E) { + if (!intel_miptree_alloc_ccs(brw, mt)) { + intel_miptree_release(&mt); + return NULL; } } @@ -730,7 +929,22 @@ intel_miptree_create_for_bo(struct brw_context *brw, { struct intel_mipmap_tree *mt; uint32_t tiling, swizzle; - GLenum target; + const GLenum target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D; + + if (brw->gen == 6 && format == MESA_FORMAT_S_UINT8) { + mt = make_surface(brw, target, MESA_FORMAT_S_UINT8, + 0, 0, width, height, depth, 1, ISL_TILING_W, + ISL_SURF_USAGE_STENCIL_BIT | + ISL_SURF_USAGE_TEXTURE_BIT, + BO_ALLOC_FOR_RENDER, bo); + if (!mt) + return NULL; + + assert(bo->size >= mt->surf.size); + + brw_bo_reference(bo); + return mt; + } brw_bo_get_tiling(bo, &tiling, &swizzle); @@ -745,8 +959,6 @@ intel_miptree_create_for_bo(struct brw_context *brw, */ assert(pitch >= 0); - target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D; - /* The BO already has a tiling format and we shouldn't confuse the lower * layers by making it try to find a tiling format again. */ @@ -767,6 +979,155 @@ intel_miptree_create_for_bo(struct brw_context *brw, mt->offset = offset; mt->tiling = tiling; + if (!(layout_flags & MIPTREE_LAYOUT_DISABLE_AUX)) + intel_miptree_choose_aux_usage(brw, mt); + + return mt; +} + +static struct intel_mipmap_tree * +miptree_create_for_planar_image(struct brw_context *brw, + __DRIimage *image, GLenum target) +{ + struct intel_image_format *f = image->planar_format; + struct intel_mipmap_tree *planar_mt; + + for (int i = 0; i < f->nplanes; i++) { + const int index = f->planes[i].buffer_index; + const uint32_t dri_format = f->planes[i].dri_format; + const mesa_format format = driImageFormatToGLFormat(dri_format); + const uint32_t width = image->width >> f->planes[i].width_shift; + const uint32_t height = image->height >> f->planes[i].height_shift; + + /* Disable creation of the texture's aux buffers because the driver + * exposes no EGL API to manage them. That is, there is no API for + * resolving the aux buffer's content to the main buffer nor for + * invalidating the aux buffer's content. + */ + struct intel_mipmap_tree *mt = + intel_miptree_create_for_bo(brw, image->bo, format, + image->offsets[index], + width, height, 1, + image->strides[index], + MIPTREE_LAYOUT_DISABLE_AUX); + if (mt == NULL) + return NULL; + + mt->target = target; + mt->total_width = width; + mt->total_height = height; + + if (i == 0) + planar_mt = mt; + else + planar_mt->plane[i - 1] = mt; + } + + return planar_mt; +} + +struct intel_mipmap_tree * +intel_miptree_create_for_dri_image(struct brw_context *brw, + __DRIimage *image, GLenum target, + enum isl_colorspace colorspace, + bool is_winsys_image) +{ + if (image->planar_format && image->planar_format->nplanes > 0) { + assert(colorspace == ISL_COLORSPACE_NONE || + colorspace == ISL_COLORSPACE_YUV); + return miptree_create_for_planar_image(brw, image, target); + } + + mesa_format format = image->format; + switch (colorspace) { + case ISL_COLORSPACE_NONE: + /* Keep the image format unmodified */ + break; + + case ISL_COLORSPACE_LINEAR: + format =_mesa_get_srgb_format_linear(format); + break; + + case ISL_COLORSPACE_SRGB: + format =_mesa_get_linear_format_srgb(format); + break; + + default: + unreachable("Inalid colorspace for non-planar image"); + } + + if (!brw->ctx.TextureFormatSupported[format]) { + /* The texture storage paths in core Mesa detect if the driver does not + * support the user-requested format, and then searches for a + * fallback format. The DRIimage code bypasses core Mesa, though. So we + * do the fallbacks here for important formats. + * + * We must support DRM_FOURCC_XBGR8888 textures because the Android + * framework produces HAL_PIXEL_FORMAT_RGBX8888 winsys surfaces, which + * the Chrome OS compositor consumes as dma_buf EGLImages. + */ + format = _mesa_format_fallback_rgbx_to_rgba(format); + } + + if (!brw->ctx.TextureFormatSupported[format]) + return NULL; + + /* If this image comes in from a window system, we have different + * requirements than if it comes in via an EGL import operation. Window + * system images can use any form of auxiliary compression we wish because + * they get "flushed" before being handed off to the window system and we + * have the opportunity to do resolves. Window system buffers also may be + * used for scanout so we need to flag that appropriately. + */ + const uint32_t mt_layout_flags = + is_winsys_image ? MIPTREE_LAYOUT_FOR_SCANOUT : MIPTREE_LAYOUT_DISABLE_AUX; + + /* Disable creation of the texture's aux buffers because the driver exposes + * no EGL API to manage them. That is, there is no API for resolving the aux + * buffer's content to the main buffer nor for invalidating the aux buffer's + * content. + */ + struct intel_mipmap_tree *mt = + intel_miptree_create_for_bo(brw, image->bo, format, + image->offset, image->width, image->height, 1, + image->pitch, mt_layout_flags); + if (mt == NULL) + return NULL; + + mt->target = target; + mt->level[0].level_x = image->tile_x; + mt->level[0].level_y = image->tile_y; + mt->level[0].slice[0].x_offset = image->tile_x; + mt->level[0].slice[0].y_offset = image->tile_y; + mt->total_width += image->tile_x; + mt->total_height += image->tile_y; + + /* From "OES_EGL_image" error reporting. We report GL_INVALID_OPERATION + * for EGL images from non-tile aligned sufaces in gen4 hw and earlier which has + * trouble resolving back to destination image due to alignment issues. + */ + if (!brw->has_surface_tile_offset) { + uint32_t draw_x, draw_y; + intel_miptree_get_tile_offsets(mt, 0, 0, &draw_x, &draw_y); + + if (draw_x != 0 || draw_y != 0) { + _mesa_error(&brw->ctx, GL_INVALID_OPERATION, __func__); + intel_miptree_release(&mt); + return NULL; + } + } + + /* Since CCS_E can compress more than just clear color, we create the CCS + * for it up-front. For CCS_D which only compresses clears, we create the + * CCS on-demand when a clear occurs that wants one. + */ + if (mt->aux_usage == ISL_AUX_USAGE_CCS_E) { + if (!intel_miptree_alloc_ccs(brw, mt)) { + intel_miptree_release(&mt); + return NULL; + } + } + return mt; } @@ -780,14 +1141,13 @@ intel_miptree_create_for_bo(struct brw_context *brw, * that will contain the actual rendering (which is lazily resolved to * irb->singlesample_mt). */ -void +bool intel_update_winsys_renderbuffer_miptree(struct brw_context *intel, struct intel_renderbuffer *irb, - struct brw_bo *bo, + struct intel_mipmap_tree *singlesample_mt, uint32_t width, uint32_t height, uint32_t pitch) { - struct intel_mipmap_tree *singlesample_mt = NULL; struct intel_mipmap_tree *multisample_mt = NULL; struct gl_renderbuffer *rb = &irb->Base.Base; mesa_format format = rb->Format; @@ -799,27 +1159,7 @@ intel_update_winsys_renderbuffer_miptree(struct brw_context *intel, assert(_mesa_get_format_base_format(format) == GL_RGB || _mesa_get_format_base_format(format) == GL_RGBA); - singlesample_mt = intel_miptree_create_for_bo(intel, - bo, - format, - 0, - width, - height, - 1, - pitch, - MIPTREE_LAYOUT_FOR_SCANOUT); - if (!singlesample_mt) - goto fail; - - /* If this miptree is capable of supporting fast color clears, set - * mcs_state appropriately to ensure that fast clears will occur. - * Allocation of the MCS miptree will be deferred until the first fast - * clear actually occurs. - */ - if (intel_tiling_supports_non_msrt_mcs(intel, singlesample_mt->tiling) && - intel_miptree_supports_non_msrt_fast_clear(intel, singlesample_mt)) { - singlesample_mt->aux_disable &= ~INTEL_AUX_DISABLE_CCS; - } + assert(singlesample_mt); if (num_samples == 0) { intel_miptree_release(&irb->mt); @@ -846,12 +1186,11 @@ intel_update_winsys_renderbuffer_miptree(struct brw_context *intel, irb->mt = multisample_mt; } } - return; + return true; fail: - intel_miptree_release(&irb->singlesample_mt); intel_miptree_release(&irb->mt); - return; + return false; } struct intel_mipmap_tree* @@ -875,7 +1214,7 @@ intel_miptree_create_for_renderbuffer(struct brw_context *brw, if (!mt) goto fail; - if (intel_miptree_wants_hiz_buffer(brw, mt)) { + if (mt->aux_usage == ISL_AUX_USAGE_HIZ) { ok = intel_miptree_alloc_hiz(brw, mt); if (!ok) goto fail; @@ -906,17 +1245,14 @@ intel_miptree_reference(struct intel_mipmap_tree **dst, } static void -intel_miptree_hiz_buffer_free(struct intel_miptree_hiz_buffer *hiz_buf) +intel_miptree_aux_buffer_free(struct intel_miptree_aux_buffer *aux_buf) { - if (hiz_buf == NULL) + if (aux_buf == NULL) return; - if (hiz_buf->mt) - intel_miptree_release(&hiz_buf->mt); - else - brw_bo_unreference(hiz_buf->aux_base.bo); + brw_bo_unreference(aux_buf->bo); - free(hiz_buf); + free(aux_buf); } void @@ -934,13 +1270,9 @@ intel_miptree_release(struct intel_mipmap_tree **mt) brw_bo_unreference((*mt)->bo); intel_miptree_release(&(*mt)->stencil_mt); intel_miptree_release(&(*mt)->r8stencil_mt); - intel_miptree_hiz_buffer_free((*mt)->hiz_buf); - if ((*mt)->mcs_buf) { - brw_bo_unreference((*mt)->mcs_buf->bo); - free((*mt)->mcs_buf); - } - intel_resolve_map_clear(&(*mt)->hiz_map); - intel_resolve_map_clear(&(*mt)->color_resolve_map); + intel_miptree_aux_buffer_free((*mt)->hiz_buf); + intel_miptree_aux_buffer_free((*mt)->mcs_buf); + free_aux_state_map((*mt)->aux_state); intel_miptree_release(&(*mt)->plane[0]); intel_miptree_release(&(*mt)->plane[1]); @@ -1024,6 +1356,21 @@ intel_miptree_match_image(struct intel_mipmap_tree *mt, if (mt->target == GL_TEXTURE_CUBE_MAP) depth = 6; + if (mt->surf.size > 0) { + if (level >= mt->surf.levels) + return false; + + const unsigned level_depth = + mt->surf.dim == ISL_SURF_DIM_3D ? + minify(mt->surf.logical_level0_px.depth, level) : + mt->surf.logical_level0_px.array_len; + + return width == minify(mt->surf.logical_level0_px.width, level) && + height == minify(mt->surf.logical_level0_px.height, level) && + depth == level_depth && + MAX2(image->NumSamples, 1) == mt->surf.samples; + } + int level_depth = mt->level[level].depth; if (mt->num_samples > 1) { switch (mt->msaa_layout) { @@ -1066,9 +1413,8 @@ intel_miptree_set_level_info(struct intel_mipmap_tree *mt, DBG("%s level %d, depth %d, offset %d,%d\n", __func__, level, d, x, y); - assert(mt->level[level].slice == NULL); + assert(mt->level[level].slice); - mt->level[level].slice = calloc(d, sizeof(*mt->level[0].slice)); mt->level[level].slice[0].x_offset = mt->level[level].level_x; mt->level[level].slice[0].y_offset = mt->level[level].level_y; } @@ -1098,6 +1444,25 @@ intel_miptree_get_image_offset(const struct intel_mipmap_tree *mt, GLuint level, GLuint slice, GLuint *x, GLuint *y) { + if (mt->surf.size > 0) { + uint32_t x_offset_sa, y_offset_sa; + + /* Given level is relative to level zero while the miptree may be + * represent just a subset of all levels starting from 'first_level'. + */ + assert(level >= mt->first_level); + level -= mt->first_level; + + const unsigned z = mt->surf.dim == ISL_SURF_DIM_3D ? slice : 0; + slice = mt->surf.dim == ISL_SURF_DIM_3D ? 0 : slice; + isl_surf_get_image_offset_sa(&mt->surf, level, slice, z, + &x_offset_sa, &y_offset_sa); + + *x = x_offset_sa; + *y = y_offset_sa; + return; + } + assert(slice < mt->level[level].depth); *x = mt->level[level].slice[slice].x_offset; @@ -1209,26 +1574,26 @@ intel_miptree_get_tile_offsets(const struct intel_mipmap_tree *mt, static void intel_miptree_copy_slice_sw(struct brw_context *brw, - struct intel_mipmap_tree *dst_mt, struct intel_mipmap_tree *src_mt, - int level, - int slice, - int width, - int height) + unsigned src_level, unsigned src_layer, + struct intel_mipmap_tree *dst_mt, + unsigned dst_level, unsigned dst_layer, + unsigned width, unsigned height) { void *src, *dst; ptrdiff_t src_stride, dst_stride; - int cpp = dst_mt->cpp; + const unsigned cpp = dst_mt->surf.size > 0 ? + (isl_format_get_layout(dst_mt->surf.format)->bpb / 8) : dst_mt->cpp; intel_miptree_map(brw, src_mt, - level, slice, + src_level, src_layer, 0, 0, width, height, GL_MAP_READ_BIT | BRW_MAP_DIRECT_BIT, &src, &src_stride); intel_miptree_map(brw, dst_mt, - level, slice, + dst_level, dst_layer, 0, 0, width, height, GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT | @@ -1254,8 +1619,8 @@ intel_miptree_copy_slice_sw(struct brw_context *brw, } } - intel_miptree_unmap(brw, dst_mt, level, slice); - intel_miptree_unmap(brw, src_mt, level, slice); + intel_miptree_unmap(brw, dst_mt, dst_level, dst_layer); + intel_miptree_unmap(brw, src_mt, src_level, src_layer); /* Don't forget to copy the stencil data over, too. We could have skipped * passing BRW_MAP_DIRECT_BIT, but that would have meant intel_miptree_map @@ -1264,31 +1629,43 @@ intel_miptree_copy_slice_sw(struct brw_context *brw, */ if (dst_mt->stencil_mt) { assert(src_mt->stencil_mt); - intel_miptree_copy_slice_sw(brw, dst_mt->stencil_mt, src_mt->stencil_mt, - level, slice, width, height); + intel_miptree_copy_slice_sw(brw, + src_mt->stencil_mt, src_level, src_layer, + dst_mt->stencil_mt, dst_level, dst_layer, + width, height); } } -static void +void intel_miptree_copy_slice(struct brw_context *brw, - struct intel_mipmap_tree *dst_mt, - struct intel_mipmap_tree *src_mt, - int level, - int face, - int depth) + struct intel_mipmap_tree *src_mt, + unsigned src_level, unsigned src_layer, + struct intel_mipmap_tree *dst_mt, + unsigned dst_level, unsigned dst_layer) { mesa_format format = src_mt->format; - uint32_t width = minify(src_mt->physical_width0, level - src_mt->first_level); - uint32_t height = minify(src_mt->physical_height0, level - src_mt->first_level); - int slice; - - if (face > 0) - slice = face; - else - slice = depth; + uint32_t width, height; + + if (src_mt->surf.size > 0) { + width = minify(src_mt->surf.phys_level0_sa.width, + src_level - src_mt->first_level); + height = minify(src_mt->surf.phys_level0_sa.height, + src_level - src_mt->first_level); + + if (src_mt->surf.dim == ISL_SURF_DIM_3D) + assert(src_layer < minify(src_mt->surf.phys_level0_sa.depth, + src_level - src_mt->first_level)); + else + assert(src_layer < src_mt->surf.phys_level0_sa.array_len); + } else { + width = minify(src_mt->physical_width0, + src_level - src_mt->first_level); + height = minify(src_mt->physical_height0, + src_level - src_mt->first_level); + assert(src_layer < src_mt->level[src_level].depth); + } - assert(depth < src_mt->level[level].depth); assert(src_mt->format == dst_mt->format); if (dst_mt->compressed) { @@ -1304,15 +1681,17 @@ intel_miptree_copy_slice(struct brw_context *brw, */ if (src_mt->stencil_mt) { intel_miptree_copy_slice_sw(brw, - dst_mt, src_mt, - level, slice, + src_mt, src_level, src_layer, + dst_mt, dst_level, dst_layer, width, height); return; } uint32_t dst_x, dst_y, src_x, src_y; - intel_miptree_get_image_offset(dst_mt, level, slice, &dst_x, &dst_y); - intel_miptree_get_image_offset(src_mt, level, slice, &src_x, &src_y); + intel_miptree_get_image_offset(dst_mt, dst_level, dst_layer, + &dst_x, &dst_y); + intel_miptree_get_image_offset(src_mt, src_level, src_layer, + &src_x, &src_y); DBG("validate blit mt %s %p %d,%d/%d -> mt %s %p %d,%d/%d (%dx%d)\n", _mesa_get_format_name(src_mt->format), @@ -1322,13 +1701,15 @@ intel_miptree_copy_slice(struct brw_context *brw, width, height); if (!intel_miptree_blit(brw, - src_mt, level, slice, 0, 0, false, - dst_mt, level, slice, 0, 0, false, + src_mt, src_level, src_layer, 0, 0, false, + dst_mt, dst_level, dst_layer, 0, 0, false, width, height, GL_COPY)) { perf_debug("miptree validate blit for %s failed\n", _mesa_get_format_name(format)); - intel_miptree_copy_slice_sw(brw, dst_mt, src_mt, level, slice, + intel_miptree_copy_slice_sw(brw, + src_mt, src_level, src_layer, + dst_mt, dst_level, dst_layer, width, height); } } @@ -1351,17 +1732,28 @@ intel_miptree_copy_teximage(struct brw_context *brw, struct intel_texture_object *intel_obj = intel_texture_object(intelImage->base.Base.TexObject); int level = intelImage->base.Base.Level; - int face = intelImage->base.Base.Face; - - GLuint depth; - if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) - depth = intelImage->base.Base.Height; - else - depth = intelImage->base.Base.Depth; + const unsigned face = intelImage->base.Base.Face; + unsigned start_layer, end_layer; + + if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) { + assert(face == 0); + assert(intelImage->base.Base.Height); + start_layer = 0; + end_layer = intelImage->base.Base.Height - 1; + } else if (face > 0) { + start_layer = face; + end_layer = face; + } else { + assert(intelImage->base.Base.Depth); + start_layer = 0; + end_layer = intelImage->base.Base.Depth - 1; + } if (!invalidate) { - for (int slice = 0; slice < depth; slice++) { - intel_miptree_copy_slice(brw, dst_mt, src_mt, level, face, slice); + for (unsigned i = start_layer; i <= end_layer; i++) { + intel_miptree_copy_slice(brw, + src_mt, level, i, + dst_mt, level, i); } } @@ -1399,51 +1791,32 @@ intel_miptree_init_mcs(struct brw_context *brw, } static struct intel_miptree_aux_buffer * -intel_mcs_miptree_buf_create(struct brw_context *brw, - struct intel_mipmap_tree *mt, - mesa_format format, - unsigned mcs_width, - unsigned mcs_height, - uint32_t layout_flags) +intel_alloc_aux_buffer(struct brw_context *brw, + const char *name, + const struct isl_surf *aux_surf, + uint32_t alloc_flags, + struct intel_mipmap_tree *mt) { struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); - struct intel_mipmap_tree *temp_mt; - if (!buf) - return NULL; + return false; - /* From the Ivy Bridge PRM, Vol4 Part1 p76, "MCS Base Address": - * - * "The MCS surface must be stored as Tile Y." + buf->size = aux_surf->size; + buf->pitch = aux_surf->row_pitch; + buf->qpitch = isl_surf_get_array_pitch_sa_rows(aux_surf); + + /* ISL has stricter set of alignment rules then the drm allocator. + * Therefore one can pass the ISL dimensions in terms of bytes instead of + * trying to recalculate based on different format block sizes. */ - layout_flags |= MIPTREE_LAYOUT_TILING_Y; - temp_mt = miptree_create(brw, - mt->target, - format, - mt->first_level, - mt->last_level, - mcs_width, - mcs_height, - mt->logical_depth0, - 0 /* num_samples */, - layout_flags); - if (!temp_mt) { + buf->bo = brw_bo_alloc_tiled(brw->bufmgr, name, buf->size, + I915_TILING_Y, buf->pitch, alloc_flags); + if (!buf->bo) { free(buf); return NULL; } - buf->bo = temp_mt->bo; - buf->offset = temp_mt->offset; - buf->size = temp_mt->total_height * temp_mt->pitch; - buf->pitch = temp_mt->pitch; - buf->qpitch = temp_mt->qpitch; - - /* Just hang on to the BO which backs the AUX buffer; the rest of the miptree - * structure should go away. We use miptree create simply as a means to make - * sure all the constraints for the buffer are satisfied. - */ - brw_bo_reference(temp_mt->bo); - intel_miptree_release(&temp_mt); + buf->surf = *aux_surf; return buf; } @@ -1455,66 +1828,53 @@ intel_miptree_alloc_mcs(struct brw_context *brw, { assert(brw->gen >= 7); /* MCS only used on Gen7+ */ assert(mt->mcs_buf == NULL); - assert((mt->aux_disable & INTEL_AUX_DISABLE_MCS) == 0); + assert(mt->aux_usage == ISL_AUX_USAGE_MCS); - /* Choose the correct format for the MCS buffer. All that really matters - * is that we allocate the right buffer size, since we'll always be - * accessing this miptree using MCS-specific hardware mechanisms, which - * infer the correct format based on num_samples. - */ - mesa_format format; - switch (num_samples) { - case 2: - case 4: - /* 8 bits/pixel are required for MCS data when using 4x MSAA (2 bits for - * each sample). - */ - format = MESA_FORMAT_R_UNORM8; - break; - case 8: - /* 32 bits/pixel are required for MCS data when using 8x MSAA (3 bits - * for each sample, plus 8 padding bits). - */ - format = MESA_FORMAT_R_UINT32; - break; - case 16: - /* 64 bits/pixel are required for MCS data when using 16x MSAA (4 bits - * for each sample). - */ - format = MESA_FORMAT_RG_UINT32; - break; - default: - unreachable("Unrecognized sample count in intel_miptree_alloc_mcs"); - }; + /* Multisampled miptrees are only supported for single level. */ + assert(mt->first_level == 0); + enum isl_aux_state **aux_state = + create_aux_state_map(mt, ISL_AUX_STATE_CLEAR); + if (!aux_state) + return false; - mt->mcs_buf = - intel_mcs_miptree_buf_create(brw, mt, - format, - mt->logical_width0, - mt->logical_height0, - MIPTREE_LAYOUT_ACCELERATED_UPLOAD); - if (!mt->mcs_buf) + struct isl_surf temp_main_surf; + struct isl_surf temp_mcs_surf; + + /* Create first an ISL presentation for the main color surface and let ISL + * calculate equivalent MCS surface against it. + */ + intel_miptree_get_isl_surf(brw, mt, &temp_main_surf); + MAYBE_UNUSED bool ok = + isl_surf_get_mcs_surf(&brw->isl_dev, &temp_main_surf, &temp_mcs_surf); + assert(ok); + + /* Buffer needs to be initialised requiring the buffer to be immediately + * mapped to cpu space for writing. Therefore do not use the gpu access + * flag which can cause an unnecessary delay if the backing pages happened + * to be just used by the GPU. + */ + const uint32_t alloc_flags = 0; + mt->mcs_buf = intel_alloc_aux_buffer(brw, "mcs-miptree", + &temp_mcs_surf, alloc_flags, mt); + if (!mt->mcs_buf) { + free(aux_state); return false; + } - intel_miptree_init_mcs(brw, mt, 0xFF); + mt->aux_state = aux_state; - /* Multisampled miptrees are only supported for single level. */ - assert(mt->first_level == 0); - intel_miptree_set_fast_clear_state(brw, mt, mt->first_level, 0, - mt->logical_depth0, - INTEL_FAST_CLEAR_STATE_CLEAR); + intel_miptree_init_mcs(brw, mt, 0xFF); return true; } - bool -intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, - struct intel_mipmap_tree *mt, - bool is_lossless_compressed) +intel_miptree_alloc_ccs(struct brw_context *brw, + struct intel_mipmap_tree *mt) { assert(mt->mcs_buf == NULL); - assert(!(mt->aux_disable & (INTEL_AUX_DISABLE_MCS | INTEL_AUX_DISABLE_CCS))); + assert(mt->aux_usage == ISL_AUX_USAGE_CCS_E || + mt->aux_usage == ISL_AUX_USAGE_CCS_D); struct isl_surf temp_main_surf; struct isl_surf temp_ccs_surf; @@ -1529,41 +1889,32 @@ intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw, assert(temp_ccs_surf.size && (temp_ccs_surf.size % temp_ccs_surf.row_pitch == 0)); - struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); - if (!buf) + enum isl_aux_state **aux_state = + create_aux_state_map(mt, ISL_AUX_STATE_PASS_THROUGH); + if (!aux_state) return false; - buf->size = temp_ccs_surf.size; - buf->pitch = temp_ccs_surf.row_pitch; - buf->qpitch = isl_surf_get_array_pitch_sa_rows(&temp_ccs_surf); - /* In case of compression mcs buffer needs to be initialised requiring the * buffer to be immediately mapped to cpu space for writing. Therefore do * not use the gpu access flag which can cause an unnecessary delay if the * backing pages happened to be just used by the GPU. */ const uint32_t alloc_flags = - is_lossless_compressed ? 0 : BO_ALLOC_FOR_RENDER; - - /* ISL has stricter set of alignment rules then the drm allocator. - * Therefore one can pass the ISL dimensions in terms of bytes instead of - * trying to recalculate based on different format block sizes. - */ - buf->bo = brw_bo_alloc_tiled(brw->bufmgr, "ccs-miptree", - buf->pitch, buf->size / buf->pitch, - 1, I915_TILING_Y, &buf->pitch, alloc_flags); - if (!buf->bo) { - free(buf); + mt->aux_usage == ISL_AUX_USAGE_CCS_E ? 0 : BO_ALLOC_FOR_RENDER; + mt->mcs_buf = intel_alloc_aux_buffer(brw, "ccs-miptree", + &temp_ccs_surf, alloc_flags, mt); + if (!mt->mcs_buf) { + free(aux_state); return false; } - - mt->mcs_buf = buf; + + mt->aux_state = aux_state; /* From Gen9 onwards single-sampled (non-msrt) auxiliary buffers are * used for lossless compression which requires similar initialisation * as multi-sample compression. */ - if (is_lossless_compressed) { + if (mt->aux_usage == ISL_AUX_USAGE_CCS_E) { /* Hardware sets the auxiliary buffer to all zeroes when it does full * resolve. Initialize it accordingly in case the first renderer is * cpu (or other none compression aware party). @@ -1613,275 +1964,40 @@ intel_miptree_level_enable_hiz(struct brw_context *brw, return true; } - -/** - * Helper for intel_miptree_alloc_hiz() that determines the required hiz - * buffer dimensions and allocates a bo for the hiz buffer. - */ -static struct intel_miptree_hiz_buffer * -intel_gen7_hiz_buf_create(struct brw_context *brw, - struct intel_mipmap_tree *mt) -{ - unsigned z_width = mt->logical_width0; - unsigned z_height = mt->logical_height0; - const unsigned z_depth = MAX2(mt->logical_depth0, 1); - unsigned hz_width, hz_height; - struct intel_miptree_hiz_buffer *buf = calloc(sizeof(*buf), 1); - - if (!buf) - return NULL; - - /* Gen7 PRM Volume 2, Part 1, 11.5.3 "Hierarchical Depth Buffer" documents - * adjustments required for Z_Height and Z_Width based on multisampling. - */ - switch (mt->num_samples) { - case 0: - case 1: - break; - case 2: - case 4: - z_width *= 2; - z_height *= 2; - break; - case 8: - z_width *= 4; - z_height *= 2; - break; - default: - unreachable("unsupported sample count"); - } - - const unsigned vertical_align = 8; /* 'j' in the docs */ - const unsigned H0 = z_height; - const unsigned h0 = ALIGN(H0, vertical_align); - const unsigned h1 = ALIGN(minify(H0, 1), vertical_align); - const unsigned Z0 = z_depth; - - /* HZ_Width (bytes) = ceiling(Z_Width / 16) * 16 */ - hz_width = ALIGN(z_width, 16); - - if (mt->target == GL_TEXTURE_3D) { - unsigned H_i = H0; - unsigned Z_i = Z0; - hz_height = 0; - for (unsigned level = mt->first_level; level <= mt->last_level; ++level) { - unsigned h_i = ALIGN(H_i, vertical_align); - /* sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ - hz_height += h_i * Z_i; - H_i = minify(H_i, 1); - Z_i = minify(Z_i, 1); - } - /* HZ_Height = - * (1/2) * sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) - */ - hz_height = DIV_ROUND_UP(hz_height, 2); - } else { - const unsigned hz_qpitch = h0 + h1 + (12 * vertical_align); - /* HZ_Height (rows) = Ceiling ( ( Q_pitch * Z_depth/2) /8 ) * 8 */ - hz_height = DIV_ROUND_UP(hz_qpitch * Z0, 2 * 8) * 8; - } - - buf->aux_base.bo = brw_bo_alloc_tiled(brw->bufmgr, "hiz", - hz_width, hz_height, 1, - I915_TILING_Y, &buf->aux_base.pitch, - BO_ALLOC_FOR_RENDER); - if (!buf->aux_base.bo) { - free(buf); - return NULL; - } - - buf->aux_base.size = hz_width * hz_height; - - return buf; -} - - -/** - * Helper for intel_miptree_alloc_hiz() that determines the required hiz - * buffer dimensions and allocates a bo for the hiz buffer. - */ -static struct intel_miptree_hiz_buffer * -intel_gen8_hiz_buf_create(struct brw_context *brw, - struct intel_mipmap_tree *mt) -{ - unsigned z_width = mt->logical_width0; - unsigned z_height = mt->logical_height0; - const unsigned z_depth = MAX2(mt->logical_depth0, 1); - unsigned hz_width, hz_height; - struct intel_miptree_hiz_buffer *buf = calloc(sizeof(*buf), 1); - - if (!buf) - return NULL; - - /* Gen7 PRM Volume 2, Part 1, 11.5.3 "Hierarchical Depth Buffer" documents - * adjustments required for Z_Height and Z_Width based on multisampling. - */ - if (brw->gen < 9) { - switch (mt->num_samples) { - case 0: - case 1: - break; - case 2: - case 4: - z_width *= 2; - z_height *= 2; - break; - case 8: - z_width *= 4; - z_height *= 2; - break; - default: - unreachable("unsupported sample count"); - } - } - - const unsigned vertical_align = 8; /* 'j' in the docs */ - const unsigned H0 = z_height; - const unsigned h0 = ALIGN(H0, vertical_align); - const unsigned h1 = ALIGN(minify(H0, 1), vertical_align); - const unsigned Z0 = z_depth; - - /* HZ_Width (bytes) = ceiling(Z_Width / 16) * 16 */ - hz_width = ALIGN(z_width, 16); - - unsigned H_i = H0; - unsigned Z_i = Z0; - unsigned sum_h_i = 0; - unsigned hz_height_3d_sum = 0; - for (unsigned level = mt->first_level; level <= mt->last_level; ++level) { - unsigned i = level - mt->first_level; - unsigned h_i = ALIGN(H_i, vertical_align); - /* sum(i=2 to m; h_i) */ - if (i >= 2) { - sum_h_i += h_i; - } - /* sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ - hz_height_3d_sum += h_i * Z_i; - H_i = minify(H_i, 1); - Z_i = minify(Z_i, 1); - } - /* HZ_QPitch = h0 + max(h1, sum(i=2 to m; h_i)) */ - buf->aux_base.qpitch = h0 + MAX2(h1, sum_h_i); - - if (mt->target == GL_TEXTURE_3D) { - /* (1/2) * sum(i=0 to m; h_i * max(1, floor(Z_Depth/2**i))) */ - hz_height = DIV_ROUND_UP(hz_height_3d_sum, 2); - } else { - /* HZ_Height (rows) = ceiling( (HZ_QPitch/2)/8) *8 * Z_Depth */ - hz_height = DIV_ROUND_UP(buf->aux_base.qpitch, 2 * 8) * 8 * Z0; - } - - buf->aux_base.bo = brw_bo_alloc_tiled(brw->bufmgr, "hiz", - hz_width, hz_height, 1, - I915_TILING_Y, &buf->aux_base.pitch, - BO_ALLOC_FOR_RENDER); - if (!buf->aux_base.bo) { - free(buf); - return NULL; - } - - buf->aux_base.size = hz_width * hz_height; - - return buf; -} - - -static struct intel_miptree_hiz_buffer * -intel_hiz_miptree_buf_create(struct brw_context *brw, - struct intel_mipmap_tree *mt) -{ - struct intel_miptree_hiz_buffer *buf = calloc(sizeof(*buf), 1); - uint32_t layout_flags = MIPTREE_LAYOUT_ACCELERATED_UPLOAD; - - if (brw->gen == 6) - layout_flags |= MIPTREE_LAYOUT_GEN6_HIZ_STENCIL; - - if (!buf) - return NULL; - - layout_flags |= MIPTREE_LAYOUT_TILING_ANY; - buf->mt = intel_miptree_create(brw, - mt->target, - mt->format, - mt->first_level, - mt->last_level, - mt->logical_width0, - mt->logical_height0, - mt->logical_depth0, - mt->num_samples, - layout_flags); - if (!buf->mt) { - free(buf); - return NULL; - } - - buf->aux_base.bo = buf->mt->bo; - buf->aux_base.size = buf->mt->total_height * buf->mt->pitch; - buf->aux_base.pitch = buf->mt->pitch; - buf->aux_base.qpitch = buf->mt->qpitch * 2; - - return buf; -} - -bool -intel_miptree_wants_hiz_buffer(struct brw_context *brw, - struct intel_mipmap_tree *mt) -{ - if (!brw->has_hiz) - return false; - - if (mt->hiz_buf != NULL) - return false; - - if (mt->aux_disable & INTEL_AUX_DISABLE_HIZ) - return false; - - switch (mt->format) { - case MESA_FORMAT_Z_FLOAT32: - case MESA_FORMAT_Z32_FLOAT_S8X24_UINT: - case MESA_FORMAT_Z24_UNORM_X8_UINT: - case MESA_FORMAT_Z24_UNORM_S8_UINT: - case MESA_FORMAT_Z_UNORM16: - return true; - default: - return false; - } -} - bool intel_miptree_alloc_hiz(struct brw_context *brw, struct intel_mipmap_tree *mt) { assert(mt->hiz_buf == NULL); - assert((mt->aux_disable & INTEL_AUX_DISABLE_HIZ) == 0); + assert(mt->aux_usage == ISL_AUX_USAGE_HIZ); - if (brw->gen == 7) { - mt->hiz_buf = intel_gen7_hiz_buf_create(brw, mt); - } else if (brw->gen >= 8) { - mt->hiz_buf = intel_gen8_hiz_buf_create(brw, mt); - } else { - mt->hiz_buf = intel_hiz_miptree_buf_create(brw, mt); - } - - if (!mt->hiz_buf) + enum isl_aux_state **aux_state = + create_aux_state_map(mt, ISL_AUX_STATE_AUX_INVALID); + if (!aux_state) return false; - /* Mark that all slices need a HiZ resolve. */ - for (unsigned level = mt->first_level; level <= mt->last_level; ++level) { - if (!intel_miptree_level_enable_hiz(brw, mt, level)) - continue; + struct isl_surf temp_main_surf; + struct isl_surf temp_hiz_surf; - for (unsigned layer = 0; layer < mt->level[level].depth; ++layer) { - struct intel_resolve_map *m = malloc(sizeof(struct intel_resolve_map)); - exec_node_init(&m->link); - m->level = level; - m->layer = layer; - m->need = BLORP_HIZ_OP_HIZ_RESOLVE; + intel_miptree_get_isl_surf(brw, mt, &temp_main_surf); + MAYBE_UNUSED bool ok = + isl_surf_get_hiz_surf(&brw->isl_dev, &temp_main_surf, &temp_hiz_surf); + assert(ok); - exec_list_push_tail(&mt->hiz_map, &m->link); - } + const uint32_t alloc_flags = BO_ALLOC_FOR_RENDER; + mt->hiz_buf = intel_alloc_aux_buffer(brw, "hiz-miptree", + &temp_hiz_surf, alloc_flags, mt); + + if (!mt->hiz_buf) { + free(aux_state); + return false; } + for (unsigned level = mt->first_level; level <= mt->last_level; ++level) + intel_miptree_level_enable_hiz(brw, mt, level); + + mt->aux_state = aux_state; + return true; } @@ -1926,134 +2042,48 @@ intel_miptree_sample_with_hiz(struct brw_context *brw, */ return (mt->num_samples <= 1 && mt->target != GL_TEXTURE_3D && - mt->target != GL_TEXTURE_1D /* gen9+ restriction */); -} - -/** - * Does the miptree slice have hiz enabled? - */ -bool -intel_miptree_level_has_hiz(struct intel_mipmap_tree *mt, uint32_t level) -{ - intel_miptree_check_level_layer(mt, level, 0); - return mt->level[level].has_hiz; -} - -void -intel_miptree_slice_set_needs_hiz_resolve(struct intel_mipmap_tree *mt, - uint32_t level, - uint32_t layer) -{ - if (!intel_miptree_level_has_hiz(mt, level)) - return; - - intel_resolve_map_set(&mt->hiz_map, - level, layer, BLORP_HIZ_OP_HIZ_RESOLVE); -} - - -void -intel_miptree_slice_set_needs_depth_resolve(struct intel_mipmap_tree *mt, - uint32_t level, - uint32_t layer) -{ - if (!intel_miptree_level_has_hiz(mt, level)) - return; - - intel_resolve_map_set(&mt->hiz_map, - level, layer, BLORP_HIZ_OP_DEPTH_RESOLVE); -} - -void -intel_miptree_set_all_slices_need_depth_resolve(struct intel_mipmap_tree *mt, - uint32_t level) -{ - uint32_t layer; - uint32_t end_layer = mt->level[level].depth; - - for (layer = 0; layer < end_layer; layer++) { - intel_miptree_slice_set_needs_depth_resolve(mt, level, layer); - } -} - -static bool -intel_miptree_depth_hiz_resolve(struct brw_context *brw, - struct intel_mipmap_tree *mt, - uint32_t start_level, uint32_t num_levels, - uint32_t start_layer, uint32_t num_layers, - enum blorp_hiz_op need) -{ - bool did_resolve = false; - - foreach_list_typed_safe(struct intel_resolve_map, map, link, &mt->hiz_map) { - if (map->level < start_level || - map->level >= (start_level + num_levels) || - map->layer < start_layer || - map->layer >= (start_layer + num_layers)) - continue; - - if (map->need != need) - continue; - - intel_hiz_exec(brw, mt, map->level, map->layer, 1, need); - intel_resolve_map_remove(map); - did_resolve = true; - } - - return did_resolve; -} - -bool -intel_miptree_slice_resolve_hiz(struct brw_context *brw, - struct intel_mipmap_tree *mt, - uint32_t level, - uint32_t layer) -{ - return intel_miptree_depth_hiz_resolve(brw, mt, level, 1, layer, 1, - BLORP_HIZ_OP_HIZ_RESOLVE); -} - -bool -intel_miptree_slice_resolve_depth(struct brw_context *brw, - struct intel_mipmap_tree *mt, - uint32_t level, - uint32_t layer) -{ - return intel_miptree_depth_hiz_resolve(brw, mt, level, 1, layer, 1, - BLORP_HIZ_OP_DEPTH_RESOLVE); + mt->target != GL_TEXTURE_1D /* gen9+ restriction */); } +/** + * Does the miptree slice have hiz enabled? + */ bool -intel_miptree_all_slices_resolve_hiz(struct brw_context *brw, - struct intel_mipmap_tree *mt) +intel_miptree_level_has_hiz(const struct intel_mipmap_tree *mt, uint32_t level) { - return intel_miptree_depth_hiz_resolve(brw, mt, - 0, UINT32_MAX, 0, UINT32_MAX, - BLORP_HIZ_OP_HIZ_RESOLVE); + intel_miptree_check_level_layer(mt, level, 0); + return mt->level[level].has_hiz; } bool -intel_miptree_all_slices_resolve_depth(struct brw_context *brw, - struct intel_mipmap_tree *mt) -{ - return intel_miptree_depth_hiz_resolve(brw, mt, - 0, UINT32_MAX, 0, UINT32_MAX, - BLORP_HIZ_OP_DEPTH_RESOLVE); -} - -enum intel_fast_clear_state -intel_miptree_get_fast_clear_state(const struct intel_mipmap_tree *mt, - unsigned level, unsigned layer) +intel_miptree_has_color_unresolved(const struct intel_mipmap_tree *mt, + unsigned start_level, unsigned num_levels, + unsigned start_layer, unsigned num_layers) { - intel_miptree_check_level_layer(mt, level, layer); + assert(_mesa_is_format_color_format(mt->format)); - const struct intel_resolve_map *item = - intel_resolve_map_const_get(&mt->color_resolve_map, level, layer); + if (!mt->mcs_buf) + return false; - if (!item) - return INTEL_FAST_CLEAR_STATE_RESOLVED; + /* Clamp the level range to fit the miptree */ + assert(start_level + num_levels >= start_level); + const uint32_t last_level = + MIN2(mt->last_level, start_level + num_levels - 1); + start_level = MAX2(mt->first_level, start_level); + num_levels = last_level - start_level + 1; + + for (uint32_t level = start_level; level <= last_level; level++) { + const uint32_t level_layers = MIN2(num_layers, mt->level[level].depth); + for (unsigned a = 0; a < level_layers; a++) { + enum isl_aux_state aux_state = + intel_miptree_get_aux_state(mt, level, start_layer + a); + assert(aux_state != ISL_AUX_STATE_AUX_INVALID); + if (aux_state != ISL_AUX_STATE_PASS_THROUGH) + return true; + } + } - return item->fast_clear_state; + return false; } static void @@ -2062,7 +2092,7 @@ intel_miptree_check_color_resolve(const struct brw_context *brw, unsigned level, unsigned layer) { - if ((mt->aux_disable & INTEL_AUX_DISABLE_CCS) || !mt->mcs_buf) + if (!mt->mcs_buf) return; /* Fast color clear is supported for mipmapped surfaces only on Gen8+. */ @@ -2080,144 +2110,279 @@ intel_miptree_check_color_resolve(const struct brw_context *brw, (void)layer; } -void -intel_miptree_set_fast_clear_state(const struct brw_context *brw, - struct intel_mipmap_tree *mt, - unsigned level, - unsigned first_layer, - unsigned num_layers, - enum intel_fast_clear_state new_state) +static enum blorp_fast_clear_op +get_ccs_d_resolve_op(enum isl_aux_state aux_state, + bool ccs_supported, bool fast_clear_supported) { - /* Setting the state to resolved means removing the item from the list - * altogether. - */ - assert(new_state != INTEL_FAST_CLEAR_STATE_RESOLVED); + assert(ccs_supported == fast_clear_supported); - intel_miptree_check_color_resolve(brw, mt, level, first_layer); + switch (aux_state) { + case ISL_AUX_STATE_CLEAR: + case ISL_AUX_STATE_COMPRESSED_CLEAR: + if (!ccs_supported) + return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + else + return BLORP_FAST_CLEAR_OP_NONE; - assert(first_layer + num_layers <= mt->physical_depth0); + case ISL_AUX_STATE_PASS_THROUGH: + return BLORP_FAST_CLEAR_OP_NONE; - for (unsigned i = 0; i < num_layers; i++) - intel_resolve_map_set(&mt->color_resolve_map, level, - first_layer + i, new_state); -} + case ISL_AUX_STATE_RESOLVED: + case ISL_AUX_STATE_AUX_INVALID: + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + break; + } -bool -intel_miptree_has_color_unresolved(const struct intel_mipmap_tree *mt, - unsigned start_level, unsigned num_levels, - unsigned start_layer, unsigned num_layers) -{ - return intel_resolve_map_find_any(&mt->color_resolve_map, - start_level, num_levels, - start_layer, num_layers) != NULL; + unreachable("Invalid aux state for CCS_D"); } -void -intel_miptree_used_for_rendering(const struct brw_context *brw, - struct intel_mipmap_tree *mt, unsigned level, - unsigned start_layer, unsigned num_layers) +static enum blorp_fast_clear_op +get_ccs_e_resolve_op(enum isl_aux_state aux_state, + bool ccs_supported, bool fast_clear_supported) { - const bool is_lossless_compressed = - intel_miptree_is_lossless_compressed(brw, mt); + switch (aux_state) { + case ISL_AUX_STATE_CLEAR: + case ISL_AUX_STATE_COMPRESSED_CLEAR: + if (!ccs_supported) + return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + else if (!fast_clear_supported) + return BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL; + else + return BLORP_FAST_CLEAR_OP_NONE; - for (unsigned i = 0; i < num_layers; ++i) { - const enum intel_fast_clear_state fast_clear_state = - intel_miptree_get_fast_clear_state(mt, level, start_layer + i); + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + if (!ccs_supported) + return BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + else + return BLORP_FAST_CLEAR_OP_NONE; - /* If the buffer was previously in fast clear state, change it to - * unresolved state, since it won't be guaranteed to be clear after - * rendering occurs. - */ - if (is_lossless_compressed || - fast_clear_state == INTEL_FAST_CLEAR_STATE_CLEAR) { - intel_miptree_set_fast_clear_state( - brw, mt, level, start_layer + i, 1, - INTEL_FAST_CLEAR_STATE_UNRESOLVED); - } + case ISL_AUX_STATE_PASS_THROUGH: + return BLORP_FAST_CLEAR_OP_NONE; + + case ISL_AUX_STATE_RESOLVED: + case ISL_AUX_STATE_AUX_INVALID: + break; } + + unreachable("Invalid aux state for CCS_E"); } -static bool -intel_miptree_needs_color_resolve(const struct brw_context *brw, - const struct intel_mipmap_tree *mt, - int flags) +static void +intel_miptree_prepare_ccs_access(struct brw_context *brw, + struct intel_mipmap_tree *mt, + uint32_t level, uint32_t layer, + bool aux_supported, + bool fast_clear_supported) { - if (mt->aux_disable & INTEL_AUX_DISABLE_CCS) - return false; + enum isl_aux_state aux_state = intel_miptree_get_aux_state(mt, level, layer); - const bool is_lossless_compressed = - intel_miptree_is_lossless_compressed(brw, mt); + enum blorp_fast_clear_op resolve_op; + if (intel_miptree_is_lossless_compressed(brw, mt)) { + resolve_op = get_ccs_e_resolve_op(aux_state, aux_supported, + fast_clear_supported); + } else { + resolve_op = get_ccs_d_resolve_op(aux_state, aux_supported, + fast_clear_supported); + } - /* From gen9 onwards there is new compression scheme for single sampled - * surfaces called "lossless compressed". These don't need to be always - * resolved. - */ - if ((flags & INTEL_MIPTREE_IGNORE_CCS_E) && is_lossless_compressed) - return false; + if (resolve_op != BLORP_FAST_CLEAR_OP_NONE) { + intel_miptree_check_color_resolve(brw, mt, level, layer); + brw_blorp_resolve_color(brw, mt, level, layer, resolve_op); - /* Fast color clear resolves only make sense for non-MSAA buffers. */ - if (mt->msaa_layout != INTEL_MSAA_LAYOUT_NONE && !is_lossless_compressed) - return false; + switch (resolve_op) { + case BLORP_FAST_CLEAR_OP_RESOLVE_FULL: + /* The CCS full resolve operation destroys the CCS and sets it to the + * pass-through state. (You can also think of this as being both a + * resolve and an ambiguate in one operation.) + */ + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_PASS_THROUGH); + break; - return true; + case BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL: + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_NO_CLEAR); + break; + + default: + unreachable("Invalid resolve op"); + } + } } -bool -intel_miptree_resolve_color(struct brw_context *brw, - struct intel_mipmap_tree *mt, - uint32_t start_level, uint32_t num_levels, - uint32_t start_layer, uint32_t num_layers, - int flags) +static void +intel_miptree_finish_ccs_write(struct brw_context *brw, + struct intel_mipmap_tree *mt, + uint32_t level, uint32_t layer, + bool written_with_ccs) { - intel_miptree_check_color_resolve(brw, mt, start_level, start_layer); + enum isl_aux_state aux_state = intel_miptree_get_aux_state(mt, level, layer); - if (!intel_miptree_needs_color_resolve(brw, mt, flags)) - return false; + if (intel_miptree_is_lossless_compressed(brw, mt)) { + switch (aux_state) { + case ISL_AUX_STATE_CLEAR: + assert(written_with_ccs); + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_CLEAR); + break; - enum blorp_fast_clear_op resolve_op; - if (brw->gen >= 9) { - if (intel_miptree_is_lossless_compressed(brw, mt)) { - resolve_op = BLORP_FAST_CLEAR_OP_RESOLVE_FULL; - } else { - resolve_op = BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL; + case ISL_AUX_STATE_COMPRESSED_CLEAR: + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + assert(written_with_ccs); + break; /* Nothing to do */ + + case ISL_AUX_STATE_PASS_THROUGH: + if (written_with_ccs) { + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_NO_CLEAR); + } else { + /* Nothing to do */ + } + break; + + case ISL_AUX_STATE_RESOLVED: + case ISL_AUX_STATE_AUX_INVALID: + unreachable("Invalid aux state for CCS_E"); } } else { - /* Broadwell and earlier do not have a partial resolve */ - assert(!intel_miptree_is_lossless_compressed(brw, mt)); - resolve_op = BLORP_FAST_CLEAR_OP_RESOLVE_FULL; + /* CCS_D is a bit simpler */ + switch (aux_state) { + case ISL_AUX_STATE_CLEAR: + assert(written_with_ccs); + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_CLEAR); + break; + + case ISL_AUX_STATE_COMPRESSED_CLEAR: + assert(written_with_ccs); + break; /* Nothing to do */ + + case ISL_AUX_STATE_PASS_THROUGH: + /* Nothing to do */ + break; + + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + case ISL_AUX_STATE_RESOLVED: + case ISL_AUX_STATE_AUX_INVALID: + unreachable("Invalid aux state for CCS_D"); + } } +} + +static void +intel_miptree_finish_mcs_write(struct brw_context *brw, + struct intel_mipmap_tree *mt, + uint32_t level, uint32_t layer, + bool written_with_aux) +{ + switch (intel_miptree_get_aux_state(mt, level, layer)) { + case ISL_AUX_STATE_CLEAR: + assert(written_with_aux); + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_CLEAR); + break; - bool resolved = false; - foreach_list_typed_safe(struct intel_resolve_map, map, link, - &mt->color_resolve_map) { - if (map->level < start_level || - map->level >= (start_level + num_levels) || - map->layer < start_layer || - map->layer >= (start_layer + num_layers)) - continue; + case ISL_AUX_STATE_COMPRESSED_CLEAR: + assert(written_with_aux); + break; /* Nothing to do */ - /* Arrayed and mip-mapped fast clear is only supported for gen8+. */ - assert(brw->gen >= 8 || (map->level == 0 && map->layer == 0)); + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + case ISL_AUX_STATE_RESOLVED: + case ISL_AUX_STATE_PASS_THROUGH: + case ISL_AUX_STATE_AUX_INVALID: + unreachable("Invalid aux state for MCS"); + } +} + +static void +intel_miptree_prepare_hiz_access(struct brw_context *brw, + struct intel_mipmap_tree *mt, + uint32_t level, uint32_t layer, + bool hiz_supported, bool fast_clear_supported) +{ + enum blorp_hiz_op hiz_op = BLORP_HIZ_OP_NONE; + switch (intel_miptree_get_aux_state(mt, level, layer)) { + case ISL_AUX_STATE_CLEAR: + case ISL_AUX_STATE_COMPRESSED_CLEAR: + if (!hiz_supported || !fast_clear_supported) + hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE; + break; - intel_miptree_check_level_layer(mt, map->level, map->layer); + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + if (!hiz_supported) + hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE; + break; - assert(map->fast_clear_state != INTEL_FAST_CLEAR_STATE_RESOLVED); + case ISL_AUX_STATE_PASS_THROUGH: + case ISL_AUX_STATE_RESOLVED: + break; - brw_blorp_resolve_color(brw, mt, map->level, map->layer, resolve_op); - intel_resolve_map_remove(map); - resolved = true; + case ISL_AUX_STATE_AUX_INVALID: + if (hiz_supported) + hiz_op = BLORP_HIZ_OP_HIZ_RESOLVE; + break; } - return resolved; + if (hiz_op != BLORP_HIZ_OP_NONE) { + intel_hiz_exec(brw, mt, level, layer, 1, hiz_op); + + switch (hiz_op) { + case BLORP_HIZ_OP_DEPTH_RESOLVE: + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_RESOLVED); + break; + + case BLORP_HIZ_OP_HIZ_RESOLVE: + /* The HiZ resolve operation is actually an ambiguate */ + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_PASS_THROUGH); + break; + + default: + unreachable("Invalid HiZ op"); + } + } } -void -intel_miptree_all_slices_resolve_color(struct brw_context *brw, - struct intel_mipmap_tree *mt, - int flags) -{ +static void +intel_miptree_finish_hiz_write(struct brw_context *brw, + struct intel_mipmap_tree *mt, + uint32_t level, uint32_t layer, + bool written_with_hiz) +{ + switch (intel_miptree_get_aux_state(mt, level, layer)) { + case ISL_AUX_STATE_CLEAR: + assert(written_with_hiz); + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_CLEAR); + break; + + case ISL_AUX_STATE_COMPRESSED_NO_CLEAR: + case ISL_AUX_STATE_COMPRESSED_CLEAR: + assert(written_with_hiz); + break; /* Nothing to do */ + + case ISL_AUX_STATE_RESOLVED: + if (written_with_hiz) { + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_NO_CLEAR); + } else { + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_AUX_INVALID); + } + break; + + case ISL_AUX_STATE_PASS_THROUGH: + if (written_with_hiz) { + intel_miptree_set_aux_state(brw, mt, level, layer, 1, + ISL_AUX_STATE_COMPRESSED_NO_CLEAR); + } + break; - intel_miptree_resolve_color(brw, mt, 0, UINT32_MAX, 0, UINT32_MAX, flags); + case ISL_AUX_STATE_AUX_INVALID: + assert(!written_with_hiz); + break; + } } static inline uint32_t @@ -2241,7 +2406,14 @@ miptree_layer_range_length(const struct intel_mipmap_tree *mt, uint32_t level, uint32_t start_layer, uint32_t num_layers) { assert(level <= mt->last_level); - uint32_t total_num_layers = mt->level[level].depth; + uint32_t total_num_layers; + + if (mt->surf.size > 0) + total_num_layers = mt->surf.dim == ISL_SURF_DIM_3D ? + minify(mt->surf.phys_level0_sa.depth, level) : + mt->surf.phys_level0_sa.array_len; + else + total_num_layers = mt->level[level].depth; assert(start_layer < total_num_layers); if (num_layers == INTEL_REMAINING_LAYERS) @@ -2268,11 +2440,17 @@ intel_miptree_prepare_access(struct brw_context *brw, if (mt->num_samples > 1) { /* Nothing to do for MSAA */ + assert(aux_supported && fast_clear_supported); } else { - /* TODO: This is fairly terrible. We can do better. */ - if (!aux_supported || !fast_clear_supported) { - intel_miptree_resolve_color(brw, mt, start_level, num_levels, - start_layer, num_layers, 0); + for (uint32_t l = 0; l < num_levels; l++) { + const uint32_t level = start_level + l; + const uint32_t level_layers = + miptree_layer_range_length(mt, level, start_layer, num_layers); + for (uint32_t a = 0; a < level_layers; a++) { + intel_miptree_prepare_ccs_access(brw, mt, level, + start_layer + a, aux_supported, + fast_clear_supported); + } } } } else if (mt->format == MESA_FORMAT_S_UINT8) { @@ -2281,16 +2459,18 @@ intel_miptree_prepare_access(struct brw_context *brw, if (!mt->hiz_buf) return; - if (aux_supported) { - assert(fast_clear_supported); - intel_miptree_depth_hiz_resolve(brw, mt, start_level, num_levels, - start_layer, num_layers, - BLORP_HIZ_OP_HIZ_RESOLVE); - } else { - assert(!fast_clear_supported); - intel_miptree_depth_hiz_resolve(brw, mt, start_level, num_levels, - start_layer, num_layers, - BLORP_HIZ_OP_DEPTH_RESOLVE); + for (uint32_t l = 0; l < num_levels; l++) { + const uint32_t level = start_level + l; + if (!intel_miptree_level_has_hiz(mt, level)) + continue; + + const uint32_t level_layers = + miptree_layer_range_length(mt, level, start_layer, num_layers); + for (uint32_t a = 0; a < level_layers; a++) { + intel_miptree_prepare_hiz_access(brw, mt, level, start_layer + a, + aux_supported, + fast_clear_supported); + } } } } @@ -2304,29 +2484,29 @@ intel_miptree_finish_write(struct brw_context *brw, num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers); if (_mesa_is_format_color_format(mt->format)) { + if (!mt->mcs_buf) + return; + if (mt->num_samples > 1) { - /* Nothing to do for MSAA */ + for (uint32_t a = 0; a < num_layers; a++) { + intel_miptree_finish_mcs_write(brw, mt, level, start_layer + a, + written_with_aux); + } } else { - if (written_with_aux) { - intel_miptree_used_for_rendering(brw, mt, level, - start_layer, num_layers); + for (uint32_t a = 0; a < num_layers; a++) { + intel_miptree_finish_ccs_write(brw, mt, level, start_layer + a, + written_with_aux); } } } else if (mt->format == MESA_FORMAT_S_UINT8) { /* Nothing to do for stencil */ } else { - if (written_with_aux) { - for (unsigned a = 0; a < num_layers; a++) { - intel_miptree_check_level_layer(mt, level, start_layer); - intel_miptree_slice_set_needs_depth_resolve(mt, level, - start_layer + a); - } - } else { - for (unsigned a = 0; a < num_layers; a++) { - intel_miptree_check_level_layer(mt, level, start_layer); - intel_miptree_slice_set_needs_hiz_resolve(mt, level, - start_layer + a); - } + if (!intel_miptree_level_has_hiz(mt, level)) + return; + + for (uint32_t a = 0; a < num_layers; a++) { + intel_miptree_finish_hiz_write(brw, mt, level, start_layer + a, + written_with_aux); } } } @@ -2335,39 +2515,18 @@ enum isl_aux_state intel_miptree_get_aux_state(const struct intel_mipmap_tree *mt, uint32_t level, uint32_t layer) { + intel_miptree_check_level_layer(mt, level, layer); + if (_mesa_is_format_color_format(mt->format)) { assert(mt->mcs_buf != NULL); - if (mt->num_samples > 1) { - return ISL_AUX_STATE_COMPRESSED_CLEAR; - } else { - switch (intel_miptree_get_fast_clear_state(mt, level, layer)) { - case INTEL_FAST_CLEAR_STATE_RESOLVED: - return ISL_AUX_STATE_RESOLVED; - case INTEL_FAST_CLEAR_STATE_UNRESOLVED: - return ISL_AUX_STATE_COMPRESSED_CLEAR; - case INTEL_FAST_CLEAR_STATE_CLEAR: - return ISL_AUX_STATE_CLEAR; - default: - unreachable("Invalid fast clear state"); - } - } + assert(mt->num_samples <= 1 || mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS); } else if (mt->format == MESA_FORMAT_S_UINT8) { unreachable("Cannot get aux state for stencil"); } else { - assert(mt->hiz_buf != NULL); - const struct intel_resolve_map *map = - intel_resolve_map_const_get(&mt->hiz_map, level, layer); - if (!map) - return ISL_AUX_STATE_RESOLVED; - switch (map->need) { - case BLORP_HIZ_OP_DEPTH_RESOLVE: - return ISL_AUX_STATE_COMPRESSED_CLEAR; - case BLORP_HIZ_OP_HIZ_RESOLVE: - return ISL_AUX_STATE_AUX_INVALID; - default: - unreachable("Invalid hiz op"); - } + assert(intel_miptree_level_has_hiz(mt, level)); } + + return mt->aux_state[level][layer]; } void @@ -2378,24 +2537,169 @@ intel_miptree_set_aux_state(struct brw_context *brw, { num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers); - /* Right now, this only applies to clears. */ - assert(aux_state == ISL_AUX_STATE_CLEAR); + if (_mesa_is_format_color_format(mt->format)) { + assert(mt->mcs_buf != NULL); + assert(mt->num_samples <= 1 || mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS); + } else if (mt->format == MESA_FORMAT_S_UINT8) { + unreachable("Cannot get aux state for stencil"); + } else { + assert(intel_miptree_level_has_hiz(mt, level)); + } + + for (unsigned a = 0; a < num_layers; a++) + mt->aux_state[level][start_layer + a] = aux_state; +} + +/* On Gen9 color buffers may be compressed by the hardware (lossless + * compression). There are, however, format restrictions and care needs to be + * taken that the sampler engine is capable for re-interpreting a buffer with + * format different the buffer was originally written with. + * + * For example, SRGB formats are not compressible and the sampler engine isn't + * capable of treating RGBA_UNORM as SRGB_ALPHA. In such a case the underlying + * color buffer needs to be resolved so that the sampling surface can be + * sampled as non-compressed (i.e., without the auxiliary MCS buffer being + * set). + */ +static bool +can_texture_with_ccs(struct brw_context *brw, + struct intel_mipmap_tree *mt, + mesa_format view_format) +{ + if (!intel_miptree_is_lossless_compressed(brw, mt)) + return false; + + enum isl_format isl_mt_format = brw_isl_format_for_mesa_format(mt->format); + enum isl_format isl_view_format = brw_isl_format_for_mesa_format(view_format); + + if (!isl_formats_are_ccs_e_compatible(&brw->screen->devinfo, + isl_mt_format, isl_view_format)) { + perf_debug("Incompatible sampling format (%s) for rbc (%s)\n", + _mesa_get_format_name(view_format), + _mesa_get_format_name(mt->format)); + return false; + } + + return true; +} +static void +intel_miptree_prepare_texture_slices(struct brw_context *brw, + struct intel_mipmap_tree *mt, + mesa_format view_format, + uint32_t start_level, uint32_t num_levels, + uint32_t start_layer, uint32_t num_layers, + bool *aux_supported_out) +{ + bool aux_supported, clear_supported; if (_mesa_is_format_color_format(mt->format)) { - if (mt->num_samples > 1) - assert(mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS); + if (mt->num_samples > 1) { + aux_supported = clear_supported = true; + } else { + aux_supported = can_texture_with_ccs(brw, mt, view_format); - assert(level == 0 && start_layer == 0 && num_layers == 1); - intel_miptree_set_fast_clear_state(brw, mt, 0, 0, 1, - INTEL_FAST_CLEAR_STATE_CLEAR); + /* Clear color is specified as ints or floats and the conversion is + * done by the sampler. If we have a texture view, we would have to + * perform the clear color conversion manually. Just disable clear + * color. + */ + clear_supported = aux_supported && (mt->format == view_format); + } } else if (mt->format == MESA_FORMAT_S_UINT8) { - assert(!"Cannot set aux state for stencil"); + aux_supported = clear_supported = false; } else { - for (unsigned a = 0; a < num_layers; a++) { - intel_miptree_check_level_layer(mt, level, start_layer); - intel_miptree_slice_set_needs_depth_resolve(mt, level, - start_layer + a); - } + aux_supported = clear_supported = intel_miptree_sample_with_hiz(brw, mt); + } + + intel_miptree_prepare_access(brw, mt, start_level, num_levels, + start_layer, num_layers, + aux_supported, clear_supported); + if (aux_supported_out) + *aux_supported_out = aux_supported; +} + +void +intel_miptree_prepare_texture(struct brw_context *brw, + struct intel_mipmap_tree *mt, + mesa_format view_format, + bool *aux_supported_out) +{ + intel_miptree_prepare_texture_slices(brw, mt, view_format, + 0, INTEL_REMAINING_LEVELS, + 0, INTEL_REMAINING_LAYERS, + aux_supported_out); +} + +void +intel_miptree_prepare_image(struct brw_context *brw, + struct intel_mipmap_tree *mt) +{ + /* The data port doesn't understand any compression */ + intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS, + 0, INTEL_REMAINING_LAYERS, false, false); +} + +void +intel_miptree_prepare_fb_fetch(struct brw_context *brw, + struct intel_mipmap_tree *mt, uint32_t level, + uint32_t start_layer, uint32_t num_layers) +{ + intel_miptree_prepare_texture_slices(brw, mt, mt->format, level, 1, + start_layer, num_layers, NULL); +} + +void +intel_miptree_prepare_render(struct brw_context *brw, + struct intel_mipmap_tree *mt, uint32_t level, + uint32_t start_layer, uint32_t layer_count, + bool srgb_enabled) +{ + /* If FRAMEBUFFER_SRGB is used on Gen9+ then we need to resolve any of + * the single-sampled color renderbuffers because the CCS buffer isn't + * supported for SRGB formats. This only matters if FRAMEBUFFER_SRGB is + * enabled because otherwise the surface state will be programmed with + * the linear equivalent format anyway. + */ + if (brw->gen == 9 && srgb_enabled && mt->num_samples <= 1 && + _mesa_get_srgb_format_linear(mt->format) != mt->format) { + + /* Lossless compression is not supported for SRGB formats, it + * should be impossible to get here with such surfaces. + */ + assert(!intel_miptree_is_lossless_compressed(brw, mt)); + intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count, + false, false); + } +} + +void +intel_miptree_finish_render(struct brw_context *brw, + struct intel_mipmap_tree *mt, uint32_t level, + uint32_t start_layer, uint32_t layer_count) +{ + assert(_mesa_is_format_color_format(mt->format)); + intel_miptree_finish_write(brw, mt, level, start_layer, layer_count, + mt->mcs_buf != NULL); +} + +void +intel_miptree_prepare_depth(struct brw_context *brw, + struct intel_mipmap_tree *mt, uint32_t level, + uint32_t start_layer, uint32_t layer_count) +{ + intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count, + mt->hiz_buf != NULL, mt->hiz_buf != NULL); +} + +void +intel_miptree_finish_depth(struct brw_context *brw, + struct intel_mipmap_tree *mt, uint32_t level, + uint32_t start_layer, uint32_t layer_count, + bool depth_written) +{ + if (depth_written) { + intel_miptree_finish_write(brw, mt, level, start_layer, layer_count, + mt->hiz_buf != NULL); } } @@ -2424,7 +2728,6 @@ intel_miptree_make_shareable(struct brw_context *brw, 0, INTEL_REMAINING_LAYERS, false, false); if (mt->mcs_buf) { - mt->aux_disable |= (INTEL_AUX_DISABLE_CCS | INTEL_AUX_DISABLE_MCS); brw_bo_unreference(mt->mcs_buf->bo); free(mt->mcs_buf); mt->mcs_buf = NULL; @@ -2433,12 +2736,12 @@ intel_miptree_make_shareable(struct brw_context *brw, * execute any will likely crash due to the missing aux buffer. So let's * delete all pending ops. */ - exec_list_make_empty(&mt->color_resolve_map); + free(mt->aux_state); + mt->aux_state = NULL; } if (mt->hiz_buf) { - mt->aux_disable |= INTEL_AUX_DISABLE_HIZ; - intel_miptree_hiz_buffer_free(mt->hiz_buf); + intel_miptree_aux_buffer_free(mt->hiz_buf); mt->hiz_buf = NULL; for (uint32_t l = mt->first_level; l <= mt->last_level; ++l) { @@ -2449,8 +2752,11 @@ intel_miptree_make_shareable(struct brw_context *brw, * any will likely crash due to the missing aux buffer. So let's delete * all pending ops. */ - exec_list_make_empty(&mt->hiz_map); + free(mt->aux_state); + mt->aux_state = NULL; } + + mt->aux_usage = ISL_AUX_USAGE_NONE; } @@ -2515,27 +2821,57 @@ intel_miptree_updownsample(struct brw_context *brw, struct intel_mipmap_tree *src, struct intel_mipmap_tree *dst) { + unsigned src_w, src_h, dst_w, dst_h; + + if (src->surf.size > 0) { + src_w = src->surf.logical_level0_px.width; + src_h = src->surf.logical_level0_px.height; + } else { + src_w = src->logical_width0; + src_h = src->logical_height0; + } + + if (dst->surf.size > 0) { + dst_w = dst->surf.logical_level0_px.width; + dst_h = dst->surf.logical_level0_px.height; + } else { + dst_w = dst->logical_width0; + dst_h = dst->logical_height0; + } + brw_blorp_blit_miptrees(brw, src, 0 /* level */, 0 /* layer */, src->format, SWIZZLE_XYZW, dst, 0 /* level */, 0 /* layer */, dst->format, - 0, 0, - src->logical_width0, src->logical_height0, - 0, 0, - dst->logical_width0, dst->logical_height0, + 0, 0, src_w, src_h, + 0, 0, dst_w, dst_h, GL_NEAREST, false, false /*mirror x, y*/, false, false); if (src->stencil_mt) { + if (src->stencil_mt->surf.size > 0) { + src_w = src->stencil_mt->surf.logical_level0_px.width; + src_h = src->stencil_mt->surf.logical_level0_px.height; + } else { + src_w = src->stencil_mt->logical_width0; + src_h = src->stencil_mt->logical_height0; + } + + if (dst->stencil_mt->surf.size > 0) { + dst_w = dst->stencil_mt->surf.logical_level0_px.width; + dst_h = dst->stencil_mt->surf.logical_level0_px.height; + } else { + dst_w = dst->stencil_mt->logical_width0; + dst_h = dst->stencil_mt->logical_height0; + } + brw_blorp_blit_miptrees(brw, src->stencil_mt, 0 /* level */, 0 /* layer */, src->stencil_mt->format, SWIZZLE_XYZW, dst->stencil_mt, 0 /* level */, 0 /* layer */, dst->stencil_mt->format, - 0, 0, - src->logical_width0, src->logical_height0, - 0, 0, - dst->logical_width0, dst->logical_height0, + 0, 0, src_w, src_h, + 0, 0, dst_w, dst_h, GL_NEAREST, false, false /*mirror x, y*/, false, false /* decode/encode srgb */); } @@ -2629,11 +2965,13 @@ intel_miptree_map_gtt(struct brw_context *brw, y /= bh; x /= bw; - base = intel_miptree_map_raw(brw, mt, map->mode) + mt->offset; + base = intel_miptree_map_raw(brw, mt, map->mode); if (base == NULL) map->ptr = NULL; else { + base += mt->offset; + /* Note that in the case of cube maps, the caller must have passed the * slice number referencing the face. */ @@ -2822,6 +3160,12 @@ intel_miptree_map_s8(struct brw_context *brw, * temporary buffer back out. */ if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { + /* ISL uses a stencil pitch value that is expected by hardware whereas + * traditional miptree uses half of that. Below the value gets supplied + * to intel_offset_S8() which expects the legacy interpretation. + */ + const unsigned pitch = mt->surf.size > 0 ? + mt->surf.row_pitch / 2 : mt->pitch; uint8_t *untiled_s8_map = map->ptr; uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_READ_BIT); unsigned int image_x, image_y; @@ -2830,7 +3174,7 @@ intel_miptree_map_s8(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t offset = intel_offset_S8(mt->pitch, + ptrdiff_t offset = intel_offset_S8(pitch, x + image_x + map->x, y + image_y + map->y, brw->has_swizzling); @@ -2858,6 +3202,12 @@ intel_miptree_unmap_s8(struct brw_context *brw, unsigned int slice) { if (map->mode & GL_MAP_WRITE_BIT) { + /* ISL uses a stencil pitch value that is expected by hardware whereas + * traditional miptree uses half of that. Below the value gets supplied + * to intel_offset_S8() which expects the legacy interpretation. + */ + const unsigned pitch = mt->surf.size > 0 ? + mt->surf.row_pitch / 2: mt->pitch; unsigned int image_x, image_y; uint8_t *untiled_s8_map = map->ptr; uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT); @@ -2866,7 +3216,7 @@ intel_miptree_unmap_s8(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t offset = intel_offset_S8(mt->pitch, + ptrdiff_t offset = intel_offset_S8(pitch, image_x + x + map->x, image_y + y + map->y, brw->has_swizzling); @@ -2965,6 +3315,12 @@ intel_miptree_map_depthstencil(struct brw_context *brw, * temporary buffer back out. */ if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { + /* ISL uses a stencil pitch value that is expected by hardware whereas + * traditional miptree uses half of that. Below the value gets supplied + * to intel_offset_S8() which expects the legacy interpretation. + */ + const unsigned s_pitch = s_mt->surf.size > 0 ? + s_mt->surf.row_pitch / 2 : s_mt->pitch; uint32_t *packed_map = map->ptr; uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_READ_BIT); uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_READ_BIT); @@ -2979,7 +3335,7 @@ intel_miptree_map_depthstencil(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { int map_x = map->x + x, map_y = map->y + y; - ptrdiff_t s_offset = intel_offset_S8(s_mt->pitch, + ptrdiff_t s_offset = intel_offset_S8(s_pitch, map_x + s_image_x, map_y + s_image_y, brw->has_swizzling); @@ -3026,6 +3382,12 @@ intel_miptree_unmap_depthstencil(struct brw_context *brw, bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32; if (map->mode & GL_MAP_WRITE_BIT) { + /* ISL uses a stencil pitch value that is expected by hardware whereas + * traditional miptree uses half of that. Below the value gets supplied + * to intel_offset_S8() which expects the legacy interpretation. + */ + const unsigned s_pitch = s_mt->surf.size > 0 ? + s_mt->surf.row_pitch / 2 : s_mt->pitch; uint32_t *packed_map = map->ptr; uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_WRITE_BIT); uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_WRITE_BIT); @@ -3039,7 +3401,7 @@ intel_miptree_unmap_depthstencil(struct brw_context *brw, for (uint32_t y = 0; y < map->h; y++) { for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t s_offset = intel_offset_S8(s_mt->pitch, + ptrdiff_t s_offset = intel_offset_S8(s_pitch, x + s_image_x + map->x, y + s_image_y + map->y, brw->has_swizzling); @@ -3469,7 +3831,7 @@ intel_miptree_get_isl_surf(struct brw_context *brw, break; default: surf->usage = ISL_SURF_USAGE_TEXTURE_BIT; - if (brw->format_supported_as_render_target[mt->format]) + if (brw->mesa_format_supports_render[mt->format]) surf->usage = ISL_SURF_USAGE_RENDER_TARGET_BIT; break; } @@ -3478,94 +3840,15 @@ intel_miptree_get_isl_surf(struct brw_context *brw, surf->usage |= ISL_SURF_USAGE_CUBE_BIT; } -/* WARNING: THE SURFACE CREATED BY THIS FUNCTION IS NOT COMPLETE AND CANNOT BE - * USED FOR ANY REAL CALCULATIONS. THE ONLY VALID USE OF SUCH A SURFACE IS TO - * PASS IT INTO isl_surf_fill_state. - */ -void -intel_miptree_get_aux_isl_surf(struct brw_context *brw, - const struct intel_mipmap_tree *mt, - struct isl_surf *surf, - enum isl_aux_usage *usage) +enum isl_aux_usage +intel_miptree_get_aux_isl_usage(const struct brw_context *brw, + const struct intel_mipmap_tree *mt) { - uint32_t aux_pitch, aux_qpitch; - if (mt->mcs_buf) { - aux_pitch = mt->mcs_buf->pitch; - aux_qpitch = mt->mcs_buf->qpitch; - - if (mt->num_samples > 1) { - assert(mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS); - *usage = ISL_AUX_USAGE_MCS; - } else if (intel_miptree_is_lossless_compressed(brw, mt)) { - assert(brw->gen >= 9); - *usage = ISL_AUX_USAGE_CCS_E; - } else if ((mt->aux_disable & INTEL_AUX_DISABLE_CCS) == 0) { - *usage = ISL_AUX_USAGE_CCS_D; - } else { - unreachable("Invalid MCS miptree"); - } - } else if (mt->hiz_buf) { - aux_pitch = mt->hiz_buf->aux_base.pitch; - aux_qpitch = mt->hiz_buf->aux_base.qpitch; - - *usage = ISL_AUX_USAGE_HIZ; - } else { - *usage = ISL_AUX_USAGE_NONE; - return; - } - - /* Start with a copy of the original surface. */ - intel_miptree_get_isl_surf(brw, mt, surf); - - /* Figure out the format and tiling of the auxiliary surface */ - switch (*usage) { - case ISL_AUX_USAGE_NONE: - unreachable("Invalid auxiliary usage"); - - case ISL_AUX_USAGE_HIZ: - isl_surf_get_hiz_surf(&brw->isl_dev, surf, surf); - break; + if (mt->hiz_buf) + return ISL_AUX_USAGE_HIZ; - case ISL_AUX_USAGE_MCS: - /* - * From the SKL PRM: - * "When Auxiliary Surface Mode is set to AUX_CCS_D or AUX_CCS_E, - * HALIGN 16 must be used." - */ - if (brw->gen >= 9) - assert(mt->halign == 16); - - isl_surf_get_mcs_surf(&brw->isl_dev, surf, surf); - break; - - case ISL_AUX_USAGE_CCS_D: - case ISL_AUX_USAGE_CCS_E: - /* - * From the BDW PRM, Volume 2d, page 260 (RENDER_SURFACE_STATE): - * - * "When MCS is enabled for non-MSRT, HALIGN_16 must be used" - * - * From the hardware spec for GEN9: - * - * "When Auxiliary Surface Mode is set to AUX_CCS_D or AUX_CCS_E, - * HALIGN 16 must be used." - */ - assert(mt->num_samples <= 1); - if (brw->gen >= 8) - assert(mt->halign == 16); - - isl_surf_get_ccs_surf(&brw->isl_dev, surf, surf); - break; - } - - /* We want the pitch of the actual aux buffer. */ - surf->row_pitch = aux_pitch; + if (!mt->mcs_buf) + return ISL_AUX_USAGE_NONE; - /* Auxiliary surfaces in ISL have compressed formats and array_pitch_el_rows - * is in elements. This doesn't match intel_mipmap_tree::qpitch which is - * in elements of the primary color surface so we have to divide by the - * compression block height. - */ - surf->array_pitch_el_rows = - aux_qpitch / isl_format_get_layout(surf->format)->bh; + return mt->aux_usage; }