X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fcommon%2Fac_surface.c;h=461fd503622fa974062247440e43052556a2d9a7;hb=64b0b7c274c321b44e58ff0f48b444c3799c0833;hp=545ef4207567fa9e0f3509781fe93c23ddc750e0;hpb=69aadc493310bb7306d10559bf48412eb5865962;p=mesa.git diff --git a/src/amd/common/ac_surface.c b/src/amd/common/ac_surface.c index 545ef420756..461fd503622 100644 --- a/src/amd/common/ac_surface.c +++ b/src/amd/common/ac_surface.c @@ -29,15 +29,19 @@ #include "amd_family.h" #include "addrlib/src/amdgpu_asic_addr.h" #include "ac_gpu_info.h" +#include "util/hash_table.h" #include "util/macros.h" +#include "util/simple_mtx.h" #include "util/u_atomic.h" #include "util/u_math.h" +#include "util/u_memory.h" +#include "sid.h" #include #include #include #include -#include +#include "drm-uapi/amdgpu_drm.h" #include "addrlib/inc/addrinterface.h" @@ -49,6 +53,282 @@ #define CIASICIDGFXENGINE_ARCTICISLAND 0x0000000D #endif +struct ac_addrlib { + ADDR_HANDLE handle; + + /* The cache of DCC retile maps for reuse when allocating images of + * similar sizes. + */ + simple_mtx_t dcc_retile_map_lock; + struct hash_table *dcc_retile_maps; + struct hash_table *dcc_retile_tile_indices; +}; + +struct dcc_retile_map_key { + enum radeon_family family; + unsigned retile_width; + unsigned retile_height; + bool rb_aligned; + bool pipe_aligned; + unsigned dcc_retile_num_elements; + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT input; +}; + +static uint32_t dcc_retile_map_hash_key(const void *key) +{ + return _mesa_hash_data(key, sizeof(struct dcc_retile_map_key)); +} + +static bool dcc_retile_map_keys_equal(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(struct dcc_retile_map_key)) == 0; +} + +static void dcc_retile_map_free(struct hash_entry *entry) +{ + free((void*)entry->key); + free(entry->data); +} + +struct dcc_retile_tile_key { + enum radeon_family family; + unsigned bpp; + unsigned swizzle_mode; + bool rb_aligned; + bool pipe_aligned; +}; + +struct dcc_retile_tile_data { + unsigned tile_width_log2; + unsigned tile_height_log2; + uint16_t *data; +}; + +static uint32_t dcc_retile_tile_hash_key(const void *key) +{ + return _mesa_hash_data(key, sizeof(struct dcc_retile_tile_key)); +} + +static bool dcc_retile_tile_keys_equal(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(struct dcc_retile_tile_key)) == 0; +} + +static void dcc_retile_tile_free(struct hash_entry *entry) +{ + free((void*)entry->key); + free(((struct dcc_retile_tile_data*)entry->data)->data); + free(entry->data); +} + +/* Assumes dcc_retile_map_lock is taken. */ +static const struct dcc_retile_tile_data * +ac_compute_dcc_retile_tile_indices(struct ac_addrlib *addrlib, + const struct radeon_info *info, + unsigned bpp, unsigned swizzle_mode, + bool rb_aligned, bool pipe_aligned) +{ + struct dcc_retile_tile_key key = (struct dcc_retile_tile_key) { + .family = info->family, + .bpp = bpp, + .swizzle_mode = swizzle_mode, + .rb_aligned = rb_aligned, + .pipe_aligned = pipe_aligned + }; + + struct hash_entry *entry = _mesa_hash_table_search(addrlib->dcc_retile_tile_indices, &key); + if (entry) + return entry->data; + + ADDR2_COMPUTE_DCCINFO_INPUT din = {0}; + ADDR2_COMPUTE_DCCINFO_OUTPUT dout = {0}; + din.size = sizeof(ADDR2_COMPUTE_DCCINFO_INPUT); + dout.size = sizeof(ADDR2_COMPUTE_DCCINFO_OUTPUT); + + din.dccKeyFlags.pipeAligned = pipe_aligned; + din.dccKeyFlags.rbAligned = rb_aligned; + din.resourceType = ADDR_RSRC_TEX_2D; + din.swizzleMode = swizzle_mode; + din.bpp = bpp; + din.unalignedWidth = 1; + din.unalignedHeight = 1; + din.numSlices = 1; + din.numFrags = 1; + din.numMipLevels = 1; + + ADDR_E_RETURNCODE ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); + if (ret != ADDR_OK) + return NULL; + + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin = {0}; + addrin.size = sizeof(addrin); + addrin.swizzleMode = swizzle_mode; + addrin.resourceType = ADDR_RSRC_TEX_2D; + addrin.bpp = bpp; + addrin.numSlices = 1; + addrin.numMipLevels = 1; + addrin.numFrags = 1; + addrin.pitch = dout.pitch; + addrin.height = dout.height; + addrin.compressBlkWidth = dout.compressBlkWidth; + addrin.compressBlkHeight = dout.compressBlkHeight; + addrin.compressBlkDepth = dout.compressBlkDepth; + addrin.metaBlkWidth = dout.metaBlkWidth; + addrin.metaBlkHeight = dout.metaBlkHeight; + addrin.metaBlkDepth = dout.metaBlkDepth; + addrin.dccKeyFlags.pipeAligned = pipe_aligned; + addrin.dccKeyFlags.rbAligned = rb_aligned; + + unsigned w = dout.metaBlkWidth / dout.compressBlkWidth; + unsigned h = dout.metaBlkHeight / dout.compressBlkHeight; + uint16_t *indices = malloc(w * h * sizeof (uint16_t)); + if (!indices) + return NULL; + + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_OUTPUT addrout = {}; + addrout.size = sizeof(addrout); + + for (unsigned y = 0; y < h; ++y) { + addrin.y = y * dout.compressBlkHeight; + for (unsigned x = 0; x < w; ++x) { + addrin.x = x * dout.compressBlkWidth; + addrout.addr = 0; + + if (Addr2ComputeDccAddrFromCoord(addrlib->handle, &addrin, &addrout) != ADDR_OK) { + free(indices); + return NULL; + } + indices[y * w + x] = addrout.addr; + } + } + + struct dcc_retile_tile_data *data = calloc(1, sizeof(*data)); + if (!data) { + free(indices); + return NULL; + } + + data->tile_width_log2 = util_logbase2(w); + data->tile_height_log2 = util_logbase2(h); + data->data = indices; + + struct dcc_retile_tile_key *heap_key = mem_dup(&key, sizeof(key)); + if (!heap_key) { + free(data); + free(indices); + return NULL; + } + + entry = _mesa_hash_table_insert(addrlib->dcc_retile_tile_indices, heap_key, data); + if (!entry) { + free(heap_key); + free(data); + free(indices); + } + return data; +} + +static uint32_t ac_compute_retile_tile_addr(const struct dcc_retile_tile_data *tile, + unsigned stride, unsigned x, unsigned y) +{ + unsigned x_mask = (1u << tile->tile_width_log2) - 1; + unsigned y_mask = (1u << tile->tile_height_log2) - 1; + unsigned tile_size_log2 = tile->tile_width_log2 + tile->tile_height_log2; + + unsigned base = ((y >> tile->tile_height_log2) * stride + (x >> tile->tile_width_log2)) << tile_size_log2; + unsigned offset_in_tile = tile->data[((y & y_mask) << tile->tile_width_log2) + (x & x_mask)]; + return base + offset_in_tile; +} + +static uint32_t *ac_compute_dcc_retile_map(struct ac_addrlib *addrlib, + const struct radeon_info *info, + unsigned retile_width, unsigned retile_height, + bool rb_aligned, bool pipe_aligned, bool use_uint16, + unsigned dcc_retile_num_elements, + const ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT *in) +{ + unsigned dcc_retile_map_size = dcc_retile_num_elements * (use_uint16 ? 2 : 4); + struct dcc_retile_map_key key; + + assert(in->numFrags == 1 && in->numSlices == 1 && in->numMipLevels == 1); + + memset(&key, 0, sizeof(key)); + key.family = info->family; + key.retile_width = retile_width; + key.retile_height = retile_height; + key.rb_aligned = rb_aligned; + key.pipe_aligned = pipe_aligned; + key.dcc_retile_num_elements = dcc_retile_num_elements; + memcpy(&key.input, in, sizeof(*in)); + + simple_mtx_lock(&addrlib->dcc_retile_map_lock); + + /* If we have already computed this retile map, get it from the hash table. */ + struct hash_entry *entry = _mesa_hash_table_search(addrlib->dcc_retile_maps, &key); + if (entry) { + uint32_t *map = entry->data; + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return map; + } + + const struct dcc_retile_tile_data *src_tile = + ac_compute_dcc_retile_tile_indices(addrlib, info, in->bpp, + in->swizzleMode, + rb_aligned, pipe_aligned); + const struct dcc_retile_tile_data *dst_tile = + ac_compute_dcc_retile_tile_indices(addrlib, info, in->bpp, + in->swizzleMode, false, false); + if (!src_tile || !dst_tile) { + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return NULL; + } + + void *dcc_retile_map = malloc(dcc_retile_map_size); + if (!dcc_retile_map) { + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return NULL; + } + + unsigned index = 0; + unsigned w = DIV_ROUND_UP(retile_width, in->compressBlkWidth); + unsigned h = DIV_ROUND_UP(retile_height, in->compressBlkHeight); + unsigned src_stride = DIV_ROUND_UP(w, 1u << src_tile->tile_width_log2); + unsigned dst_stride = DIV_ROUND_UP(w, 1u << dst_tile->tile_width_log2); + + for (unsigned y = 0; y < h; ++y) { + for (unsigned x = 0; x < w; ++x) { + unsigned src_addr = ac_compute_retile_tile_addr(src_tile, src_stride, x, y); + unsigned dst_addr = ac_compute_retile_tile_addr(dst_tile, dst_stride, x, y); + + if (use_uint16) { + ((uint16_t*)dcc_retile_map)[2 * index] = src_addr; + ((uint16_t*)dcc_retile_map)[2 * index + 1] = dst_addr; + } else { + ((uint32_t*)dcc_retile_map)[2 * index] = src_addr; + ((uint32_t*)dcc_retile_map)[2 * index + 1] = dst_addr; + } + ++index; + } + } + + /* Fill the remaining pairs with the last one (for the compute shader). */ + for (unsigned i = index * 2; i < dcc_retile_num_elements; i++) { + if (use_uint16) + ((uint16_t*)dcc_retile_map)[i] = ((uint16_t*)dcc_retile_map)[i - 2]; + else + ((uint32_t*)dcc_retile_map)[i] = ((uint32_t*)dcc_retile_map)[i - 2]; + } + + /* Insert the retile map into the hash table, so that it can be reused and + * the computation can be skipped for similar image sizes. + */ + _mesa_hash_table_insert(addrlib->dcc_retile_maps, + mem_dup(&key, sizeof(key)), dcc_retile_map); + + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return dcc_retile_map; +} + static void *ADDR_API allocSysMem(const ADDR_ALLOCSYSMEM_INPUT * pInput) { return malloc(pInput->sizeInBytes); @@ -60,9 +340,9 @@ static ADDR_E_RETURNCODE ADDR_API freeSysMem(const ADDR_FREESYSMEM_INPUT * pInpu return ADDR_OK; } -ADDR_HANDLE amdgpu_addr_create(const struct radeon_info *info, - const struct amdgpu_gpu_info *amdinfo, - uint64_t *max_alignment) +struct ac_addrlib *ac_addrlib_create(const struct radeon_info *info, + const struct amdgpu_gpu_info *amdinfo, + uint64_t *max_alignment) { ADDR_CREATE_INPUT addrCreateInput = {0}; ADDR_CREATE_OUTPUT addrCreateOutput = {0}; @@ -122,7 +402,29 @@ ADDR_HANDLE amdgpu_addr_create(const struct radeon_info *info, *max_alignment = addrGetMaxAlignmentsOutput.baseAlign; } } - return addrCreateOutput.hLib; + + struct ac_addrlib *addrlib = calloc(1, sizeof(struct ac_addrlib)); + if (!addrlib) { + AddrDestroy(addrCreateOutput.hLib); + return NULL; + } + + addrlib->handle = addrCreateOutput.hLib; + simple_mtx_init(&addrlib->dcc_retile_map_lock, mtx_plain); + addrlib->dcc_retile_maps = _mesa_hash_table_create(NULL, dcc_retile_map_hash_key, + dcc_retile_map_keys_equal); + addrlib->dcc_retile_tile_indices = _mesa_hash_table_create(NULL, dcc_retile_tile_hash_key, + dcc_retile_tile_keys_equal); + return addrlib; +} + +void ac_addrlib_destroy(struct ac_addrlib *addrlib) +{ + AddrDestroy(addrlib->handle); + simple_mtx_destroy(&addrlib->dcc_retile_map_lock); + _mesa_hash_table_destroy(addrlib->dcc_retile_maps, dcc_retile_map_free); + _mesa_hash_table_destroy(addrlib->dcc_retile_tile_indices, dcc_retile_tile_free); + free(addrlib); } static int surf_config_sanity(const struct ac_surf_config *config, @@ -342,6 +644,13 @@ static int gfx6_compute_level(ADDR_HANDLE addrlib, else surf_level->dcc_slice_fast_clear_size = 0; } + + if (surf->flags & RADEON_SURF_CONTIGUOUS_DCC_LAYERS && + surf->dcc_slice_size != surf_level->dcc_slice_fast_clear_size) { + surf->dcc_size = 0; + surf->num_dcc_levels = 0; + AddrDccOut->subLvlCompressible = false; + } } else { surf_level->dcc_slice_fast_clear_size = surf_level->dcc_fast_clear_size; } @@ -378,10 +687,6 @@ static int gfx6_compute_level(ADDR_HANDLE addrlib, return 0; } -#define G_009910_MICRO_TILE_MODE(x) (((x) >> 0) & 0x03) -#define V_009910_ADDR_SURF_THICK_MICRO_TILING 0x03 -#define G_009910_MICRO_TILE_MODE_NEW(x) (((x) >> 22) & 0x07) - static void gfx6_set_micro_tile_mode(struct radeon_surf *surf, const struct radeon_info *info) { @@ -413,7 +718,9 @@ static bool get_display_flag(const struct ac_surf_config *config, unsigned num_channels = config->info.num_channels; unsigned bpe = surf->bpe; - if (!(surf->flags & RADEON_SURF_Z_OR_SBUFFER) && + if (!config->is_3d && + !config->is_cube && + !(surf->flags & RADEON_SURF_Z_OR_SBUFFER) && surf->flags & RADEON_SURF_SCANOUT && config->info.samples <= 1 && surf->blk_w <= 2 && surf->blk_h == 1) { @@ -499,7 +806,7 @@ static void ac_compute_cmask(const struct radeon_info *info, unsigned num_pipes = info->num_tile_pipes; unsigned cl_width, cl_height; - if (surf->flags & RADEON_SURF_Z_OR_SBUFFER || + if (surf->flags & RADEON_SURF_Z_OR_SBUFFER || surf->is_linear || (config->info.samples >= 2 && !surf->fmask_size)) return; @@ -660,7 +967,7 @@ static int gfx6_compute_surface(ADDR_HANDLE addrlib, AddrSurfInfoIn.flags.opt4Space = !AddrSurfInfoIn.flags.tcCompatible && !AddrSurfInfoIn.flags.fmask && config->info.samples <= 1 && - (surf->flags & RADEON_SURF_OPTIMIZE_FOR_SPACE); + !(surf->flags & RADEON_SURF_FORCE_SWIZZLE_MODE); /* DCC notes: * - If we add MSAA support, keep in mind that CB can't decompress 8bpp @@ -947,19 +1254,22 @@ static int gfx6_compute_surface(ADDR_HANDLE addrlib, surf->htile_size = (total_pixels / htile_block_size) * htile_element_size; surf->htile_size = align(surf->htile_size, surf->htile_alignment); + } else if (!surf->htile_size) { + /* Unset this if HTILE is not present. */ + surf->flags &= ~RADEON_SURF_TC_COMPATIBLE_HTILE; } surf->is_linear = surf->u.legacy.level[0].mode == RADEON_SURF_MODE_LINEAR_ALIGNED; surf->is_displayable = surf->is_linear || surf->micro_tile_mode == RADEON_MICRO_MODE_DISPLAY || - surf->micro_tile_mode == RADEON_MICRO_MODE_ROTATED; + surf->micro_tile_mode == RADEON_MICRO_MODE_RENDER; /* The rotated micro tile mode doesn't work if both CMASK and RB+ are * used at the same time. This case is not currently expected to occur * because we don't use rotated. Enforce this restriction on all chips * to facilitate testing. */ - if (surf->micro_tile_mode == RADEON_MICRO_MODE_ROTATED) { + if (surf->micro_tile_mode == RADEON_MICRO_MODE_RENDER) { assert(!"rotate micro tile mode is unsupported"); return ADDR_ERROR; } @@ -1008,11 +1318,11 @@ gfx9_get_preferred_swizzle_mode(ADDR_HANDLE addrlib, if (surf->micro_tile_mode == RADEON_MICRO_MODE_DISPLAY) sin.preferredSwSet.sw_D = 1; - else if (surf->micro_tile_mode == RADEON_MICRO_MODE_THIN) + else if (surf->micro_tile_mode == RADEON_MICRO_MODE_STANDARD) sin.preferredSwSet.sw_S = 1; else if (surf->micro_tile_mode == RADEON_MICRO_MODE_DEPTH) sin.preferredSwSet.sw_Z = 1; - else if (surf->micro_tile_mode == RADEON_MICRO_MODE_ROTATED) + else if (surf->micro_tile_mode == RADEON_MICRO_MODE_RENDER) sin.preferredSwSet.sw_R = 1; } @@ -1024,7 +1334,7 @@ gfx9_get_preferred_swizzle_mode(ADDR_HANDLE addrlib, return 0; } -static bool gfx9_is_dcc_capable(const struct radeon_info *info, unsigned sw_mode) +static bool is_dcc_supported_by_CB(const struct radeon_info *info, unsigned sw_mode) { if (info->chip_class >= GFX10) return sw_mode == ADDR_SW_64KB_Z_X || sw_mode == ADDR_SW_64KB_R_X; @@ -1032,7 +1342,90 @@ static bool gfx9_is_dcc_capable(const struct radeon_info *info, unsigned sw_mode return sw_mode != ADDR_SW_LINEAR; } -static int gfx9_compute_miptree(ADDR_HANDLE addrlib, +ASSERTED static bool is_dcc_supported_by_L2(const struct radeon_info *info, + const struct radeon_surf *surf) +{ + if (info->chip_class <= GFX9) { + /* Only independent 64B blocks are supported. */ + return surf->u.gfx9.dcc.independent_64B_blocks && + !surf->u.gfx9.dcc.independent_128B_blocks && + surf->u.gfx9.dcc.max_compressed_block_size == V_028C78_MAX_BLOCK_SIZE_64B; + } + + if (info->family == CHIP_NAVI10) { + /* Only independent 128B blocks are supported. */ + return !surf->u.gfx9.dcc.independent_64B_blocks && + surf->u.gfx9.dcc.independent_128B_blocks && + surf->u.gfx9.dcc.max_compressed_block_size <= V_028C78_MAX_BLOCK_SIZE_128B; + } + + if (info->family == CHIP_NAVI12 || + info->family == CHIP_NAVI14) { + /* Either 64B or 128B can be used, but not both. + * If 64B is used, DCC image stores are unsupported. + */ + return surf->u.gfx9.dcc.independent_64B_blocks != + surf->u.gfx9.dcc.independent_128B_blocks && + (!surf->u.gfx9.dcc.independent_64B_blocks || + surf->u.gfx9.dcc.max_compressed_block_size == V_028C78_MAX_BLOCK_SIZE_64B) && + (!surf->u.gfx9.dcc.independent_128B_blocks || + surf->u.gfx9.dcc.max_compressed_block_size <= V_028C78_MAX_BLOCK_SIZE_128B); + } + + /* 128B is recommended, but 64B can be set too if needed for 4K by DCN. + * Since there is no reason to ever disable 128B, require it. + * DCC image stores are always supported. + */ + return surf->u.gfx9.dcc.independent_128B_blocks && + surf->u.gfx9.dcc.max_compressed_block_size <= V_028C78_MAX_BLOCK_SIZE_128B; +} + +static bool is_dcc_supported_by_DCN(const struct radeon_info *info, + const struct ac_surf_config *config, + const struct radeon_surf *surf, + bool rb_aligned, bool pipe_aligned) +{ + if (!info->use_display_dcc_unaligned && + !info->use_display_dcc_with_retile_blit) + return false; + + /* 16bpp and 64bpp are more complicated, so they are disallowed for now. */ + if (surf->bpe != 4) + return false; + + /* Handle unaligned DCC. */ + if (info->use_display_dcc_unaligned && + (rb_aligned || pipe_aligned)) + return false; + + switch (info->chip_class) { + case GFX9: + /* There are more constraints, but we always set + * INDEPENDENT_64B_BLOCKS = 1 and MAX_COMPRESSED_BLOCK_SIZE = 64B, + * which always works. + */ + assert(surf->u.gfx9.dcc.independent_64B_blocks && + surf->u.gfx9.dcc.max_compressed_block_size == V_028C78_MAX_BLOCK_SIZE_64B); + return true; + case GFX10: + case GFX10_3: + /* DCN requires INDEPENDENT_128B_BLOCKS = 0 only on Navi1x. */ + if (info->chip_class == GFX10 && + surf->u.gfx9.dcc.independent_128B_blocks) + return false; + + /* For 4K, DCN requires INDEPENDENT_64B_BLOCKS = 1. */ + return ((config->info.width <= 2560 && + config->info.height <= 2560) || + (surf->u.gfx9.dcc.independent_64B_blocks && + surf->u.gfx9.dcc.max_compressed_block_size == V_028C78_MAX_BLOCK_SIZE_64B)); + default: + unreachable("unhandled chip"); + return false; + } +} + +static int gfx9_compute_miptree(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, struct radeon_surf *surf, bool compressed, @@ -1045,7 +1438,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, out.size = sizeof(ADDR2_COMPUTE_SURFACE_INFO_OUTPUT); out.pMipInfo = mip_info; - ret = Addr2ComputeSurfaceInfo(addrlib, in, &out); + ret = Addr2ComputeSurfaceInfo(addrlib->handle, in, &out); if (ret != ADDR_OK) return ret; @@ -1071,15 +1464,28 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, surf->u.gfx9.surf_slice_size = out.sliceSize; surf->u.gfx9.surf_pitch = out.pitch; - if (!compressed && surf->blk_w > 1 && out.pitch == out.pixelPitch) { - /* Adjust surf_pitch to be in elements units, - * not in pixels */ - surf->u.gfx9.surf_pitch /= surf->blk_w; - } surf->u.gfx9.surf_height = out.height; surf->surf_size = out.surfSize; surf->surf_alignment = out.baseAlign; + if (!compressed && surf->blk_w > 1 && out.pitch == out.pixelPitch && + surf->u.gfx9.surf.swizzle_mode == ADDR_SW_LINEAR) { + /* Adjust surf_pitch to be in elements units not in pixels */ + surf->u.gfx9.surf_pitch = + align(surf->u.gfx9.surf_pitch / surf->blk_w, 256 / surf->bpe); + surf->u.gfx9.surf.epitch = MAX2(surf->u.gfx9.surf.epitch, + surf->u.gfx9.surf_pitch * surf->blk_w - 1); + /* The surface is really a surf->bpe bytes per pixel surface even if we + * use it as a surf->bpe bytes per element one. + * Adjust surf_slice_size and surf_size to reflect the change + * made to surf_pitch. + */ + surf->u.gfx9.surf_slice_size = MAX2( + surf->u.gfx9.surf_slice_size, + surf->u.gfx9.surf_pitch * out.height * surf->bpe * surf->blk_w); + surf->surf_size = surf->u.gfx9.surf_slice_size * in->numSlices; + } + if (in->swizzleMode == ADDR_SW_LINEAR) { for (unsigned i = 0; i < in->numMipLevels; i++) { surf->u.gfx9.offset[i] = mip_info[i].offset; @@ -1100,8 +1506,11 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, hin.size = sizeof(ADDR2_COMPUTE_HTILE_INFO_INPUT); hout.size = sizeof(ADDR2_COMPUTE_HTILE_INFO_OUTPUT); - hin.hTileFlags.pipeAligned = !in->flags.metaPipeUnaligned; - hin.hTileFlags.rbAligned = !in->flags.metaRbUnaligned; + assert(in->flags.metaPipeUnaligned == 0); + assert(in->flags.metaRbUnaligned == 0); + + hin.hTileFlags.pipeAligned = 1; + hin.hTileFlags.rbAligned = 1; hin.depthFlags = in->flags; hin.swizzleMode = in->swizzleMode; hin.unalignedWidth = in->width; @@ -1110,12 +1519,10 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, hin.numMipLevels = in->numMipLevels; hin.firstMipIdInTail = out.firstMipIdInTail; - ret = Addr2ComputeHtileInfo(addrlib, &hin, &hout); + ret = Addr2ComputeHtileInfo(addrlib->handle, &hin, &hout); if (ret != ADDR_OK) return ret; - surf->u.gfx9.htile.rb_aligned = hin.hTileFlags.rbAligned; - surf->u.gfx9.htile.pipe_aligned = hin.hTileFlags.pipeAligned; surf->htile_size = hout.htileBytes; surf->htile_slice_size = hout.sliceSize; surf->htile_alignment = hout.baseAlign; @@ -1145,7 +1552,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, xin.numSamples = in->numSamples; xin.numFrags = in->numFrags; - ret = Addr2ComputePipeBankXor(addrlib, &xin, &xout); + ret = Addr2ComputePipeBankXor(addrlib->handle, &xin, &xout); if (ret != ADDR_OK) return ret; @@ -1158,7 +1565,11 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, if (info->has_graphics && !(surf->flags & RADEON_SURF_DISABLE_DCC) && !compressed && - gfx9_is_dcc_capable(info, in->swizzleMode)) { + is_dcc_supported_by_CB(info, in->swizzleMode) && + (!in->flags.display || + is_dcc_supported_by_DCN(info, config, surf, + !in->flags.metaRbUnaligned, + !in->flags.metaPipeUnaligned))) { ADDR2_COMPUTE_DCCINFO_INPUT din = {0}; ADDR2_COMPUTE_DCCINFO_OUTPUT dout = {0}; ADDR2_META_MIP_INFO meta_mip_info[RADEON_SURF_MAX_LEVELS] = {}; @@ -1169,7 +1580,6 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, din.dccKeyFlags.pipeAligned = !in->flags.metaPipeUnaligned; din.dccKeyFlags.rbAligned = !in->flags.metaRbUnaligned; - din.colorFlags = in->flags; din.resourceType = in->resourceType; din.swizzleMode = in->swizzleMode; din.bpp = in->bpp; @@ -1181,12 +1591,15 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, din.dataSurfaceSize = out.surfSize; din.firstMipIdInTail = out.firstMipIdInTail; - ret = Addr2ComputeDccInfo(addrlib, &din, &dout); + ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); if (ret != ADDR_OK) return ret; surf->u.gfx9.dcc.rb_aligned = din.dccKeyFlags.rbAligned; surf->u.gfx9.dcc.pipe_aligned = din.dccKeyFlags.pipeAligned; + surf->u.gfx9.dcc_block_width = dout.compressBlkWidth; + surf->u.gfx9.dcc_block_height = dout.compressBlkHeight; + surf->u.gfx9.dcc_block_depth = dout.compressBlkDepth; surf->dcc_size = dout.dccRamSize; surf->dcc_alignment = dout.dccRamBaseAlign; surf->num_dcc_levels = in->numMipLevels; @@ -1215,7 +1628,16 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, */ for (unsigned i = 0; i < in->numMipLevels; i++) { if (meta_mip_info[i].inMiptail) { - surf->num_dcc_levels = i; + /* GFX10 can only compress the first level + * in the mip tail. + * + * TODO: Try to do the same thing for gfx9 + * if there are no regressions. + */ + if (info->chip_class >= GFX10) + surf->num_dcc_levels = i + 1; + else + surf->num_dcc_levels = i; break; } } @@ -1242,7 +1664,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, assert(surf->u.gfx9.dcc.pipe_aligned || surf->u.gfx9.dcc.rb_aligned); - ret = Addr2ComputeDccInfo(addrlib, &din, &dout); + ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); if (ret != ADDR_OK) return ret; @@ -1251,76 +1673,91 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, surf->u.gfx9.display_dcc_pitch_max = dout.pitch - 1; assert(surf->u.gfx9.display_dcc_size <= surf->dcc_size); - /* Compute address mapping from non-displayable to displayable DCC. */ - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin = {}; - addrin.size = sizeof(addrin); - addrin.colorFlags.color = 1; - addrin.swizzleMode = din.swizzleMode; - addrin.resourceType = din.resourceType; - addrin.bpp = din.bpp; - addrin.unalignedWidth = din.unalignedWidth; - addrin.unalignedHeight = din.unalignedHeight; - addrin.numSlices = 1; - addrin.numMipLevels = 1; - addrin.numFrags = 1; - - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_OUTPUT addrout = {}; - addrout.size = sizeof(addrout); + surf->u.gfx9.dcc_retile_use_uint16 = + surf->u.gfx9.display_dcc_size <= UINT16_MAX + 1 && + surf->dcc_size <= UINT16_MAX + 1; + + /* Align the retile map size to get more hash table hits and + * decrease the maximum memory footprint when all retile maps + * are cached in the hash table. + */ + unsigned retile_dim[2] = {in->width, in->height}; + + for (unsigned i = 0; i < 2; i++) { + /* Increase the alignment as the size increases. + * Greater alignment increases retile compute work, + * but decreases maximum memory footprint for the cache. + * + * With this alignment, the worst case memory footprint of + * the cache is: + * 1920x1080: 55 MB + * 2560x1440: 99 MB + * 3840x2160: 305 MB + * + * The worst case size in MB can be computed in Haskell as follows: + * (sum (map get_retile_size (map get_dcc_size (deduplicate (map align_pair + * [(i*16,j*16) | i <- [1..maxwidth`div`16], j <- [1..maxheight`div`16]]))))) `div` 1024^2 + * where + * alignment x = if x <= 512 then 16 else if x <= 1024 then 32 else if x <= 2048 then 64 else 128 + * align x = (x + (alignment x) - 1) `div` (alignment x) * (alignment x) + * align_pair e = (align (fst e), align (snd e)) + * deduplicate = map head . groupBy (\ a b -> ((fst a) == (fst b)) && ((snd a) == (snd b))) . sortBy compare + * get_dcc_size e = ((fst e) * (snd e) * bpp) `div` 256 + * get_retile_size dcc_size = dcc_size * 2 * (if dcc_size <= 2^16 then 2 else 4) + * bpp = 4; maxwidth = 3840; maxheight = 2160 + */ + if (retile_dim[i] <= 512) + retile_dim[i] = align(retile_dim[i], 16); + else if (retile_dim[i] <= 1024) + retile_dim[i] = align(retile_dim[i], 32); + else if (retile_dim[i] <= 2048) + retile_dim[i] = align(retile_dim[i], 64); + else + retile_dim[i] = align(retile_dim[i], 128); + + /* Don't align more than the DCC pixel alignment. */ + assert(dout.metaBlkWidth >= 128 && dout.metaBlkHeight >= 128); + } surf->u.gfx9.dcc_retile_num_elements = - DIV_ROUND_UP(in->width, dout.compressBlkWidth) * - DIV_ROUND_UP(in->height, dout.compressBlkHeight) * 2; + DIV_ROUND_UP(retile_dim[0], dout.compressBlkWidth) * + DIV_ROUND_UP(retile_dim[1], dout.compressBlkHeight) * 2; /* Align the size to 4 (for the compute shader). */ surf->u.gfx9.dcc_retile_num_elements = align(surf->u.gfx9.dcc_retile_num_elements, 4); - surf->u.gfx9.dcc_retile_map = - malloc(surf->u.gfx9.dcc_retile_num_elements * 4); - if (!surf->u.gfx9.dcc_retile_map) - return ADDR_OUTOFMEMORY; - - unsigned index = 0; - surf->u.gfx9.dcc_retile_use_uint16 = true; - - for (unsigned y = 0; y < in->height; y += dout.compressBlkHeight) { - addrin.y = y; - - for (unsigned x = 0; x < in->width; x += dout.compressBlkWidth) { - addrin.x = x; - - /* Compute src DCC address */ - addrin.dccKeyFlags.pipeAligned = surf->u.gfx9.dcc.pipe_aligned; - addrin.dccKeyFlags.rbAligned = surf->u.gfx9.dcc.rb_aligned; - addrout.addr = 0; - - ret = Addr2ComputeDccAddrFromCoord(addrlib, &addrin, &addrout); - if (ret != ADDR_OK) - return ret; - - surf->u.gfx9.dcc_retile_map[index * 2] = addrout.addr; - if (addrout.addr > UINT16_MAX) - surf->u.gfx9.dcc_retile_use_uint16 = false; - - /* Compute dst DCC address */ - addrin.dccKeyFlags.pipeAligned = 0; - addrin.dccKeyFlags.rbAligned = 0; - addrout.addr = 0; - - ret = Addr2ComputeDccAddrFromCoord(addrlib, &addrin, &addrout); - if (ret != ADDR_OK) - return ret; - - surf->u.gfx9.dcc_retile_map[index * 2 + 1] = addrout.addr; - if (addrout.addr > UINT16_MAX) - surf->u.gfx9.dcc_retile_use_uint16 = false; - - assert(index * 2 + 1 < surf->u.gfx9.dcc_retile_num_elements); - index++; - } + if (!(surf->flags & RADEON_SURF_IMPORTED)) { + /* Compute address mapping from non-displayable to displayable DCC. */ + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin; + memset(&addrin, 0, sizeof(addrin)); + addrin.size = sizeof(addrin); + addrin.swizzleMode = din.swizzleMode; + addrin.resourceType = din.resourceType; + addrin.bpp = din.bpp; + addrin.numSlices = 1; + addrin.numMipLevels = 1; + addrin.numFrags = 1; + addrin.pitch = dout.pitch; + addrin.height = dout.height; + addrin.compressBlkWidth = dout.compressBlkWidth; + addrin.compressBlkHeight = dout.compressBlkHeight; + addrin.compressBlkDepth = dout.compressBlkDepth; + addrin.metaBlkWidth = dout.metaBlkWidth; + addrin.metaBlkHeight = dout.metaBlkHeight; + addrin.metaBlkDepth = dout.metaBlkDepth; + addrin.dccRamSliceSize = 0; /* Don't care for non-layered images. */ + + surf->u.gfx9.dcc_retile_map = + ac_compute_dcc_retile_map(addrlib, info, + retile_dim[0], retile_dim[1], + surf->u.gfx9.dcc.rb_aligned, + surf->u.gfx9.dcc.pipe_aligned, + surf->u.gfx9.dcc_retile_use_uint16, + surf->u.gfx9.dcc_retile_num_elements, + &addrin); + if (!surf->u.gfx9.dcc_retile_map) + return ADDR_OUTOFMEMORY; } - /* Fill the remaining pairs with the last one (for the compute shader). */ - for (unsigned i = index * 2; i < surf->u.gfx9.dcc_retile_num_elements; i++) - surf->u.gfx9.dcc_retile_map[i] = surf->u.gfx9.dcc_retile_map[i - 2]; } } @@ -1333,7 +1770,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, fin.size = sizeof(ADDR2_COMPUTE_FMASK_INFO_INPUT); fout.size = sizeof(ADDR2_COMPUTE_FMASK_INFO_OUTPUT); - ret = gfx9_get_preferred_swizzle_mode(addrlib, surf, in, + ret = gfx9_get_preferred_swizzle_mode(addrlib->handle, surf, in, true, &fin.swizzleMode); if (ret != ADDR_OK) return ret; @@ -1344,7 +1781,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, fin.numSamples = in->numSamples; fin.numFrags = in->numFrags; - ret = Addr2ComputeFmaskInfo(addrlib, &fin, &fout); + ret = Addr2ComputeFmaskInfo(addrlib->handle, &fin, &fout); if (ret != ADDR_OK) return ret; @@ -1372,7 +1809,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, xin.numSamples = in->numSamples; xin.numFrags = in->numFrags; - ret = Addr2ComputePipeBankXor(addrlib, &xin, &xout); + ret = Addr2ComputePipeBankXor(addrlib->handle, &xin, &xout); if (ret != ADDR_OK) return ret; @@ -1385,7 +1822,10 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, /* CMASK -- on GFX10 only for FMASK */ if (in->swizzleMode != ADDR_SW_LINEAR && in->resourceType == ADDR_RSRC_TEX_2D && - ((info->chip_class <= GFX9 && in->numSamples == 1) || + ((info->chip_class <= GFX9 && + in->numSamples == 1 && + in->flags.metaPipeUnaligned == 0 && + in->flags.metaRbUnaligned == 0) || (surf->fmask_size && in->numSamples >= 2))) { ADDR2_COMPUTE_CMASK_INFO_INPUT cin = {0}; ADDR2_COMPUTE_CMASK_INFO_OUTPUT cout = {0}; @@ -1393,15 +1833,11 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, cin.size = sizeof(ADDR2_COMPUTE_CMASK_INFO_INPUT); cout.size = sizeof(ADDR2_COMPUTE_CMASK_INFO_OUTPUT); - if (in->numSamples > 1) { - /* FMASK is always aligned. */ - cin.cMaskFlags.pipeAligned = 1; - cin.cMaskFlags.rbAligned = 1; - } else { - cin.cMaskFlags.pipeAligned = !in->flags.metaPipeUnaligned; - cin.cMaskFlags.rbAligned = !in->flags.metaRbUnaligned; - } - cin.colorFlags = in->flags; + assert(in->flags.metaPipeUnaligned == 0); + assert(in->flags.metaRbUnaligned == 0); + + cin.cMaskFlags.pipeAligned = 1; + cin.cMaskFlags.rbAligned = 1; cin.resourceType = in->resourceType; cin.unalignedWidth = in->width; cin.unalignedHeight = in->height; @@ -1412,12 +1848,10 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, else cin.swizzleMode = in->swizzleMode; - ret = Addr2ComputeCmaskInfo(addrlib, &cin, &cout); + ret = Addr2ComputeCmaskInfo(addrlib->handle, &cin, &cout); if (ret != ADDR_OK) return ret; - surf->u.gfx9.cmask.rb_aligned = cin.cMaskFlags.rbAligned; - surf->u.gfx9.cmask.pipe_aligned = cin.cMaskFlags.pipeAligned; surf->cmask_size = cout.cmaskBytes; surf->cmask_alignment = cout.baseAlign; } @@ -1426,7 +1860,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, return 0; } -static int gfx9_compute_surface(ADDR_HANDLE addrlib, +static int gfx9_compute_surface(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, enum radeon_surf_mode mode, @@ -1524,21 +1958,53 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, else AddrSurfInfoIn.numSlices = config->info.array_size; - /* This is propagated to HTILE/DCC/CMASK. */ + /* This is propagated to DCC. It must be 0 for HTILE and CMASK. */ AddrSurfInfoIn.flags.metaPipeUnaligned = 0; AddrSurfInfoIn.flags.metaRbUnaligned = 0; - /* The display hardware can only read DCC with RB_ALIGNED=0 and - * PIPE_ALIGNED=0. PIPE_ALIGNED really means L2CACHE_ALIGNED. - * - * The CB block requires RB_ALIGNED=1 except 1 RB chips. - * PIPE_ALIGNED is optional, but PIPE_ALIGNED=0 requires L2 flushes - * after rendering, so PIPE_ALIGNED=1 is recommended. - */ - if (info->use_display_dcc_unaligned && is_color_surface && - AddrSurfInfoIn.flags.display) { - AddrSurfInfoIn.flags.metaPipeUnaligned = 1; - AddrSurfInfoIn.flags.metaRbUnaligned = 1; + /* Optimal values for the L2 cache. */ + if (info->chip_class == GFX9) { + surf->u.gfx9.dcc.independent_64B_blocks = 1; + surf->u.gfx9.dcc.independent_128B_blocks = 0; + surf->u.gfx9.dcc.max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B; + } else if (info->chip_class >= GFX10) { + surf->u.gfx9.dcc.independent_64B_blocks = 0; + surf->u.gfx9.dcc.independent_128B_blocks = 1; + surf->u.gfx9.dcc.max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B; + } + + if (AddrSurfInfoIn.flags.display) { + /* The display hardware can only read DCC with RB_ALIGNED=0 and + * PIPE_ALIGNED=0. PIPE_ALIGNED really means L2CACHE_ALIGNED. + * + * The CB block requires RB_ALIGNED=1 except 1 RB chips. + * PIPE_ALIGNED is optional, but PIPE_ALIGNED=0 requires L2 flushes + * after rendering, so PIPE_ALIGNED=1 is recommended. + */ + if (info->use_display_dcc_unaligned) { + AddrSurfInfoIn.flags.metaPipeUnaligned = 1; + AddrSurfInfoIn.flags.metaRbUnaligned = 1; + } + + /* Adjust DCC settings to meet DCN requirements. */ + if (info->use_display_dcc_unaligned || + info->use_display_dcc_with_retile_blit) { + /* Only Navi12/14 support independent 64B blocks in L2, + * but without DCC image stores. + */ + if (info->family == CHIP_NAVI12 || + info->family == CHIP_NAVI14) { + surf->u.gfx9.dcc.independent_64B_blocks = 1; + surf->u.gfx9.dcc.independent_128B_blocks = 0; + surf->u.gfx9.dcc.max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B; + } + + if (info->chip_class >= GFX10_3) { + surf->u.gfx9.dcc.independent_64B_blocks = 1; + surf->u.gfx9.dcc.independent_128B_blocks = 1; + surf->u.gfx9.dcc.max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B; + } + } } switch (mode) { @@ -1550,12 +2016,14 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, case RADEON_SURF_MODE_1D: case RADEON_SURF_MODE_2D: - if (surf->flags & (RADEON_SURF_IMPORTED | RADEON_SURF_FORCE_SWIZZLE_MODE)) { + if (surf->flags & RADEON_SURF_IMPORTED || + (info->chip_class >= GFX10 && + surf->flags & RADEON_SURF_FORCE_SWIZZLE_MODE)) { AddrSurfInfoIn.swizzleMode = surf->u.gfx9.surf.swizzle_mode; break; } - r = gfx9_get_preferred_swizzle_mode(addrlib, surf, &AddrSurfInfoIn, + r = gfx9_get_preferred_swizzle_mode(addrlib->handle, surf, &AddrSurfInfoIn, false, &AddrSurfInfoIn.swizzleMode); if (r) return r; @@ -1585,7 +2053,7 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, r = gfx9_compute_miptree(addrlib, info, config, surf, compressed, &AddrSurfInfoIn); if (r) - goto error; + return r; /* Calculate texture layout information for stencil. */ if (surf->flags & RADEON_SURF_SBUFFER) { @@ -1594,38 +2062,85 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, AddrSurfInfoIn.format = ADDR_FMT_8; if (!AddrSurfInfoIn.flags.depth) { - r = gfx9_get_preferred_swizzle_mode(addrlib, surf, &AddrSurfInfoIn, + r = gfx9_get_preferred_swizzle_mode(addrlib->handle, surf, &AddrSurfInfoIn, false, &AddrSurfInfoIn.swizzleMode); if (r) - goto error; + return r; } else AddrSurfInfoIn.flags.depth = 0; r = gfx9_compute_miptree(addrlib, info, config, surf, compressed, &AddrSurfInfoIn); if (r) - goto error; + return r; } surf->is_linear = surf->u.gfx9.surf.swizzle_mode == ADDR_SW_LINEAR; /* Query whether the surface is displayable. */ + /* This is only useful for surfaces that are allocated without SCANOUT. */ bool displayable = false; if (!config->is_3d && !config->is_cube) { - r = Addr2IsValidDisplaySwizzleMode(addrlib, surf->u.gfx9.surf.swizzle_mode, - surf->bpe * 8, &displayable); + r = Addr2IsValidDisplaySwizzleMode(addrlib->handle, surf->u.gfx9.surf.swizzle_mode, + surf->bpe * 8, &displayable); if (r) - goto error; + return r; /* Display needs unaligned DCC. */ - if (info->use_display_dcc_unaligned && - surf->num_dcc_levels && - (surf->u.gfx9.dcc.pipe_aligned || - surf->u.gfx9.dcc.rb_aligned)) + if (surf->num_dcc_levels && + (!is_dcc_supported_by_DCN(info, config, surf, + surf->u.gfx9.dcc.rb_aligned, + surf->u.gfx9.dcc.pipe_aligned) || + /* Don't set is_displayable if displayable DCC is missing. */ + (info->use_display_dcc_with_retile_blit && + !surf->u.gfx9.dcc_retile_num_elements))) displayable = false; } surf->is_displayable = displayable; + /* Validate that we allocated a displayable surface if requested. */ + assert(!AddrSurfInfoIn.flags.display || surf->is_displayable); + + /* Validate that DCC is set up correctly. */ + if (surf->num_dcc_levels) { + assert(is_dcc_supported_by_L2(info, surf)); + if (AddrSurfInfoIn.flags.color) + assert(is_dcc_supported_by_CB(info, surf->u.gfx9.surf.swizzle_mode)); + if (AddrSurfInfoIn.flags.display) { + assert(is_dcc_supported_by_DCN(info, config, surf, + surf->u.gfx9.dcc.rb_aligned, + surf->u.gfx9.dcc.pipe_aligned)); + } + } + + if (info->has_graphics && + !compressed && + !config->is_3d && + config->info.levels == 1 && + AddrSurfInfoIn.flags.color && + !surf->is_linear && + surf->surf_alignment >= 64 * 1024 && /* 64KB tiling */ + !(surf->flags & (RADEON_SURF_DISABLE_DCC | + RADEON_SURF_FORCE_SWIZZLE_MODE | + RADEON_SURF_FORCE_MICRO_TILE_MODE))) { + /* Validate that DCC is enabled if DCN can do it. */ + if ((info->use_display_dcc_unaligned || + info->use_display_dcc_with_retile_blit) && + AddrSurfInfoIn.flags.display && + surf->bpe == 4) { + assert(surf->num_dcc_levels); + } + + /* Validate that non-scanout DCC is always enabled. */ + if (!AddrSurfInfoIn.flags.display) + assert(surf->num_dcc_levels); + } + + if (!surf->htile_size) { + /* Unset this if HTILE is not present. */ + surf->flags &= ~RADEON_SURF_TC_COMPATIBLE_HTILE; + } + switch (surf->u.gfx9.surf.swizzle_mode) { /* S = standard. */ case ADDR_SW_256B_S: @@ -1634,7 +2149,7 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, case ADDR_SW_64KB_S_T: case ADDR_SW_4KB_S_X: case ADDR_SW_64KB_S_X: - surf->micro_tile_mode = RADEON_MICRO_MODE_THIN; + surf->micro_tile_mode = RADEON_MICRO_MODE_STANDARD; break; /* D = display. */ @@ -1662,7 +2177,7 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, */ assert(info->chip_class >= GFX10 || !"rotate micro tile mode is unsupported"); - surf->micro_tile_mode = RADEON_MICRO_MODE_ROTATED; + surf->micro_tile_mode = RADEON_MICRO_MODE_RENDER; break; /* Z = depth. */ @@ -1680,14 +2195,9 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, } return 0; - -error: - free(surf->u.gfx9.dcc_retile_map); - surf->u.gfx9.dcc_retile_map = NULL; - return r; } -int ac_compute_surface(ADDR_HANDLE addrlib, const struct radeon_info *info, +int ac_compute_surface(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, enum radeon_surf_mode mode, struct radeon_surf *surf) @@ -1701,38 +2211,44 @@ int ac_compute_surface(ADDR_HANDLE addrlib, const struct radeon_info *info, if (info->chip_class >= GFX9) r = gfx9_compute_surface(addrlib, info, config, mode, surf); else - r = gfx6_compute_surface(addrlib, info, config, mode, surf); + r = gfx6_compute_surface(addrlib->handle, info, config, mode, surf); if (r) return r; /* Determine the memory layout of multiple allocations in one buffer. */ surf->total_size = surf->surf_size; + surf->alignment = surf->surf_alignment; if (surf->htile_size) { surf->htile_offset = align64(surf->total_size, surf->htile_alignment); surf->total_size = surf->htile_offset + surf->htile_size; + surf->alignment = MAX2(surf->alignment, surf->htile_alignment); } if (surf->fmask_size) { assert(config->info.samples >= 2); surf->fmask_offset = align64(surf->total_size, surf->fmask_alignment); surf->total_size = surf->fmask_offset + surf->fmask_size; + surf->alignment = MAX2(surf->alignment, surf->fmask_alignment); } /* Single-sample CMASK is in a separate buffer. */ if (surf->cmask_size && config->info.samples >= 2) { surf->cmask_offset = align64(surf->total_size, surf->cmask_alignment); surf->total_size = surf->cmask_offset + surf->cmask_size; + surf->alignment = MAX2(surf->alignment, surf->cmask_alignment); } - if (surf->dcc_size && - (info->use_display_dcc_unaligned || - info->use_display_dcc_with_retile_blit || - !(surf->flags & RADEON_SURF_SCANOUT))) { - surf->dcc_offset = align64(surf->total_size, surf->dcc_alignment); - surf->total_size = surf->dcc_offset + surf->dcc_size; + if (surf->is_displayable) + surf->flags |= RADEON_SURF_SCANOUT; + if (surf->dcc_size && + /* dcc_size is computed on GFX9+ only if it's displayable. */ + (info->chip_class >= GFX9 || !get_display_flag(config, surf))) { + /* It's better when displayable DCC is immediately after + * the image due to hw-specific reasons. + */ if (info->chip_class >= GFX9 && surf->u.gfx9.dcc_retile_num_elements) { /* Add space for the displayable DCC buffer. */ @@ -1753,7 +2269,333 @@ int ac_compute_surface(ADDR_HANDLE addrlib, const struct radeon_info *info, surf->u.gfx9.dcc_retile_num_elements * 4; } } + + surf->dcc_offset = align64(surf->total_size, surf->dcc_alignment); + surf->total_size = surf->dcc_offset + surf->dcc_size; + surf->alignment = MAX2(surf->alignment, surf->dcc_alignment); } return 0; } + +/* This is meant to be used for disabling DCC. */ +void ac_surface_zero_dcc_fields(struct radeon_surf *surf) +{ + surf->dcc_offset = 0; + surf->display_dcc_offset = 0; + surf->dcc_retile_map_offset = 0; +} + +static unsigned eg_tile_split(unsigned tile_split) +{ + switch (tile_split) { + case 0: tile_split = 64; break; + case 1: tile_split = 128; break; + case 2: tile_split = 256; break; + case 3: tile_split = 512; break; + default: + case 4: tile_split = 1024; break; + case 5: tile_split = 2048; break; + case 6: tile_split = 4096; break; + } + return tile_split; +} + +static unsigned eg_tile_split_rev(unsigned eg_tile_split) +{ + switch (eg_tile_split) { + case 64: return 0; + case 128: return 1; + case 256: return 2; + case 512: return 3; + default: + case 1024: return 4; + case 2048: return 5; + case 4096: return 6; + } +} + +#define AMDGPU_TILING_DCC_MAX_COMPRESSED_BLOCK_SIZE_SHIFT 45 +#define AMDGPU_TILING_DCC_MAX_COMPRESSED_BLOCK_SIZE_MASK 0x3 + +/* This should be called before ac_compute_surface. */ +void ac_surface_set_bo_metadata(const struct radeon_info *info, + struct radeon_surf *surf, uint64_t tiling_flags, + enum radeon_surf_mode *mode) +{ + bool scanout; + + if (info->chip_class >= GFX9) { + surf->u.gfx9.surf.swizzle_mode = AMDGPU_TILING_GET(tiling_flags, SWIZZLE_MODE); + surf->u.gfx9.dcc.independent_64B_blocks = AMDGPU_TILING_GET(tiling_flags, DCC_INDEPENDENT_64B); + surf->u.gfx9.dcc.independent_128B_blocks = AMDGPU_TILING_GET(tiling_flags, DCC_INDEPENDENT_128B); + surf->u.gfx9.dcc.max_compressed_block_size = AMDGPU_TILING_GET(tiling_flags, DCC_MAX_COMPRESSED_BLOCK_SIZE); + surf->u.gfx9.display_dcc_pitch_max = AMDGPU_TILING_GET(tiling_flags, DCC_PITCH_MAX); + scanout = AMDGPU_TILING_GET(tiling_flags, SCANOUT); + *mode = surf->u.gfx9.surf.swizzle_mode > 0 ? RADEON_SURF_MODE_2D : RADEON_SURF_MODE_LINEAR_ALIGNED; + } else { + surf->u.legacy.pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG); + surf->u.legacy.bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH); + surf->u.legacy.bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT); + surf->u.legacy.tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT)); + surf->u.legacy.mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT); + surf->u.legacy.num_banks = 2 << AMDGPU_TILING_GET(tiling_flags, NUM_BANKS); + scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */ + + if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4) /* 2D_TILED_THIN1 */ + *mode = RADEON_SURF_MODE_2D; + else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */ + *mode = RADEON_SURF_MODE_1D; + else + *mode = RADEON_SURF_MODE_LINEAR_ALIGNED; + } + + if (scanout) + surf->flags |= RADEON_SURF_SCANOUT; + else + surf->flags &= ~RADEON_SURF_SCANOUT; +} + +void ac_surface_get_bo_metadata(const struct radeon_info *info, + struct radeon_surf *surf, uint64_t *tiling_flags) +{ + *tiling_flags = 0; + + if (info->chip_class >= GFX9) { + uint64_t dcc_offset = 0; + + if (surf->dcc_offset) { + dcc_offset = surf->display_dcc_offset ? surf->display_dcc_offset + : surf->dcc_offset; + assert((dcc_offset >> 8) != 0 && (dcc_offset >> 8) < (1 << 24)); + } + + *tiling_flags |= AMDGPU_TILING_SET(SWIZZLE_MODE, surf->u.gfx9.surf.swizzle_mode); + *tiling_flags |= AMDGPU_TILING_SET(DCC_OFFSET_256B, dcc_offset >> 8); + *tiling_flags |= AMDGPU_TILING_SET(DCC_PITCH_MAX, surf->u.gfx9.display_dcc_pitch_max); + *tiling_flags |= AMDGPU_TILING_SET(DCC_INDEPENDENT_64B, surf->u.gfx9.dcc.independent_64B_blocks); + *tiling_flags |= AMDGPU_TILING_SET(DCC_INDEPENDENT_128B, surf->u.gfx9.dcc.independent_128B_blocks); + *tiling_flags |= AMDGPU_TILING_SET(DCC_MAX_COMPRESSED_BLOCK_SIZE, surf->u.gfx9.dcc.max_compressed_block_size); + *tiling_flags |= AMDGPU_TILING_SET(SCANOUT, (surf->flags & RADEON_SURF_SCANOUT) != 0); + } else { + if (surf->u.legacy.level[0].mode >= RADEON_SURF_MODE_2D) + *tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */ + else if (surf->u.legacy.level[0].mode >= RADEON_SURF_MODE_1D) + *tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */ + else + *tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */ + + *tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, surf->u.legacy.pipe_config); + *tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(surf->u.legacy.bankw)); + *tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(surf->u.legacy.bankh)); + if (surf->u.legacy.tile_split) + *tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(surf->u.legacy.tile_split)); + *tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(surf->u.legacy.mtilea)); + *tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(surf->u.legacy.num_banks)-1); + + if (surf->flags & RADEON_SURF_SCANOUT) + *tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */ + else + *tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */ + } +} + +static uint32_t ac_get_umd_metadata_word1(const struct radeon_info *info) +{ + return (ATI_VENDOR_ID << 16) | info->pci_id; +} + +/* This should be called after ac_compute_surface. */ +bool ac_surface_set_umd_metadata(const struct radeon_info *info, + struct radeon_surf *surf, + unsigned num_storage_samples, + unsigned num_mipmap_levels, + unsigned size_metadata, + uint32_t metadata[64]) +{ + uint32_t *desc = &metadata[2]; + uint64_t offset; + + if (info->chip_class >= GFX9) + offset = surf->u.gfx9.surf_offset; + else + offset = surf->u.legacy.level[0].offset; + + if (offset || /* Non-zero planes ignore metadata. */ + size_metadata < 10 * 4 || /* at least 2(header) + 8(desc) dwords */ + metadata[0] == 0 || /* invalid version number */ + metadata[1] != ac_get_umd_metadata_word1(info)) /* invalid PCI ID */ { + /* Disable DCC because it might not be enabled. */ + ac_surface_zero_dcc_fields(surf); + + /* Don't report an error if the texture comes from an incompatible driver, + * but this might not work. + */ + return true; + } + + /* Validate that sample counts and the number of mipmap levels match. */ + unsigned desc_last_level = G_008F1C_LAST_LEVEL(desc[3]); + unsigned type = G_008F1C_TYPE(desc[3]); + + if (type == V_008F1C_SQ_RSRC_IMG_2D_MSAA || type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) { + unsigned log_samples = util_logbase2(MAX2(1, num_storage_samples)); + + if (desc_last_level != log_samples) { + fprintf(stderr, + "amdgpu: invalid MSAA texture import, " + "metadata has log2(samples) = %u, the caller set %u\n", + desc_last_level, log_samples); + return false; + } + } else { + if (desc_last_level != num_mipmap_levels - 1) { + fprintf(stderr, + "amdgpu: invalid mipmapped texture import, " + "metadata has last_level = %u, the caller set %u\n", + desc_last_level, num_mipmap_levels - 1); + return false; + } + } + + if (info->chip_class >= GFX8 && G_008F28_COMPRESSION_EN(desc[6])) { + /* Read DCC information. */ + switch (info->chip_class) { + case GFX8: + surf->dcc_offset = (uint64_t)desc[7] << 8; + break; + + case GFX9: + surf->dcc_offset = + ((uint64_t)desc[7] << 8) | ((uint64_t)G_008F24_META_DATA_ADDRESS(desc[5]) << 40); + surf->u.gfx9.dcc.pipe_aligned = G_008F24_META_PIPE_ALIGNED(desc[5]); + surf->u.gfx9.dcc.rb_aligned = G_008F24_META_RB_ALIGNED(desc[5]); + + /* If DCC is unaligned, this can only be a displayable image. */ + if (!surf->u.gfx9.dcc.pipe_aligned && !surf->u.gfx9.dcc.rb_aligned) + assert(surf->is_displayable); + break; + + case GFX10: + case GFX10_3: + surf->dcc_offset = + ((uint64_t)G_00A018_META_DATA_ADDRESS_LO(desc[6]) << 8) | ((uint64_t)desc[7] << 16); + surf->u.gfx9.dcc.pipe_aligned = G_00A018_META_PIPE_ALIGNED(desc[6]); + break; + + default: + assert(0); + return false; + } + } else { + /* Disable DCC. dcc_offset is always set by texture_from_handle + * and must be cleared here. + */ + ac_surface_zero_dcc_fields(surf); + } + + return true; +} + +void ac_surface_get_umd_metadata(const struct radeon_info *info, + struct radeon_surf *surf, + unsigned num_mipmap_levels, + uint32_t desc[8], + unsigned *size_metadata, uint32_t metadata[64]) +{ + /* Clear the base address and set the relative DCC offset. */ + desc[0] = 0; + desc[1] &= C_008F14_BASE_ADDRESS_HI; + + switch (info->chip_class) { + case GFX6: + case GFX7: + break; + case GFX8: + desc[7] = surf->dcc_offset >> 8; + break; + case GFX9: + desc[7] = surf->dcc_offset >> 8; + desc[5] &= C_008F24_META_DATA_ADDRESS; + desc[5] |= S_008F24_META_DATA_ADDRESS(surf->dcc_offset >> 40); + break; + case GFX10: + case GFX10_3: + desc[6] &= C_00A018_META_DATA_ADDRESS_LO; + desc[6] |= S_00A018_META_DATA_ADDRESS_LO(surf->dcc_offset >> 8); + desc[7] = surf->dcc_offset >> 16; + break; + default: + assert(0); + } + + /* Metadata image format format version 1: + * [0] = 1 (metadata format identifier) + * [1] = (VENDOR_ID << 16) | PCI_ID + * [2:9] = image descriptor for the whole resource + * [2] is always 0, because the base address is cleared + * [9] is the DCC offset bits [39:8] from the beginning of + * the buffer + * [10:10+LAST_LEVEL] = mipmap level offset bits [39:8] for each level + */ + + metadata[0] = 1; /* metadata image format version 1 */ + + /* Tiling modes are ambiguous without a PCI ID. */ + metadata[1] = ac_get_umd_metadata_word1(info); + + /* Dwords [2:9] contain the image descriptor. */ + memcpy(&metadata[2], desc, 8 * 4); + *size_metadata = 10 * 4; + + /* Dwords [10:..] contain the mipmap level offsets. */ + if (info->chip_class <= GFX8) { + for (unsigned i = 0; i < num_mipmap_levels; i++) + metadata[10 + i] = surf->u.legacy.level[i].offset >> 8; + + *size_metadata += num_mipmap_levels * 4; + } +} + +void ac_surface_override_offset_stride(const struct radeon_info *info, + struct radeon_surf *surf, + unsigned num_mipmap_levels, + uint64_t offset, unsigned pitch) +{ + if (info->chip_class >= GFX9) { + if (pitch) { + surf->u.gfx9.surf_pitch = pitch; + if (num_mipmap_levels == 1) + surf->u.gfx9.surf.epitch = pitch - 1; + surf->u.gfx9.surf_slice_size = + (uint64_t)pitch * surf->u.gfx9.surf_height * surf->bpe; + } + surf->u.gfx9.surf_offset = offset; + if (surf->u.gfx9.stencil_offset) + surf->u.gfx9.stencil_offset += offset; + } else { + if (pitch) { + surf->u.legacy.level[0].nblk_x = pitch; + surf->u.legacy.level[0].slice_size_dw = + ((uint64_t)pitch * surf->u.legacy.level[0].nblk_y * surf->bpe) / 4; + } + + if (offset) { + for (unsigned i = 0; i < ARRAY_SIZE(surf->u.legacy.level); ++i) + surf->u.legacy.level[i].offset += offset; + } + } + + if (surf->htile_offset) + surf->htile_offset += offset; + if (surf->fmask_offset) + surf->fmask_offset += offset; + if (surf->cmask_offset) + surf->cmask_offset += offset; + if (surf->dcc_offset) + surf->dcc_offset += offset; + if (surf->display_dcc_offset) + surf->display_dcc_offset += offset; + if (surf->dcc_retile_map_offset) + surf->dcc_retile_map_offset += offset; +}