X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fcommon%2Fac_surface.c;h=461fd503622fa974062247440e43052556a2d9a7;hb=81d106d6ecab326c96fb9f8db8e7f39621c9816e;hp=8203a7f04261db157c54205820fc9fe80021dbcd;hpb=ffae3589c986574083fbd20341e6a0abe77b0741;p=mesa.git diff --git a/src/amd/common/ac_surface.c b/src/amd/common/ac_surface.c index 8203a7f0426..461fd503622 100644 --- a/src/amd/common/ac_surface.c +++ b/src/amd/common/ac_surface.c @@ -29,9 +29,12 @@ #include "amd_family.h" #include "addrlib/src/amdgpu_asic_addr.h" #include "ac_gpu_info.h" +#include "util/hash_table.h" #include "util/macros.h" +#include "util/simple_mtx.h" #include "util/u_atomic.h" #include "util/u_math.h" +#include "util/u_memory.h" #include "sid.h" #include @@ -50,6 +53,282 @@ #define CIASICIDGFXENGINE_ARCTICISLAND 0x0000000D #endif +struct ac_addrlib { + ADDR_HANDLE handle; + + /* The cache of DCC retile maps for reuse when allocating images of + * similar sizes. + */ + simple_mtx_t dcc_retile_map_lock; + struct hash_table *dcc_retile_maps; + struct hash_table *dcc_retile_tile_indices; +}; + +struct dcc_retile_map_key { + enum radeon_family family; + unsigned retile_width; + unsigned retile_height; + bool rb_aligned; + bool pipe_aligned; + unsigned dcc_retile_num_elements; + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT input; +}; + +static uint32_t dcc_retile_map_hash_key(const void *key) +{ + return _mesa_hash_data(key, sizeof(struct dcc_retile_map_key)); +} + +static bool dcc_retile_map_keys_equal(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(struct dcc_retile_map_key)) == 0; +} + +static void dcc_retile_map_free(struct hash_entry *entry) +{ + free((void*)entry->key); + free(entry->data); +} + +struct dcc_retile_tile_key { + enum radeon_family family; + unsigned bpp; + unsigned swizzle_mode; + bool rb_aligned; + bool pipe_aligned; +}; + +struct dcc_retile_tile_data { + unsigned tile_width_log2; + unsigned tile_height_log2; + uint16_t *data; +}; + +static uint32_t dcc_retile_tile_hash_key(const void *key) +{ + return _mesa_hash_data(key, sizeof(struct dcc_retile_tile_key)); +} + +static bool dcc_retile_tile_keys_equal(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(struct dcc_retile_tile_key)) == 0; +} + +static void dcc_retile_tile_free(struct hash_entry *entry) +{ + free((void*)entry->key); + free(((struct dcc_retile_tile_data*)entry->data)->data); + free(entry->data); +} + +/* Assumes dcc_retile_map_lock is taken. */ +static const struct dcc_retile_tile_data * +ac_compute_dcc_retile_tile_indices(struct ac_addrlib *addrlib, + const struct radeon_info *info, + unsigned bpp, unsigned swizzle_mode, + bool rb_aligned, bool pipe_aligned) +{ + struct dcc_retile_tile_key key = (struct dcc_retile_tile_key) { + .family = info->family, + .bpp = bpp, + .swizzle_mode = swizzle_mode, + .rb_aligned = rb_aligned, + .pipe_aligned = pipe_aligned + }; + + struct hash_entry *entry = _mesa_hash_table_search(addrlib->dcc_retile_tile_indices, &key); + if (entry) + return entry->data; + + ADDR2_COMPUTE_DCCINFO_INPUT din = {0}; + ADDR2_COMPUTE_DCCINFO_OUTPUT dout = {0}; + din.size = sizeof(ADDR2_COMPUTE_DCCINFO_INPUT); + dout.size = sizeof(ADDR2_COMPUTE_DCCINFO_OUTPUT); + + din.dccKeyFlags.pipeAligned = pipe_aligned; + din.dccKeyFlags.rbAligned = rb_aligned; + din.resourceType = ADDR_RSRC_TEX_2D; + din.swizzleMode = swizzle_mode; + din.bpp = bpp; + din.unalignedWidth = 1; + din.unalignedHeight = 1; + din.numSlices = 1; + din.numFrags = 1; + din.numMipLevels = 1; + + ADDR_E_RETURNCODE ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); + if (ret != ADDR_OK) + return NULL; + + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin = {0}; + addrin.size = sizeof(addrin); + addrin.swizzleMode = swizzle_mode; + addrin.resourceType = ADDR_RSRC_TEX_2D; + addrin.bpp = bpp; + addrin.numSlices = 1; + addrin.numMipLevels = 1; + addrin.numFrags = 1; + addrin.pitch = dout.pitch; + addrin.height = dout.height; + addrin.compressBlkWidth = dout.compressBlkWidth; + addrin.compressBlkHeight = dout.compressBlkHeight; + addrin.compressBlkDepth = dout.compressBlkDepth; + addrin.metaBlkWidth = dout.metaBlkWidth; + addrin.metaBlkHeight = dout.metaBlkHeight; + addrin.metaBlkDepth = dout.metaBlkDepth; + addrin.dccKeyFlags.pipeAligned = pipe_aligned; + addrin.dccKeyFlags.rbAligned = rb_aligned; + + unsigned w = dout.metaBlkWidth / dout.compressBlkWidth; + unsigned h = dout.metaBlkHeight / dout.compressBlkHeight; + uint16_t *indices = malloc(w * h * sizeof (uint16_t)); + if (!indices) + return NULL; + + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_OUTPUT addrout = {}; + addrout.size = sizeof(addrout); + + for (unsigned y = 0; y < h; ++y) { + addrin.y = y * dout.compressBlkHeight; + for (unsigned x = 0; x < w; ++x) { + addrin.x = x * dout.compressBlkWidth; + addrout.addr = 0; + + if (Addr2ComputeDccAddrFromCoord(addrlib->handle, &addrin, &addrout) != ADDR_OK) { + free(indices); + return NULL; + } + indices[y * w + x] = addrout.addr; + } + } + + struct dcc_retile_tile_data *data = calloc(1, sizeof(*data)); + if (!data) { + free(indices); + return NULL; + } + + data->tile_width_log2 = util_logbase2(w); + data->tile_height_log2 = util_logbase2(h); + data->data = indices; + + struct dcc_retile_tile_key *heap_key = mem_dup(&key, sizeof(key)); + if (!heap_key) { + free(data); + free(indices); + return NULL; + } + + entry = _mesa_hash_table_insert(addrlib->dcc_retile_tile_indices, heap_key, data); + if (!entry) { + free(heap_key); + free(data); + free(indices); + } + return data; +} + +static uint32_t ac_compute_retile_tile_addr(const struct dcc_retile_tile_data *tile, + unsigned stride, unsigned x, unsigned y) +{ + unsigned x_mask = (1u << tile->tile_width_log2) - 1; + unsigned y_mask = (1u << tile->tile_height_log2) - 1; + unsigned tile_size_log2 = tile->tile_width_log2 + tile->tile_height_log2; + + unsigned base = ((y >> tile->tile_height_log2) * stride + (x >> tile->tile_width_log2)) << tile_size_log2; + unsigned offset_in_tile = tile->data[((y & y_mask) << tile->tile_width_log2) + (x & x_mask)]; + return base + offset_in_tile; +} + +static uint32_t *ac_compute_dcc_retile_map(struct ac_addrlib *addrlib, + const struct radeon_info *info, + unsigned retile_width, unsigned retile_height, + bool rb_aligned, bool pipe_aligned, bool use_uint16, + unsigned dcc_retile_num_elements, + const ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT *in) +{ + unsigned dcc_retile_map_size = dcc_retile_num_elements * (use_uint16 ? 2 : 4); + struct dcc_retile_map_key key; + + assert(in->numFrags == 1 && in->numSlices == 1 && in->numMipLevels == 1); + + memset(&key, 0, sizeof(key)); + key.family = info->family; + key.retile_width = retile_width; + key.retile_height = retile_height; + key.rb_aligned = rb_aligned; + key.pipe_aligned = pipe_aligned; + key.dcc_retile_num_elements = dcc_retile_num_elements; + memcpy(&key.input, in, sizeof(*in)); + + simple_mtx_lock(&addrlib->dcc_retile_map_lock); + + /* If we have already computed this retile map, get it from the hash table. */ + struct hash_entry *entry = _mesa_hash_table_search(addrlib->dcc_retile_maps, &key); + if (entry) { + uint32_t *map = entry->data; + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return map; + } + + const struct dcc_retile_tile_data *src_tile = + ac_compute_dcc_retile_tile_indices(addrlib, info, in->bpp, + in->swizzleMode, + rb_aligned, pipe_aligned); + const struct dcc_retile_tile_data *dst_tile = + ac_compute_dcc_retile_tile_indices(addrlib, info, in->bpp, + in->swizzleMode, false, false); + if (!src_tile || !dst_tile) { + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return NULL; + } + + void *dcc_retile_map = malloc(dcc_retile_map_size); + if (!dcc_retile_map) { + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return NULL; + } + + unsigned index = 0; + unsigned w = DIV_ROUND_UP(retile_width, in->compressBlkWidth); + unsigned h = DIV_ROUND_UP(retile_height, in->compressBlkHeight); + unsigned src_stride = DIV_ROUND_UP(w, 1u << src_tile->tile_width_log2); + unsigned dst_stride = DIV_ROUND_UP(w, 1u << dst_tile->tile_width_log2); + + for (unsigned y = 0; y < h; ++y) { + for (unsigned x = 0; x < w; ++x) { + unsigned src_addr = ac_compute_retile_tile_addr(src_tile, src_stride, x, y); + unsigned dst_addr = ac_compute_retile_tile_addr(dst_tile, dst_stride, x, y); + + if (use_uint16) { + ((uint16_t*)dcc_retile_map)[2 * index] = src_addr; + ((uint16_t*)dcc_retile_map)[2 * index + 1] = dst_addr; + } else { + ((uint32_t*)dcc_retile_map)[2 * index] = src_addr; + ((uint32_t*)dcc_retile_map)[2 * index + 1] = dst_addr; + } + ++index; + } + } + + /* Fill the remaining pairs with the last one (for the compute shader). */ + for (unsigned i = index * 2; i < dcc_retile_num_elements; i++) { + if (use_uint16) + ((uint16_t*)dcc_retile_map)[i] = ((uint16_t*)dcc_retile_map)[i - 2]; + else + ((uint32_t*)dcc_retile_map)[i] = ((uint32_t*)dcc_retile_map)[i - 2]; + } + + /* Insert the retile map into the hash table, so that it can be reused and + * the computation can be skipped for similar image sizes. + */ + _mesa_hash_table_insert(addrlib->dcc_retile_maps, + mem_dup(&key, sizeof(key)), dcc_retile_map); + + simple_mtx_unlock(&addrlib->dcc_retile_map_lock); + return dcc_retile_map; +} + static void *ADDR_API allocSysMem(const ADDR_ALLOCSYSMEM_INPUT * pInput) { return malloc(pInput->sizeInBytes); @@ -61,9 +340,9 @@ static ADDR_E_RETURNCODE ADDR_API freeSysMem(const ADDR_FREESYSMEM_INPUT * pInpu return ADDR_OK; } -ADDR_HANDLE amdgpu_addr_create(const struct radeon_info *info, - const struct amdgpu_gpu_info *amdinfo, - uint64_t *max_alignment) +struct ac_addrlib *ac_addrlib_create(const struct radeon_info *info, + const struct amdgpu_gpu_info *amdinfo, + uint64_t *max_alignment) { ADDR_CREATE_INPUT addrCreateInput = {0}; ADDR_CREATE_OUTPUT addrCreateOutput = {0}; @@ -123,7 +402,29 @@ ADDR_HANDLE amdgpu_addr_create(const struct radeon_info *info, *max_alignment = addrGetMaxAlignmentsOutput.baseAlign; } } - return addrCreateOutput.hLib; + + struct ac_addrlib *addrlib = calloc(1, sizeof(struct ac_addrlib)); + if (!addrlib) { + AddrDestroy(addrCreateOutput.hLib); + return NULL; + } + + addrlib->handle = addrCreateOutput.hLib; + simple_mtx_init(&addrlib->dcc_retile_map_lock, mtx_plain); + addrlib->dcc_retile_maps = _mesa_hash_table_create(NULL, dcc_retile_map_hash_key, + dcc_retile_map_keys_equal); + addrlib->dcc_retile_tile_indices = _mesa_hash_table_create(NULL, dcc_retile_tile_hash_key, + dcc_retile_tile_keys_equal); + return addrlib; +} + +void ac_addrlib_destroy(struct ac_addrlib *addrlib) +{ + AddrDestroy(addrlib->handle); + simple_mtx_destroy(&addrlib->dcc_retile_map_lock); + _mesa_hash_table_destroy(addrlib->dcc_retile_maps, dcc_retile_map_free); + _mesa_hash_table_destroy(addrlib->dcc_retile_tile_indices, dcc_retile_tile_free); + free(addrlib); } static int surf_config_sanity(const struct ac_surf_config *config, @@ -1071,8 +1372,12 @@ ASSERTED static bool is_dcc_supported_by_L2(const struct radeon_info *info, surf->u.gfx9.dcc.max_compressed_block_size <= V_028C78_MAX_BLOCK_SIZE_128B); } - unreachable("unhandled chip"); - return false; + /* 128B is recommended, but 64B can be set too if needed for 4K by DCN. + * Since there is no reason to ever disable 128B, require it. + * DCC image stores are always supported. + */ + return surf->u.gfx9.dcc.independent_128B_blocks && + surf->u.gfx9.dcc.max_compressed_block_size <= V_028C78_MAX_BLOCK_SIZE_128B; } static bool is_dcc_supported_by_DCN(const struct radeon_info *info, @@ -1103,11 +1408,14 @@ static bool is_dcc_supported_by_DCN(const struct radeon_info *info, surf->u.gfx9.dcc.max_compressed_block_size == V_028C78_MAX_BLOCK_SIZE_64B); return true; case GFX10: - /* DCN requires INDEPENDENT_128B_BLOCKS = 0. - * For 4K, it also requires INDEPENDENT_64B_BLOCKS = 1. - */ - return !surf->u.gfx9.dcc.independent_128B_blocks && - ((config->info.width <= 2560 && + case GFX10_3: + /* DCN requires INDEPENDENT_128B_BLOCKS = 0 only on Navi1x. */ + if (info->chip_class == GFX10 && + surf->u.gfx9.dcc.independent_128B_blocks) + return false; + + /* For 4K, DCN requires INDEPENDENT_64B_BLOCKS = 1. */ + return ((config->info.width <= 2560 && config->info.height <= 2560) || (surf->u.gfx9.dcc.independent_64B_blocks && surf->u.gfx9.dcc.max_compressed_block_size == V_028C78_MAX_BLOCK_SIZE_64B)); @@ -1117,7 +1425,7 @@ static bool is_dcc_supported_by_DCN(const struct radeon_info *info, } } -static int gfx9_compute_miptree(ADDR_HANDLE addrlib, +static int gfx9_compute_miptree(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, struct radeon_surf *surf, bool compressed, @@ -1130,7 +1438,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, out.size = sizeof(ADDR2_COMPUTE_SURFACE_INFO_OUTPUT); out.pMipInfo = mip_info; - ret = Addr2ComputeSurfaceInfo(addrlib, in, &out); + ret = Addr2ComputeSurfaceInfo(addrlib->handle, in, &out); if (ret != ADDR_OK) return ret; @@ -1156,15 +1464,28 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, surf->u.gfx9.surf_slice_size = out.sliceSize; surf->u.gfx9.surf_pitch = out.pitch; - if (!compressed && surf->blk_w > 1 && out.pitch == out.pixelPitch) { - /* Adjust surf_pitch to be in elements units, - * not in pixels */ - surf->u.gfx9.surf_pitch /= surf->blk_w; - } surf->u.gfx9.surf_height = out.height; surf->surf_size = out.surfSize; surf->surf_alignment = out.baseAlign; + if (!compressed && surf->blk_w > 1 && out.pitch == out.pixelPitch && + surf->u.gfx9.surf.swizzle_mode == ADDR_SW_LINEAR) { + /* Adjust surf_pitch to be in elements units not in pixels */ + surf->u.gfx9.surf_pitch = + align(surf->u.gfx9.surf_pitch / surf->blk_w, 256 / surf->bpe); + surf->u.gfx9.surf.epitch = MAX2(surf->u.gfx9.surf.epitch, + surf->u.gfx9.surf_pitch * surf->blk_w - 1); + /* The surface is really a surf->bpe bytes per pixel surface even if we + * use it as a surf->bpe bytes per element one. + * Adjust surf_slice_size and surf_size to reflect the change + * made to surf_pitch. + */ + surf->u.gfx9.surf_slice_size = MAX2( + surf->u.gfx9.surf_slice_size, + surf->u.gfx9.surf_pitch * out.height * surf->bpe * surf->blk_w); + surf->surf_size = surf->u.gfx9.surf_slice_size * in->numSlices; + } + if (in->swizzleMode == ADDR_SW_LINEAR) { for (unsigned i = 0; i < in->numMipLevels; i++) { surf->u.gfx9.offset[i] = mip_info[i].offset; @@ -1198,7 +1519,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, hin.numMipLevels = in->numMipLevels; hin.firstMipIdInTail = out.firstMipIdInTail; - ret = Addr2ComputeHtileInfo(addrlib, &hin, &hout); + ret = Addr2ComputeHtileInfo(addrlib->handle, &hin, &hout); if (ret != ADDR_OK) return ret; @@ -1231,7 +1552,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, xin.numSamples = in->numSamples; xin.numFrags = in->numFrags; - ret = Addr2ComputePipeBankXor(addrlib, &xin, &xout); + ret = Addr2ComputePipeBankXor(addrlib->handle, &xin, &xout); if (ret != ADDR_OK) return ret; @@ -1259,7 +1580,6 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, din.dccKeyFlags.pipeAligned = !in->flags.metaPipeUnaligned; din.dccKeyFlags.rbAligned = !in->flags.metaRbUnaligned; - din.colorFlags = in->flags; din.resourceType = in->resourceType; din.swizzleMode = in->swizzleMode; din.bpp = in->bpp; @@ -1271,7 +1591,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, din.dataSurfaceSize = out.surfSize; din.firstMipIdInTail = out.firstMipIdInTail; - ret = Addr2ComputeDccInfo(addrlib, &din, &dout); + ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); if (ret != ADDR_OK) return ret; @@ -1308,7 +1628,16 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, */ for (unsigned i = 0; i < in->numMipLevels; i++) { if (meta_mip_info[i].inMiptail) { - surf->num_dcc_levels = i; + /* GFX10 can only compress the first level + * in the mip tail. + * + * TODO: Try to do the same thing for gfx9 + * if there are no regressions. + */ + if (info->chip_class >= GFX10) + surf->num_dcc_levels = i + 1; + else + surf->num_dcc_levels = i; break; } } @@ -1335,7 +1664,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, assert(surf->u.gfx9.dcc.pipe_aligned || surf->u.gfx9.dcc.rb_aligned); - ret = Addr2ComputeDccInfo(addrlib, &din, &dout); + ret = Addr2ComputeDccInfo(addrlib->handle, &din, &dout); if (ret != ADDR_OK) return ret; @@ -1344,76 +1673,91 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, surf->u.gfx9.display_dcc_pitch_max = dout.pitch - 1; assert(surf->u.gfx9.display_dcc_size <= surf->dcc_size); - /* Compute address mapping from non-displayable to displayable DCC. */ - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin = {}; - addrin.size = sizeof(addrin); - addrin.colorFlags.color = 1; - addrin.swizzleMode = din.swizzleMode; - addrin.resourceType = din.resourceType; - addrin.bpp = din.bpp; - addrin.unalignedWidth = din.unalignedWidth; - addrin.unalignedHeight = din.unalignedHeight; - addrin.numSlices = 1; - addrin.numMipLevels = 1; - addrin.numFrags = 1; - - ADDR2_COMPUTE_DCC_ADDRFROMCOORD_OUTPUT addrout = {}; - addrout.size = sizeof(addrout); + surf->u.gfx9.dcc_retile_use_uint16 = + surf->u.gfx9.display_dcc_size <= UINT16_MAX + 1 && + surf->dcc_size <= UINT16_MAX + 1; + + /* Align the retile map size to get more hash table hits and + * decrease the maximum memory footprint when all retile maps + * are cached in the hash table. + */ + unsigned retile_dim[2] = {in->width, in->height}; + + for (unsigned i = 0; i < 2; i++) { + /* Increase the alignment as the size increases. + * Greater alignment increases retile compute work, + * but decreases maximum memory footprint for the cache. + * + * With this alignment, the worst case memory footprint of + * the cache is: + * 1920x1080: 55 MB + * 2560x1440: 99 MB + * 3840x2160: 305 MB + * + * The worst case size in MB can be computed in Haskell as follows: + * (sum (map get_retile_size (map get_dcc_size (deduplicate (map align_pair + * [(i*16,j*16) | i <- [1..maxwidth`div`16], j <- [1..maxheight`div`16]]))))) `div` 1024^2 + * where + * alignment x = if x <= 512 then 16 else if x <= 1024 then 32 else if x <= 2048 then 64 else 128 + * align x = (x + (alignment x) - 1) `div` (alignment x) * (alignment x) + * align_pair e = (align (fst e), align (snd e)) + * deduplicate = map head . groupBy (\ a b -> ((fst a) == (fst b)) && ((snd a) == (snd b))) . sortBy compare + * get_dcc_size e = ((fst e) * (snd e) * bpp) `div` 256 + * get_retile_size dcc_size = dcc_size * 2 * (if dcc_size <= 2^16 then 2 else 4) + * bpp = 4; maxwidth = 3840; maxheight = 2160 + */ + if (retile_dim[i] <= 512) + retile_dim[i] = align(retile_dim[i], 16); + else if (retile_dim[i] <= 1024) + retile_dim[i] = align(retile_dim[i], 32); + else if (retile_dim[i] <= 2048) + retile_dim[i] = align(retile_dim[i], 64); + else + retile_dim[i] = align(retile_dim[i], 128); + + /* Don't align more than the DCC pixel alignment. */ + assert(dout.metaBlkWidth >= 128 && dout.metaBlkHeight >= 128); + } surf->u.gfx9.dcc_retile_num_elements = - DIV_ROUND_UP(in->width, dout.compressBlkWidth) * - DIV_ROUND_UP(in->height, dout.compressBlkHeight) * 2; + DIV_ROUND_UP(retile_dim[0], dout.compressBlkWidth) * + DIV_ROUND_UP(retile_dim[1], dout.compressBlkHeight) * 2; /* Align the size to 4 (for the compute shader). */ surf->u.gfx9.dcc_retile_num_elements = align(surf->u.gfx9.dcc_retile_num_elements, 4); - surf->u.gfx9.dcc_retile_map = - malloc(surf->u.gfx9.dcc_retile_num_elements * 4); - if (!surf->u.gfx9.dcc_retile_map) - return ADDR_OUTOFMEMORY; - - unsigned index = 0; - surf->u.gfx9.dcc_retile_use_uint16 = true; - - for (unsigned y = 0; y < in->height; y += dout.compressBlkHeight) { - addrin.y = y; - - for (unsigned x = 0; x < in->width; x += dout.compressBlkWidth) { - addrin.x = x; - - /* Compute src DCC address */ - addrin.dccKeyFlags.pipeAligned = surf->u.gfx9.dcc.pipe_aligned; - addrin.dccKeyFlags.rbAligned = surf->u.gfx9.dcc.rb_aligned; - addrout.addr = 0; - - ret = Addr2ComputeDccAddrFromCoord(addrlib, &addrin, &addrout); - if (ret != ADDR_OK) - return ret; - - surf->u.gfx9.dcc_retile_map[index * 2] = addrout.addr; - if (addrout.addr > UINT16_MAX) - surf->u.gfx9.dcc_retile_use_uint16 = false; - - /* Compute dst DCC address */ - addrin.dccKeyFlags.pipeAligned = 0; - addrin.dccKeyFlags.rbAligned = 0; - addrout.addr = 0; - - ret = Addr2ComputeDccAddrFromCoord(addrlib, &addrin, &addrout); - if (ret != ADDR_OK) - return ret; - - surf->u.gfx9.dcc_retile_map[index * 2 + 1] = addrout.addr; - if (addrout.addr > UINT16_MAX) - surf->u.gfx9.dcc_retile_use_uint16 = false; - - assert(index * 2 + 1 < surf->u.gfx9.dcc_retile_num_elements); - index++; - } + if (!(surf->flags & RADEON_SURF_IMPORTED)) { + /* Compute address mapping from non-displayable to displayable DCC. */ + ADDR2_COMPUTE_DCC_ADDRFROMCOORD_INPUT addrin; + memset(&addrin, 0, sizeof(addrin)); + addrin.size = sizeof(addrin); + addrin.swizzleMode = din.swizzleMode; + addrin.resourceType = din.resourceType; + addrin.bpp = din.bpp; + addrin.numSlices = 1; + addrin.numMipLevels = 1; + addrin.numFrags = 1; + addrin.pitch = dout.pitch; + addrin.height = dout.height; + addrin.compressBlkWidth = dout.compressBlkWidth; + addrin.compressBlkHeight = dout.compressBlkHeight; + addrin.compressBlkDepth = dout.compressBlkDepth; + addrin.metaBlkWidth = dout.metaBlkWidth; + addrin.metaBlkHeight = dout.metaBlkHeight; + addrin.metaBlkDepth = dout.metaBlkDepth; + addrin.dccRamSliceSize = 0; /* Don't care for non-layered images. */ + + surf->u.gfx9.dcc_retile_map = + ac_compute_dcc_retile_map(addrlib, info, + retile_dim[0], retile_dim[1], + surf->u.gfx9.dcc.rb_aligned, + surf->u.gfx9.dcc.pipe_aligned, + surf->u.gfx9.dcc_retile_use_uint16, + surf->u.gfx9.dcc_retile_num_elements, + &addrin); + if (!surf->u.gfx9.dcc_retile_map) + return ADDR_OUTOFMEMORY; } - /* Fill the remaining pairs with the last one (for the compute shader). */ - for (unsigned i = index * 2; i < surf->u.gfx9.dcc_retile_num_elements; i++) - surf->u.gfx9.dcc_retile_map[i] = surf->u.gfx9.dcc_retile_map[i - 2]; } } @@ -1426,7 +1770,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, fin.size = sizeof(ADDR2_COMPUTE_FMASK_INFO_INPUT); fout.size = sizeof(ADDR2_COMPUTE_FMASK_INFO_OUTPUT); - ret = gfx9_get_preferred_swizzle_mode(addrlib, surf, in, + ret = gfx9_get_preferred_swizzle_mode(addrlib->handle, surf, in, true, &fin.swizzleMode); if (ret != ADDR_OK) return ret; @@ -1437,7 +1781,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, fin.numSamples = in->numSamples; fin.numFrags = in->numFrags; - ret = Addr2ComputeFmaskInfo(addrlib, &fin, &fout); + ret = Addr2ComputeFmaskInfo(addrlib->handle, &fin, &fout); if (ret != ADDR_OK) return ret; @@ -1465,7 +1809,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, xin.numSamples = in->numSamples; xin.numFrags = in->numFrags; - ret = Addr2ComputePipeBankXor(addrlib, &xin, &xout); + ret = Addr2ComputePipeBankXor(addrlib->handle, &xin, &xout); if (ret != ADDR_OK) return ret; @@ -1494,7 +1838,6 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, cin.cMaskFlags.pipeAligned = 1; cin.cMaskFlags.rbAligned = 1; - cin.colorFlags = in->flags; cin.resourceType = in->resourceType; cin.unalignedWidth = in->width; cin.unalignedHeight = in->height; @@ -1505,7 +1848,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, else cin.swizzleMode = in->swizzleMode; - ret = Addr2ComputeCmaskInfo(addrlib, &cin, &cout); + ret = Addr2ComputeCmaskInfo(addrlib->handle, &cin, &cout); if (ret != ADDR_OK) return ret; @@ -1517,7 +1860,7 @@ static int gfx9_compute_miptree(ADDR_HANDLE addrlib, return 0; } -static int gfx9_compute_surface(ADDR_HANDLE addrlib, +static int gfx9_compute_surface(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, enum radeon_surf_mode mode, @@ -1655,6 +1998,12 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, surf->u.gfx9.dcc.independent_128B_blocks = 0; surf->u.gfx9.dcc.max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B; } + + if (info->chip_class >= GFX10_3) { + surf->u.gfx9.dcc.independent_64B_blocks = 1; + surf->u.gfx9.dcc.independent_128B_blocks = 1; + surf->u.gfx9.dcc.max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B; + } } } @@ -1674,7 +2023,7 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, break; } - r = gfx9_get_preferred_swizzle_mode(addrlib, surf, &AddrSurfInfoIn, + r = gfx9_get_preferred_swizzle_mode(addrlib->handle, surf, &AddrSurfInfoIn, false, &AddrSurfInfoIn.swizzleMode); if (r) return r; @@ -1704,7 +2053,7 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, r = gfx9_compute_miptree(addrlib, info, config, surf, compressed, &AddrSurfInfoIn); if (r) - goto error; + return r; /* Calculate texture layout information for stencil. */ if (surf->flags & RADEON_SURF_SBUFFER) { @@ -1713,17 +2062,17 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, AddrSurfInfoIn.format = ADDR_FMT_8; if (!AddrSurfInfoIn.flags.depth) { - r = gfx9_get_preferred_swizzle_mode(addrlib, surf, &AddrSurfInfoIn, + r = gfx9_get_preferred_swizzle_mode(addrlib->handle, surf, &AddrSurfInfoIn, false, &AddrSurfInfoIn.swizzleMode); if (r) - goto error; + return r; } else AddrSurfInfoIn.flags.depth = 0; r = gfx9_compute_miptree(addrlib, info, config, surf, compressed, &AddrSurfInfoIn); if (r) - goto error; + return r; } surf->is_linear = surf->u.gfx9.surf.swizzle_mode == ADDR_SW_LINEAR; @@ -1732,16 +2081,19 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, /* This is only useful for surfaces that are allocated without SCANOUT. */ bool displayable = false; if (!config->is_3d && !config->is_cube) { - r = Addr2IsValidDisplaySwizzleMode(addrlib, surf->u.gfx9.surf.swizzle_mode, - surf->bpe * 8, &displayable); + r = Addr2IsValidDisplaySwizzleMode(addrlib->handle, surf->u.gfx9.surf.swizzle_mode, + surf->bpe * 8, &displayable); if (r) - goto error; + return r; /* Display needs unaligned DCC. */ if (surf->num_dcc_levels && - !is_dcc_supported_by_DCN(info, config, surf, - surf->u.gfx9.dcc.rb_aligned, - surf->u.gfx9.dcc.pipe_aligned)) + (!is_dcc_supported_by_DCN(info, config, surf, + surf->u.gfx9.dcc.rb_aligned, + surf->u.gfx9.dcc.pipe_aligned) || + /* Don't set is_displayable if displayable DCC is missing. */ + (info->use_display_dcc_with_retile_blit && + !surf->u.gfx9.dcc_retile_num_elements))) displayable = false; } surf->is_displayable = displayable; @@ -1843,14 +2195,9 @@ static int gfx9_compute_surface(ADDR_HANDLE addrlib, } return 0; - -error: - free(surf->u.gfx9.dcc_retile_map); - surf->u.gfx9.dcc_retile_map = NULL; - return r; } -int ac_compute_surface(ADDR_HANDLE addrlib, const struct radeon_info *info, +int ac_compute_surface(struct ac_addrlib *addrlib, const struct radeon_info *info, const struct ac_surf_config *config, enum radeon_surf_mode mode, struct radeon_surf *surf) @@ -1864,31 +2211,38 @@ int ac_compute_surface(ADDR_HANDLE addrlib, const struct radeon_info *info, if (info->chip_class >= GFX9) r = gfx9_compute_surface(addrlib, info, config, mode, surf); else - r = gfx6_compute_surface(addrlib, info, config, mode, surf); + r = gfx6_compute_surface(addrlib->handle, info, config, mode, surf); if (r) return r; /* Determine the memory layout of multiple allocations in one buffer. */ surf->total_size = surf->surf_size; + surf->alignment = surf->surf_alignment; if (surf->htile_size) { surf->htile_offset = align64(surf->total_size, surf->htile_alignment); surf->total_size = surf->htile_offset + surf->htile_size; + surf->alignment = MAX2(surf->alignment, surf->htile_alignment); } if (surf->fmask_size) { assert(config->info.samples >= 2); surf->fmask_offset = align64(surf->total_size, surf->fmask_alignment); surf->total_size = surf->fmask_offset + surf->fmask_size; + surf->alignment = MAX2(surf->alignment, surf->fmask_alignment); } /* Single-sample CMASK is in a separate buffer. */ if (surf->cmask_size && config->info.samples >= 2) { surf->cmask_offset = align64(surf->total_size, surf->cmask_alignment); surf->total_size = surf->cmask_offset + surf->cmask_size; + surf->alignment = MAX2(surf->alignment, surf->cmask_alignment); } + if (surf->is_displayable) + surf->flags |= RADEON_SURF_SCANOUT; + if (surf->dcc_size && /* dcc_size is computed on GFX9+ only if it's displayable. */ (info->chip_class >= GFX9 || !get_display_flag(config, surf))) { @@ -1918,6 +2272,7 @@ int ac_compute_surface(ADDR_HANDLE addrlib, const struct radeon_info *info, surf->dcc_offset = align64(surf->total_size, surf->dcc_alignment); surf->total_size = surf->dcc_offset + surf->dcc_size; + surf->alignment = MAX2(surf->alignment, surf->dcc_alignment); } return 0; @@ -2122,6 +2477,7 @@ bool ac_surface_set_umd_metadata(const struct radeon_info *info, break; case GFX10: + case GFX10_3: surf->dcc_offset = ((uint64_t)G_00A018_META_DATA_ADDRESS_LO(desc[6]) << 8) | ((uint64_t)desc[7] << 16); surf->u.gfx9.dcc.pipe_aligned = G_00A018_META_PIPE_ALIGNED(desc[6]); @@ -2164,6 +2520,7 @@ void ac_surface_get_umd_metadata(const struct radeon_info *info, desc[5] |= S_008F24_META_DATA_ADDRESS(surf->dcc_offset >> 40); break; case GFX10: + case GFX10_3: desc[6] &= C_00A018_META_DATA_ADDRESS_LO; desc[6] |= S_00A018_META_DATA_ADDRESS_LO(surf->dcc_offset >> 8); desc[7] = surf->dcc_offset >> 16;