src/amd/vulkan/radv_image.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  25  * IN THE SOFTWARE.
  26  */
  27
  28 #include "radv_private.h"
  29 #include "vk_format.h"
  30 #include "radv_radeon_winsys.h"
  31 #include "sid.h"
  32 #include "util/debug.h"
  33 static unsigned
  34 radv_choose_tiling(struct radv_device *Device,
  35                    const struct radv_image_create_info *create_info)
  36 {
  37         const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
  38
  39         if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR) {
  40                 assert(pCreateInfo->samples <= 1);
  41                 return RADEON_SURF_MODE_LINEAR_ALIGNED;
  42         }
  43
  44         /* Textures with a very small height are recommended to be linear. */
  45         if (pCreateInfo->imageType == VK_IMAGE_TYPE_1D ||
  46             /* Only very thin and long 2D textures should benefit from
  47              * linear_aligned. */
  48             (pCreateInfo->extent.width > 8 && pCreateInfo->extent.height <= 2))
  49                 return RADEON_SURF_MODE_LINEAR_ALIGNED;
  50
  51         /* MSAA resources must be 2D tiled. */
  52         if (pCreateInfo->samples > 1)
  53                 return RADEON_SURF_MODE_2D;
  54
  55         return RADEON_SURF_MODE_2D;
  56 }
  57 static int
  58 radv_init_surface(struct radv_device *device,
  59                   struct radeon_surf *surface,
  60                   const struct radv_image_create_info *create_info)
  61 {
  62         const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
  63         unsigned array_mode = radv_choose_tiling(device, create_info);
  64         const struct vk_format_description *desc =
  65                 vk_format_description(pCreateInfo->format);
  66         bool is_depth, is_stencil, blendable;
  67
  68         is_depth = vk_format_has_depth(desc);
  69         is_stencil = vk_format_has_stencil(desc);
  70
  71         surface->blk_w = vk_format_get_blockwidth(pCreateInfo->format);
  72         surface->blk_h = vk_format_get_blockheight(pCreateInfo->format);
  73
  74         surface->bpe = vk_format_get_blocksize(pCreateInfo->format);
  75         /* align byte per element on dword */
  76         if (surface->bpe == 3) {
  77                 surface->bpe = 4;
  78         }
  79         surface->flags = RADEON_SURF_SET(array_mode, MODE);
  80
  81         switch (pCreateInfo->imageType){
  82         case VK_IMAGE_TYPE_1D:
  83                 if (pCreateInfo->arrayLayers > 1)
  84                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D_ARRAY, TYPE);
  85                 else
  86                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D, TYPE);
  87                 break;
  88         case VK_IMAGE_TYPE_2D:
  89                 if (pCreateInfo->arrayLayers > 1)
  90                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D_ARRAY, TYPE);
  91                 else
  92                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D, TYPE);
  93                 break;
  94         case VK_IMAGE_TYPE_3D:
  95                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_3D, TYPE);
  96                 break;
  97         default:
  98                 unreachable("unhandled image type");
  99         }
 100
 101         if (is_depth) {
 102                 surface->flags |= RADEON_SURF_ZBUFFER;
 103         }
 104
 105         if (is_stencil)
 106                 surface->flags |= RADEON_SURF_SBUFFER;
 107
 108         surface->flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
 109
 110         if ((pCreateInfo->usage & (VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
 111                                    VK_IMAGE_USAGE_STORAGE_BIT)) ||
 112             (pCreateInfo->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) ||
 113             (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR) ||
 114             device->physical_device->rad_info.chip_class < VI ||
 115             create_info->scanout || (device->debug_flags & RADV_DEBUG_NO_DCC) ||
 116             !radv_is_colorbuffer_format_supported(pCreateInfo->format, &blendable))
 117                 surface->flags |= RADEON_SURF_DISABLE_DCC;
 118         if (create_info->scanout)
 119                 surface->flags |= RADEON_SURF_SCANOUT;
 120         return 0;
 121 }
 122 #define ATI_VENDOR_ID 0x1002
 123 static uint32_t si_get_bo_metadata_word1(struct radv_device *device)
 124 {
 125         return (ATI_VENDOR_ID << 16) | device->physical_device->rad_info.pci_id;
 126 }
 127
 128 static inline unsigned
 129 si_tile_mode_index(const struct radv_image *image, unsigned level, bool stencil)
 130 {
 131         if (stencil)
 132                 return image->surface.stencil_tiling_index[level];
 133         else
 134                 return image->surface.tiling_index[level];
 135 }
 136
 137 static unsigned radv_map_swizzle(unsigned swizzle)
 138 {
 139         switch (swizzle) {
 140         case VK_SWIZZLE_Y:
 141                 return V_008F0C_SQ_SEL_Y;
 142         case VK_SWIZZLE_Z:
 143                 return V_008F0C_SQ_SEL_Z;
 144         case VK_SWIZZLE_W:
 145                 return V_008F0C_SQ_SEL_W;
 146         case VK_SWIZZLE_0:
 147                 return V_008F0C_SQ_SEL_0;
 148         case VK_SWIZZLE_1:
 149                 return V_008F0C_SQ_SEL_1;
 150         default: /* VK_SWIZZLE_X */
 151                 return V_008F0C_SQ_SEL_X;
 152         }
 153 }
 154
 155 static void
 156 radv_make_buffer_descriptor(struct radv_device *device,
 157                             struct radv_buffer *buffer,
 158                             VkFormat vk_format,
 159                             unsigned offset,
 160                             unsigned range,
 161                             uint32_t *state)
 162 {
 163         const struct vk_format_description *desc;
 164         unsigned stride;
 165         uint64_t gpu_address = device->ws->buffer_get_va(buffer->bo);
 166         uint64_t va = gpu_address + buffer->offset;
 167         unsigned num_format, data_format;
 168         int first_non_void;
 169         desc = vk_format_description(vk_format);
 170         first_non_void = vk_format_get_first_non_void_channel(vk_format);
 171         stride = desc->block.bits / 8;
 172
 173         num_format = radv_translate_buffer_numformat(desc, first_non_void);
 174         data_format = radv_translate_buffer_dataformat(desc, first_non_void);
 175
 176         va += offset;
 177         state[0] = va;
 178         state[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
 179                 S_008F04_STRIDE(stride);
 180         state[2] = range;
 181         state[3] = S_008F0C_DST_SEL_X(radv_map_swizzle(desc->swizzle[0])) |
 182                    S_008F0C_DST_SEL_Y(radv_map_swizzle(desc->swizzle[1])) |
 183                    S_008F0C_DST_SEL_Z(radv_map_swizzle(desc->swizzle[2])) |
 184                    S_008F0C_DST_SEL_W(radv_map_swizzle(desc->swizzle[3])) |
 185                    S_008F0C_NUM_FORMAT(num_format) |
 186                    S_008F0C_DATA_FORMAT(data_format);
 187 }
 188
 189 static void
 190 si_set_mutable_tex_desc_fields(struct radv_device *device,
 191                                struct radv_image *image,
 192                                const struct radeon_surf_level *base_level_info,
 193                                unsigned base_level, unsigned first_level,
 194                                unsigned block_width, bool is_stencil,
 195                                uint32_t *state)
 196 {
 197         uint64_t gpu_address = device->ws->buffer_get_va(image->bo) + image->offset;
 198         uint64_t va = gpu_address + base_level_info->offset;
 199         unsigned pitch = base_level_info->nblk_x * block_width;
 200
 201         state[1] &= C_008F14_BASE_ADDRESS_HI;
 202         state[3] &= C_008F1C_TILING_INDEX;
 203         state[4] &= C_008F20_PITCH_GFX6;
 204         state[6] &= C_008F28_COMPRESSION_EN;
 205
 206         assert(!(va & 255));
 207
 208         state[0] = va >> 8;
 209         state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
 210         state[3] |= S_008F1C_TILING_INDEX(si_tile_mode_index(image, base_level,
 211                                                              is_stencil));
 212         state[4] |= S_008F20_PITCH_GFX6(pitch - 1);
 213
 214         if (image->surface.dcc_size && first_level < image->surface.num_dcc_levels) {
 215                 state[6] |= S_008F28_COMPRESSION_EN(1);
 216                 state[7] = (gpu_address +
 217                             image->dcc_offset +
 218                             base_level_info->dcc_offset) >> 8;
 219         }
 220 }
 221
 222 static unsigned radv_tex_dim(VkImageType image_type, VkImageViewType view_type,
 223                              unsigned nr_layers, unsigned nr_samples, bool is_storage_image)
 224 {
 225         if (view_type == VK_IMAGE_VIEW_TYPE_CUBE || view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)
 226                 return is_storage_image ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY : V_008F1C_SQ_RSRC_IMG_CUBE;
 227         switch (image_type) {
 228         case VK_IMAGE_TYPE_1D:
 229                 return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_1D_ARRAY : V_008F1C_SQ_RSRC_IMG_1D;
 230         case VK_IMAGE_TYPE_2D:
 231                 if (nr_samples > 1)
 232                         return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY : V_008F1C_SQ_RSRC_IMG_2D_MSAA;
 233                 else
 234                         return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY : V_008F1C_SQ_RSRC_IMG_2D;
 235         case VK_IMAGE_TYPE_3D:
 236                 if (view_type == VK_IMAGE_VIEW_TYPE_3D)
 237                         return V_008F1C_SQ_RSRC_IMG_3D;
 238                 else
 239                         return V_008F1C_SQ_RSRC_IMG_2D_ARRAY;
 240         default:
 241                 unreachable("illegale image type");
 242         }
 243 }
 244 /**
 245  * Build the sampler view descriptor for a texture.
 246  */
 247 static void
 248 si_make_texture_descriptor(struct radv_device *device,
 249                            struct radv_image *image,
 250                            bool sampler,
 251                            VkImageViewType view_type,
 252                            VkFormat vk_format,
 253                            const VkComponentMapping *mapping,
 254                            unsigned first_level, unsigned last_level,
 255                            unsigned first_layer, unsigned last_layer,
 256                            unsigned width, unsigned height, unsigned depth,
 257                            uint32_t *state,
 258                            uint32_t *fmask_state)
 259 {
 260         const struct vk_format_description *desc;
 261         enum vk_swizzle swizzle[4];
 262         int first_non_void;
 263         unsigned num_format, data_format, type;
 264
 265         desc = vk_format_description(vk_format);
 266
 267         if (desc->colorspace == VK_FORMAT_COLORSPACE_ZS) {
 268                 const unsigned char swizzle_xxxx[4] = {0, 0, 0, 0};
 269                 vk_format_compose_swizzles(mapping, swizzle_xxxx, swizzle);
 270         } else {
 271                 vk_format_compose_swizzles(mapping, desc->swizzle, swizzle);
 272         }
 273
 274         first_non_void = vk_format_get_first_non_void_channel(vk_format);
 275
 276         num_format = radv_translate_tex_numformat(vk_format, desc, first_non_void);
 277         if (num_format == ~0) {
 278                 num_format = 0;
 279         }
 280
 281         data_format = radv_translate_tex_dataformat(vk_format, desc, first_non_void);
 282         if (data_format == ~0) {
 283                 data_format = 0;
 284         }
 285
 286         type = radv_tex_dim(image->type, view_type, image->info.array_size, image->info.samples,
 287                             (image->usage & VK_IMAGE_USAGE_STORAGE_BIT));
 288         if (type == V_008F1C_SQ_RSRC_IMG_1D_ARRAY) {
 289                 height = 1;
 290                 depth = image->info.array_size;
 291         } else if (type == V_008F1C_SQ_RSRC_IMG_2D_ARRAY ||
 292                    type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
 293                 if (view_type != VK_IMAGE_VIEW_TYPE_3D)
 294                         depth = image->info.array_size;
 295         } else if (type == V_008F1C_SQ_RSRC_IMG_CUBE)
 296                 depth = image->info.array_size / 6;
 297
 298         state[0] = 0;
 299         state[1] = (S_008F14_DATA_FORMAT_GFX6(data_format) |
 300                     S_008F14_NUM_FORMAT_GFX6(num_format));
 301         state[2] = (S_008F18_WIDTH(width - 1) |
 302                     S_008F18_HEIGHT(height - 1) |
 303                     S_008F18_PERF_MOD(4));
 304         state[3] = (S_008F1C_DST_SEL_X(radv_map_swizzle(swizzle[0])) |
 305                     S_008F1C_DST_SEL_Y(radv_map_swizzle(swizzle[1])) |
 306                     S_008F1C_DST_SEL_Z(radv_map_swizzle(swizzle[2])) |
 307                     S_008F1C_DST_SEL_W(radv_map_swizzle(swizzle[3])) |
 308                     S_008F1C_BASE_LEVEL(image->info.samples > 1 ?
 309                                         0 : first_level) |
 310                     S_008F1C_LAST_LEVEL(image->info.samples > 1 ?
 311                                         util_logbase2(image->info.samples) :
 312                                         last_level) |
 313                     S_008F1C_POW2_PAD(image->info.levels > 1) |
 314                     S_008F1C_TYPE(type));
 315         state[4] = S_008F20_DEPTH(depth - 1);
 316         state[5] = (S_008F24_BASE_ARRAY(first_layer) |
 317                     S_008F24_LAST_ARRAY(last_layer));
 318         state[6] = 0;
 319         state[7] = 0;
 320
 321         if (image->dcc_offset) {
 322                 unsigned swap = radv_translate_colorswap(vk_format, FALSE);
 323
 324                 state[6] = S_008F28_ALPHA_IS_ON_MSB(swap <= 1);
 325         } else {
 326                 /* The last dword is unused by hw. The shader uses it to clear
 327                  * bits in the first dword of sampler state.
 328                  */
 329                 if (device->physical_device->rad_info.chip_class <= CIK && image->info.samples <= 1) {
 330                         if (first_level == last_level)
 331                                 state[7] = C_008F30_MAX_ANISO_RATIO;
 332                         else
 333                                 state[7] = 0xffffffff;
 334                 }
 335         }
 336
 337         /* Initialize the sampler view for FMASK. */
 338         if (image->fmask.size) {
 339                 uint32_t fmask_format;
 340                 uint64_t gpu_address = device->ws->buffer_get_va(image->bo);
 341                 uint64_t va;
 342
 343                 va = gpu_address + image->offset + image->fmask.offset;
 344
 345                 switch (image->info.samples) {
 346                 case 2:
 347                         fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK8_S2_F2;
 348                         break;
 349                 case 4:
 350                         fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK8_S4_F4;
 351                         break;
 352                 case 8:
 353                         fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK32_S8_F8;
 354                         break;
 355                 default:
 356                         assert(0);
 357                         fmask_format = V_008F14_IMG_DATA_FORMAT_INVALID;
 358                 }
 359
 360                 fmask_state[0] = va >> 8;
 361                 fmask_state[1] = S_008F14_BASE_ADDRESS_HI(va >> 40) |
 362                         S_008F14_DATA_FORMAT_GFX6(fmask_format) |
 363                         S_008F14_NUM_FORMAT_GFX6(V_008F14_IMG_NUM_FORMAT_UINT);
 364                 fmask_state[2] = S_008F18_WIDTH(width - 1) |
 365                         S_008F18_HEIGHT(height - 1);
 366                 fmask_state[3] = S_008F1C_DST_SEL_X(V_008F1C_SQ_SEL_X) |
 367                         S_008F1C_DST_SEL_Y(V_008F1C_SQ_SEL_X) |
 368                         S_008F1C_DST_SEL_Z(V_008F1C_SQ_SEL_X) |
 369                         S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_X) |
 370                         S_008F1C_TILING_INDEX(image->fmask.tile_mode_index) |
 371                         S_008F1C_TYPE(radv_tex_dim(image->type, view_type, 1, 0, false));
 372                 fmask_state[4] = S_008F20_DEPTH(depth - 1) |
 373                         S_008F20_PITCH_GFX6(image->fmask.pitch_in_pixels - 1);
 374                 fmask_state[5] = S_008F24_BASE_ARRAY(first_layer) |
 375                         S_008F24_LAST_ARRAY(last_layer);
 376                 fmask_state[6] = 0;
 377                 fmask_state[7] = 0;
 378         }
 379 }
 380
 381 static void
 382 radv_query_opaque_metadata(struct radv_device *device,
 383                            struct radv_image *image,
 384                            struct radeon_bo_metadata *md)
 385 {
 386         static const VkComponentMapping fixedmapping;
 387         uint32_t desc[8], i;
 388
 389         /* Metadata image format format version 1:
 390          * [0] = 1 (metadata format identifier)
 391          * [1] = (VENDOR_ID << 16) | PCI_ID
 392          * [2:9] = image descriptor for the whole resource
 393          *         [2] is always 0, because the base address is cleared
 394          *         [9] is the DCC offset bits [39:8] from the beginning of
 395          *             the buffer
 396          * [10:10+LAST_LEVEL] = mipmap level offset bits [39:8] for each level
 397          */
 398         md->metadata[0] = 1; /* metadata image format version 1 */
 399
 400         /* TILE_MODE_INDEX is ambiguous without a PCI ID. */
 401         md->metadata[1] = si_get_bo_metadata_word1(device);
 402
 403
 404         si_make_texture_descriptor(device, image, true,
 405                                    (VkImageViewType)image->type, image->vk_format,
 406                                    &fixedmapping, 0, image->info.levels - 1, 0,
 407                                    image->info.array_size,
 408                                    image->info.width, image->info.height,
 409                                    image->info.depth,
 410                                    desc, NULL);
 411
 412         si_set_mutable_tex_desc_fields(device, image, &image->surface.level[0], 0, 0,
 413                                        image->surface.blk_w, false, desc);
 414
 415         /* Clear the base address and set the relative DCC offset. */
 416         desc[0] = 0;
 417         desc[1] &= C_008F14_BASE_ADDRESS_HI;
 418         desc[7] = image->dcc_offset >> 8;
 419
 420         /* Dwords [2:9] contain the image descriptor. */
 421         memcpy(&md->metadata[2], desc, sizeof(desc));
 422
 423         /* Dwords [10:..] contain the mipmap level offsets. */
 424         for (i = 0; i <= image->info.levels - 1; i++)
 425                 md->metadata[10+i] = image->surface.level[i].offset >> 8;
 426
 427         md->size_metadata = (11 + image->info.levels - 1) * 4;
 428 }
 429
 430 void
 431 radv_init_metadata(struct radv_device *device,
 432                    struct radv_image *image,
 433                    struct radeon_bo_metadata *metadata)
 434 {
 435         struct radeon_surf *surface = &image->surface;
 436
 437         memset(metadata, 0, sizeof(*metadata));
 438         metadata->microtile = surface->level[0].mode >= RADEON_SURF_MODE_1D ?
 439                 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
 440         metadata->macrotile = surface->level[0].mode >= RADEON_SURF_MODE_2D ?
 441                 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
 442         metadata->pipe_config = surface->pipe_config;
 443         metadata->bankw = surface->bankw;
 444         metadata->bankh = surface->bankh;
 445         metadata->tile_split = surface->tile_split;
 446         metadata->mtilea = surface->mtilea;
 447         metadata->num_banks = surface->num_banks;
 448         metadata->stride = surface->level[0].nblk_x * surface->bpe;
 449         metadata->scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
 450
 451         radv_query_opaque_metadata(device, image, metadata);
 452 }
 453
 454 /* The number of samples can be specified independently of the texture. */
 455 static void
 456 radv_image_get_fmask_info(struct radv_device *device,
 457                           struct radv_image *image,
 458                           unsigned nr_samples,
 459                           struct radv_fmask_info *out)
 460 {
 461         /* FMASK is allocated like an ordinary texture. */
 462         struct radeon_surf fmask = image->surface;
 463         struct radeon_surf_info info = image->info;
 464         memset(out, 0, sizeof(*out));
 465
 466         fmask.surf_alignment = 0;
 467         fmask.surf_size = 0;
 468         fmask.flags |= RADEON_SURF_FMASK;
 469         info.samples = 1;
 470         /* Force 2D tiling if it wasn't set. This may occur when creating
 471          * FMASK for MSAA resolve on R6xx. On R6xx, the single-sample
 472          * destination buffer must have an FMASK too. */
 473         fmask.flags = RADEON_SURF_CLR(fmask.flags, MODE);
 474         fmask.flags |= RADEON_SURF_SET(RADEON_SURF_MODE_2D, MODE);
 475
 476         fmask.flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
 477
 478         switch (nr_samples) {
 479         case 2:
 480         case 4:
 481                 fmask.bpe = 1;
 482                 break;
 483         case 8:
 484                 fmask.bpe = 4;
 485                 break;
 486         default:
 487                 return;
 488         }
 489
 490         device->ws->surface_init(device->ws, &info, &fmask);
 491         assert(fmask.level[0].mode == RADEON_SURF_MODE_2D);
 492
 493         out->slice_tile_max = (fmask.level[0].nblk_x * fmask.level[0].nblk_y) / 64;
 494         if (out->slice_tile_max)
 495                 out->slice_tile_max -= 1;
 496
 497         out->tile_mode_index = fmask.tiling_index[0];
 498         out->pitch_in_pixels = fmask.level[0].nblk_x;
 499         out->bank_height = fmask.bankh;
 500         out->alignment = MAX2(256, fmask.surf_alignment);
 501         out->size = fmask.surf_size;
 502 }
 503
 504 static void
 505 radv_image_alloc_fmask(struct radv_device *device,
 506                        struct radv_image *image)
 507 {
 508         radv_image_get_fmask_info(device, image, image->info.samples, &image->fmask);
 509
 510         image->fmask.offset = align64(image->size, image->fmask.alignment);
 511         image->size = image->fmask.offset + image->fmask.size;
 512         image->alignment = MAX2(image->alignment, image->fmask.alignment);
 513 }
 514
 515 static void
 516 radv_image_get_cmask_info(struct radv_device *device,
 517                           struct radv_image *image,
 518                           struct radv_cmask_info *out)
 519 {
 520         unsigned pipe_interleave_bytes = device->physical_device->rad_info.pipe_interleave_bytes;
 521         unsigned num_pipes = device->physical_device->rad_info.num_tile_pipes;
 522         unsigned cl_width, cl_height;
 523
 524         switch (num_pipes) {
 525         case 2:
 526                 cl_width = 32;
 527                 cl_height = 16;
 528                 break;
 529         case 4:
 530                 cl_width = 32;
 531                 cl_height = 32;
 532                 break;
 533         case 8:
 534                 cl_width = 64;
 535                 cl_height = 32;
 536                 break;
 537         case 16: /* Hawaii */
 538                 cl_width = 64;
 539                 cl_height = 64;
 540                 break;
 541         default:
 542                 assert(0);
 543                 return;
 544         }
 545
 546         unsigned base_align = num_pipes * pipe_interleave_bytes;
 547
 548         unsigned width = align(image->info.width, cl_width*8);
 549         unsigned height = align(image->info.height, cl_height*8);
 550         unsigned slice_elements = (width * height) / (8*8);
 551
 552         /* Each element of CMASK is a nibble. */
 553         unsigned slice_bytes = slice_elements / 2;
 554
 555         out->slice_tile_max = (width * height) / (128*128);
 556         if (out->slice_tile_max)
 557                 out->slice_tile_max -= 1;
 558
 559         out->alignment = MAX2(256, base_align);
 560         out->size = (image->type == VK_IMAGE_TYPE_3D ? image->info.depth : image->info.array_size) *
 561                     align(slice_bytes, base_align);
 562 }
 563
 564 static void
 565 radv_image_alloc_cmask(struct radv_device *device,
 566                        struct radv_image *image)
 567 {
 568         radv_image_get_cmask_info(device, image, &image->cmask);
 569
 570         image->cmask.offset = align64(image->size, image->cmask.alignment);
 571         /* + 8 for storing the clear values */
 572         image->clear_value_offset = image->cmask.offset + image->cmask.size;
 573         image->size = image->cmask.offset + image->cmask.size + 8;
 574         image->alignment = MAX2(image->alignment, image->cmask.alignment);
 575 }
 576
 577 static void
 578 radv_image_alloc_dcc(struct radv_device *device,
 579                        struct radv_image *image)
 580 {
 581         image->dcc_offset = align64(image->size, image->surface.dcc_alignment);
 582         /* + 8 for storing the clear values */
 583         image->clear_value_offset = image->dcc_offset + image->surface.dcc_size;
 584         image->size = image->dcc_offset + image->surface.dcc_size + 8;
 585         image->alignment = MAX2(image->alignment, image->surface.dcc_alignment);
 586 }
 587
 588 static void
 589 radv_image_alloc_htile(struct radv_device *device,
 590                        struct radv_image *image)
 591 {
 592         if ((device->debug_flags & RADV_DEBUG_NO_HIZ) || image->info.levels > 1) {
 593                 image->surface.htile_size = 0;
 594                 return;
 595         }
 596
 597         image->htile_offset = align64(image->size, image->surface.htile_alignment);
 598
 599         /* + 8 for storing the clear values */
 600         image->clear_value_offset = image->htile_offset + image->surface.htile_size;
 601         image->size = image->clear_value_offset + 8;
 602         image->alignment = align64(image->alignment, image->surface.htile_alignment);
 603 }
 604
 605 VkResult
 606 radv_image_create(VkDevice _device,
 607                   const struct radv_image_create_info *create_info,
 608                   const VkAllocationCallbacks* alloc,
 609                   VkImage *pImage)
 610 {
 611         RADV_FROM_HANDLE(radv_device, device, _device);
 612         const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
 613         struct radv_image *image = NULL;
 614         bool can_cmask_dcc = false;
 615         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO);
 616
 617         radv_assert(pCreateInfo->mipLevels > 0);
 618         radv_assert(pCreateInfo->arrayLayers > 0);
 619         radv_assert(pCreateInfo->samples > 0);
 620         radv_assert(pCreateInfo->extent.width > 0);
 621         radv_assert(pCreateInfo->extent.height > 0);
 622         radv_assert(pCreateInfo->extent.depth > 0);
 623
 624         image = vk_alloc2(&device->alloc, alloc, sizeof(*image), 8,
 625                             VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 626         if (!image)
 627                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 628
 629         memset(image, 0, sizeof(*image));
 630         image->type = pCreateInfo->imageType;
 631         image->info.width = pCreateInfo->extent.width;
 632         image->info.height = pCreateInfo->extent.height;
 633         image->info.depth = pCreateInfo->extent.depth;
 634         image->info.samples = pCreateInfo->samples;
 635         image->info.array_size = pCreateInfo->arrayLayers;
 636         image->info.levels = pCreateInfo->mipLevels;
 637
 638         image->vk_format = pCreateInfo->format;
 639         image->tiling = pCreateInfo->tiling;
 640         image->usage = pCreateInfo->usage;
 641         image->flags = pCreateInfo->flags;
 642
 643         image->exclusive = pCreateInfo->sharingMode == VK_SHARING_MODE_EXCLUSIVE;
 644         if (pCreateInfo->sharingMode == VK_SHARING_MODE_CONCURRENT) {
 645                 for (uint32_t i = 0; i < pCreateInfo->queueFamilyIndexCount; ++i)
 646                         if (pCreateInfo->pQueueFamilyIndices[i] == VK_QUEUE_FAMILY_EXTERNAL_KHX)
 647                                 image->queue_family_mask |= (1u << RADV_MAX_QUEUE_FAMILIES) - 1u;
 648                         else
 649                                 image->queue_family_mask |= 1u << pCreateInfo->pQueueFamilyIndices[i];
 650         }
 651
 652         radv_init_surface(device, &image->surface, create_info);
 653
 654         device->ws->surface_init(device->ws, &image->info, &image->surface);
 655
 656         image->size = image->surface.surf_size;
 657         image->alignment = image->surface.surf_alignment;
 658
 659         if (image->exclusive || image->queue_family_mask == 1)
 660                 can_cmask_dcc = true;
 661
 662         if ((pCreateInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
 663             image->surface.dcc_size && can_cmask_dcc)
 664                 radv_image_alloc_dcc(device, image);
 665         else
 666                 image->surface.dcc_size = 0;
 667
 668         if ((pCreateInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
 669             pCreateInfo->mipLevels == 1 &&
 670             !image->surface.dcc_size && image->info.depth == 1 && can_cmask_dcc)
 671                 radv_image_alloc_cmask(device, image);
 672         if (image->info.samples > 1 && vk_format_is_color(pCreateInfo->format)) {
 673                 radv_image_alloc_fmask(device, image);
 674         } else if (vk_format_is_depth(pCreateInfo->format)) {
 675
 676                 radv_image_alloc_htile(device, image);
 677         }
 678
 679         if (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) {
 680                 image->alignment = MAX2(image->alignment, 4096);
 681                 image->size = align64(image->size, image->alignment);
 682                 image->offset = 0;
 683
 684                 image->bo = device->ws->buffer_create(device->ws, image->size, image->alignment,
 685                                                       0, RADEON_FLAG_VIRTUAL);
 686                 if (!image->bo) {
 687                         vk_free2(&device->alloc, alloc, image);
 688                         return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY);
 689                 }
 690         }
 691
 692         *pImage = radv_image_to_handle(image);
 693
 694         return VK_SUCCESS;
 695 }
 696
 697 void
 698 radv_image_view_init(struct radv_image_view *iview,
 699                      struct radv_device *device,
 700                      const VkImageViewCreateInfo* pCreateInfo,
 701                      struct radv_cmd_buffer *cmd_buffer,
 702                      VkImageUsageFlags usage_mask)
 703 {
 704         RADV_FROM_HANDLE(radv_image, image, pCreateInfo->image);
 705         const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
 706         uint32_t blk_w;
 707         bool is_stencil = false;
 708         switch (image->type) {
 709         case VK_IMAGE_TYPE_1D:
 710         case VK_IMAGE_TYPE_2D:
 711                 assert(range->baseArrayLayer + radv_get_layerCount(image, range) - 1 <= image->info.array_size);
 712                 break;
 713         case VK_IMAGE_TYPE_3D:
 714                 assert(range->baseArrayLayer + radv_get_layerCount(image, range) - 1
 715                        <= radv_minify(image->info.depth, range->baseMipLevel));
 716                 break;
 717         default:
 718                 unreachable("bad VkImageType");
 719         }
 720         iview->image = image;
 721         iview->bo = image->bo;
 722         iview->type = pCreateInfo->viewType;
 723         iview->vk_format = pCreateInfo->format;
 724         iview->aspect_mask = pCreateInfo->subresourceRange.aspectMask;
 725
 726         if (iview->aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 727                 is_stencil = true;
 728                 iview->vk_format = vk_format_stencil_only(iview->vk_format);
 729         } else if (iview->aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 730                 iview->vk_format = vk_format_depth_only(iview->vk_format);
 731         }
 732
 733         iview->extent = (VkExtent3D) {
 734                 .width  = radv_minify(image->info.width , range->baseMipLevel),
 735                 .height = radv_minify(image->info.height, range->baseMipLevel),
 736                 .depth  = radv_minify(image->info.depth , range->baseMipLevel),
 737         };
 738
 739         iview->extent.width = round_up_u32(iview->extent.width * vk_format_get_blockwidth(iview->vk_format),
 740                                            vk_format_get_blockwidth(image->vk_format));
 741         iview->extent.height = round_up_u32(iview->extent.height * vk_format_get_blockheight(iview->vk_format),
 742                                             vk_format_get_blockheight(image->vk_format));
 743
 744         assert(image->surface.blk_w % vk_format_get_blockwidth(image->vk_format) == 0);
 745         blk_w = image->surface.blk_w / vk_format_get_blockwidth(image->vk_format) * vk_format_get_blockwidth(iview->vk_format);
 746         iview->base_layer = range->baseArrayLayer;
 747         iview->layer_count = radv_get_layerCount(image, range);
 748         iview->base_mip = range->baseMipLevel;
 749
 750         si_make_texture_descriptor(device, image, false,
 751                                    iview->type,
 752                                    iview->vk_format,
 753                                    &pCreateInfo->components,
 754                                    0, radv_get_levelCount(image, range) - 1,
 755                                    range->baseArrayLayer,
 756                                    range->baseArrayLayer + radv_get_layerCount(image, range) - 1,
 757                                    iview->extent.width,
 758                                    iview->extent.height,
 759                                    iview->extent.depth,
 760                                    iview->descriptor,
 761                                    iview->fmask_descriptor);
 762         si_set_mutable_tex_desc_fields(device, image,
 763                                        is_stencil ? &image->surface.stencil_level[range->baseMipLevel] : &image->surface.level[range->baseMipLevel], range->baseMipLevel,
 764                                        range->baseMipLevel,
 765                                        blk_w, is_stencil, iview->descriptor);
 766 }
 767
 768 bool radv_layout_has_htile(const struct radv_image *image,
 769                            VkImageLayout layout,
 770                            unsigned queue_mask)
 771 {
 772         return image->surface.htile_size &&
 773                (layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL ||
 774                 layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) &&
 775                queue_mask == (1u << RADV_QUEUE_GENERAL);
 776 }
 777
 778 bool radv_layout_is_htile_compressed(const struct radv_image *image,
 779                                      VkImageLayout layout,
 780                                      unsigned queue_mask)
 781 {
 782         return image->surface.htile_size &&
 783                (layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL ||
 784                 layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) &&
 785                queue_mask == (1u << RADV_QUEUE_GENERAL);
 786 }
 787
 788 bool radv_layout_can_fast_clear(const struct radv_image *image,
 789                                 VkImageLayout layout,
 790                                 unsigned queue_mask)
 791 {
 792         return layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL &&
 793                 queue_mask == (1u << RADV_QUEUE_GENERAL);
 794 }
 795
 796
 797 unsigned radv_image_queue_family_mask(const struct radv_image *image, uint32_t family, uint32_t queue_family)
 798 {
 799         if (!image->exclusive)
 800                 return image->queue_family_mask;
 801         if (family == VK_QUEUE_FAMILY_EXTERNAL_KHX)
 802                 return (1u << RADV_MAX_QUEUE_FAMILIES) - 1u;
 803         if (family == VK_QUEUE_FAMILY_IGNORED)
 804                 return 1u << queue_family;
 805         return 1u << family;
 806 }
 807
 808 VkResult
 809 radv_CreateImage(VkDevice device,
 810                  const VkImageCreateInfo *pCreateInfo,
 811                  const VkAllocationCallbacks *pAllocator,
 812                  VkImage *pImage)
 813 {
 814         return radv_image_create(device,
 815                                  &(struct radv_image_create_info) {
 816                                          .vk_info = pCreateInfo,
 817                                                  .scanout = false,
 818                                                  },
 819                                  pAllocator,
 820                                  pImage);
 821 }
 822
 823 void
 824 radv_DestroyImage(VkDevice _device, VkImage _image,
 825                   const VkAllocationCallbacks *pAllocator)
 826 {
 827         RADV_FROM_HANDLE(radv_device, device, _device);
 828         RADV_FROM_HANDLE(radv_image, image, _image);
 829
 830         if (!image)
 831                 return;
 832
 833         if (image->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
 834                 device->ws->buffer_destroy(image->bo);
 835
 836         vk_free2(&device->alloc, pAllocator, image);
 837 }
 838
 839 void radv_GetImageSubresourceLayout(
 840         VkDevice                                    device,
 841         VkImage                                     _image,
 842         const VkImageSubresource*                   pSubresource,
 843         VkSubresourceLayout*                        pLayout)
 844 {
 845         RADV_FROM_HANDLE(radv_image, image, _image);
 846         int level = pSubresource->mipLevel;
 847         int layer = pSubresource->arrayLayer;
 848         struct radeon_surf *surface = &image->surface;
 849
 850         pLayout->offset = surface->level[level].offset + surface->level[level].slice_size * layer;
 851         pLayout->rowPitch = surface->level[level].nblk_x * surface->bpe;
 852         pLayout->arrayPitch = surface->level[level].slice_size;
 853         pLayout->depthPitch = surface->level[level].slice_size;
 854         pLayout->size = surface->level[level].slice_size;
 855         if (image->type == VK_IMAGE_TYPE_3D)
 856                 pLayout->size *= u_minify(image->info.depth, level);
 857 }
 858
 859
 860 VkResult
 861 radv_CreateImageView(VkDevice _device,
 862                      const VkImageViewCreateInfo *pCreateInfo,
 863                      const VkAllocationCallbacks *pAllocator,
 864                      VkImageView *pView)
 865 {
 866         RADV_FROM_HANDLE(radv_device, device, _device);
 867         struct radv_image_view *view;
 868
 869         view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
 870                            VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 871         if (view == NULL)
 872                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 873
 874         radv_image_view_init(view, device, pCreateInfo, NULL, ~0);
 875
 876         *pView = radv_image_view_to_handle(view);
 877
 878         return VK_SUCCESS;
 879 }
 880
 881 void
 882 radv_DestroyImageView(VkDevice _device, VkImageView _iview,
 883                       const VkAllocationCallbacks *pAllocator)
 884 {
 885         RADV_FROM_HANDLE(radv_device, device, _device);
 886         RADV_FROM_HANDLE(radv_image_view, iview, _iview);
 887
 888         if (!iview)
 889                 return;
 890         vk_free2(&device->alloc, pAllocator, iview);
 891 }
 892
 893 void radv_buffer_view_init(struct radv_buffer_view *view,
 894                            struct radv_device *device,
 895                            const VkBufferViewCreateInfo* pCreateInfo,
 896                            struct radv_cmd_buffer *cmd_buffer)
 897 {
 898         RADV_FROM_HANDLE(radv_buffer, buffer, pCreateInfo->buffer);
 899
 900         view->bo = buffer->bo;
 901         view->range = pCreateInfo->range == VK_WHOLE_SIZE ?
 902                 buffer->size - pCreateInfo->offset : pCreateInfo->range;
 903         view->vk_format = pCreateInfo->format;
 904
 905         radv_make_buffer_descriptor(device, buffer, view->vk_format,
 906                                     pCreateInfo->offset, view->range, view->state);
 907 }
 908
 909 VkResult
 910 radv_CreateBufferView(VkDevice _device,
 911                       const VkBufferViewCreateInfo *pCreateInfo,
 912                       const VkAllocationCallbacks *pAllocator,
 913                       VkBufferView *pView)
 914 {
 915         RADV_FROM_HANDLE(radv_device, device, _device);
 916         struct radv_buffer_view *view;
 917
 918         view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
 919                            VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 920         if (!view)
 921                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 922
 923         radv_buffer_view_init(view, device, pCreateInfo, NULL);
 924
 925         *pView = radv_buffer_view_to_handle(view);
 926
 927         return VK_SUCCESS;
 928 }
 929
 930 void
 931 radv_DestroyBufferView(VkDevice _device, VkBufferView bufferView,
 932                        const VkAllocationCallbacks *pAllocator)
 933 {
 934         RADV_FROM_HANDLE(radv_device, device, _device);
 935         RADV_FROM_HANDLE(radv_buffer_view, view, bufferView);
 936
 937         if (!view)
 938                 return;
 939
 940         vk_free2(&device->alloc, pAllocator, view);
 941 }