src/freedreno/vulkan/tu_image.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "util/debug.h"
  31 #include "util/u_atomic.h"
  32 #include "vk_format.h"
  33 #include "vk_util.h"
  34
  35 static inline bool
  36 image_level_linear(struct tu_image *image, int level)
  37 {
  38    unsigned w = u_minify(image->extent.width, level);
  39    return w < 16;
  40 }
  41
  42 /* indexed by cpp: */
  43 static const struct
  44 {
  45    unsigned pitchalign;
  46    unsigned heightalign;
  47 } tile_alignment[] = {
  48    [1] = { 128, 32 }, [2] = { 128, 16 }, [3] = { 128, 16 }, [4] = { 64, 16 },
  49    [8] = { 64, 16 },  [12] = { 64, 16 }, [16] = { 64, 16 },
  50 };
  51
  52 static void
  53 setup_slices(struct tu_image *image, const VkImageCreateInfo *pCreateInfo)
  54 {
  55    enum vk_format_layout layout =
  56       vk_format_description(pCreateInfo->format)->layout;
  57    uint32_t layer_size = 0;
  58    uint32_t width = pCreateInfo->extent.width;
  59    uint32_t height = pCreateInfo->extent.height;
  60    uint32_t depth = pCreateInfo->extent.depth;
  61    bool layer_first = pCreateInfo->imageType != VK_IMAGE_TYPE_3D;
  62    uint32_t alignment = pCreateInfo->imageType == VK_IMAGE_TYPE_3D ? 4096 : 1;
  63    uint32_t cpp = vk_format_get_blocksize(pCreateInfo->format);
  64
  65    uint32_t heightalign = tile_alignment[cpp].heightalign;
  66
  67    for (unsigned level = 0; level < pCreateInfo->mipLevels; level++) {
  68       struct tu_image_level *slice = &image->levels[level];
  69       bool linear_level = image_level_linear(image, level);
  70       uint32_t aligned_height = height;
  71       uint32_t blocks;
  72       uint32_t pitchalign;
  73
  74       if (image->tile_mode && !linear_level) {
  75          pitchalign = tile_alignment[cpp].pitchalign;
  76          aligned_height = align(aligned_height, heightalign);
  77       } else {
  78          pitchalign = 64;
  79
  80          /* The blits used for mem<->gmem work at a granularity of
  81           * 32x32, which can cause faults due to over-fetch on the
  82           * last level.  The simple solution is to over-allocate a
  83           * bit the last level to ensure any over-fetch is harmless.
  84           * The pitch is already sufficiently aligned, but height
  85           * may not be:
  86           */
  87          if ((level + 1 == pCreateInfo->mipLevels))
  88             aligned_height = align(aligned_height, 32);
  89       }
  90
  91       if (layout == VK_FORMAT_LAYOUT_ASTC)
  92          slice->pitch = util_align_npot(
  93             width,
  94             pitchalign * vk_format_get_blockwidth(pCreateInfo->format));
  95       else
  96          slice->pitch = align(width, pitchalign);
  97
  98       slice->offset = layer_size;
  99       blocks = vk_format_get_block_count(pCreateInfo->format, slice->pitch,
 100                                          aligned_height);
 101
 102       /* 1d array and 2d array textures must all have the same layer size
 103        * for each miplevel on a3xx. 3d textures can have different layer
 104        * sizes for high levels, but the hw auto-sizer is buggy (or at least
 105        * different than what this code does), so as soon as the layer size
 106        * range gets into range, we stop reducing it.
 107        */
 108       if (pCreateInfo->imageType == VK_IMAGE_TYPE_3D &&
 109           (level == 1 ||
 110            (level > 1 && image->levels[level - 1].size > 0xf000)))
 111          slice->size = align(blocks * cpp, alignment);
 112       else if (level == 0 || layer_first || alignment == 1)
 113          slice->size = align(blocks * cpp, alignment);
 114       else
 115          slice->size = image->levels[level - 1].size;
 116
 117       layer_size += slice->size * depth;
 118
 119       width = u_minify(width, 1);
 120       height = u_minify(height, 1);
 121       depth = u_minify(depth, 1);
 122    }
 123
 124    image->layer_size = align(layer_size, 4096);
 125 }
 126
 127 VkResult
 128 tu_image_create(VkDevice _device,
 129                 const struct tu_image_create_info *create_info,
 130                 const VkAllocationCallbacks *alloc,
 131                 VkImage *pImage)
 132 {
 133    TU_FROM_HANDLE(tu_device, device, _device);
 134    const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
 135    struct tu_image *image = NULL;
 136    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO);
 137
 138    tu_assert(pCreateInfo->mipLevels > 0);
 139    tu_assert(pCreateInfo->arrayLayers > 0);
 140    tu_assert(pCreateInfo->samples > 0);
 141    tu_assert(pCreateInfo->extent.width > 0);
 142    tu_assert(pCreateInfo->extent.height > 0);
 143    tu_assert(pCreateInfo->extent.depth > 0);
 144
 145    image = vk_zalloc2(&device->alloc, alloc, sizeof(*image), 8,
 146                       VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 147    if (!image)
 148       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 149
 150    image->type = pCreateInfo->imageType;
 151
 152    image->vk_format = pCreateInfo->format;
 153    image->tiling = pCreateInfo->tiling;
 154    image->usage = pCreateInfo->usage;
 155    image->flags = pCreateInfo->flags;
 156    image->extent = pCreateInfo->extent;
 157    image->level_count = pCreateInfo->mipLevels;
 158    image->layer_count = pCreateInfo->arrayLayers;
 159
 160    image->exclusive = pCreateInfo->sharingMode == VK_SHARING_MODE_EXCLUSIVE;
 161    if (pCreateInfo->sharingMode == VK_SHARING_MODE_CONCURRENT) {
 162       for (uint32_t i = 0; i < pCreateInfo->queueFamilyIndexCount; ++i)
 163          if (pCreateInfo->pQueueFamilyIndices[i] ==
 164              VK_QUEUE_FAMILY_EXTERNAL)
 165             image->queue_family_mask |= (1u << TU_MAX_QUEUE_FAMILIES) - 1u;
 166          else
 167             image->queue_family_mask |=
 168                1u << pCreateInfo->pQueueFamilyIndices[i];
 169    }
 170
 171    image->shareable =
 172       vk_find_struct_const(pCreateInfo->pNext,
 173                            EXTERNAL_MEMORY_IMAGE_CREATE_INFO) != NULL;
 174
 175    image->tile_mode = TILE6_LINEAR;
 176    if (pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL && !create_info->scanout)
 177       image->tile_mode = TILE6_3;
 178
 179    setup_slices(image, pCreateInfo);
 180
 181    image->size = image->layer_size * pCreateInfo->arrayLayers;
 182    *pImage = tu_image_to_handle(image);
 183
 184    return VK_SUCCESS;
 185 }
 186
 187 static enum a6xx_tex_fetchsize
 188 tu6_fetchsize(VkFormat format)
 189 {
 190    if (vk_format_description(format)->layout == VK_FORMAT_LAYOUT_ASTC)
 191       return TFETCH6_16_BYTE;
 192
 193    switch (vk_format_get_blocksize(format) / vk_format_get_blockwidth(format)) {
 194    case 1: return TFETCH6_1_BYTE;
 195    case 2: return TFETCH6_2_BYTE;
 196    case 4: return TFETCH6_4_BYTE;
 197    case 8: return TFETCH6_8_BYTE;
 198    case 16: return TFETCH6_16_BYTE;
 199    default:
 200       unreachable("bad block size");
 201    }
 202 }
 203
 204 static uint32_t
 205 tu6_texswiz(const VkComponentMapping *comps, const unsigned char *fmt_swiz)
 206 {
 207    unsigned char swiz[4] = {comps->r, comps->g, comps->b, comps->a};
 208    unsigned char vk_swizzle[] = {
 209       [VK_COMPONENT_SWIZZLE_ZERO] = A6XX_TEX_ZERO,
 210       [VK_COMPONENT_SWIZZLE_ONE]  = A6XX_TEX_ONE,
 211       [VK_COMPONENT_SWIZZLE_R] = A6XX_TEX_X,
 212       [VK_COMPONENT_SWIZZLE_G] = A6XX_TEX_Y,
 213       [VK_COMPONENT_SWIZZLE_B] = A6XX_TEX_Z,
 214       [VK_COMPONENT_SWIZZLE_A] = A6XX_TEX_W,
 215    };
 216    for (unsigned i = 0; i < 4; i++) {
 217       swiz[i] = (swiz[i] == VK_COMPONENT_SWIZZLE_IDENTITY) ? i : vk_swizzle[swiz[i]];
 218       /* if format has 0/1 in channel, use that (needed for bc1_rgb) */
 219       if (swiz[i] < 4) {
 220          switch (fmt_swiz[swiz[i]]) {
 221          case VK_SWIZZLE_0: swiz[i] = A6XX_TEX_ZERO; break;
 222          case VK_SWIZZLE_1: swiz[i] = A6XX_TEX_ONE;  break;
 223          }
 224       }
 225    }
 226
 227    return A6XX_TEX_CONST_0_SWIZ_X(swiz[0]) |
 228           A6XX_TEX_CONST_0_SWIZ_Y(swiz[1]) |
 229           A6XX_TEX_CONST_0_SWIZ_Z(swiz[2]) |
 230           A6XX_TEX_CONST_0_SWIZ_W(swiz[3]);
 231 }
 232
 233 static enum a6xx_tex_type
 234 tu6_tex_type(VkImageViewType type)
 235 {
 236    switch (type) {
 237    default:
 238    case VK_IMAGE_VIEW_TYPE_1D:
 239    case VK_IMAGE_VIEW_TYPE_1D_ARRAY:
 240       return A6XX_TEX_1D;
 241    case VK_IMAGE_VIEW_TYPE_2D:
 242    case VK_IMAGE_VIEW_TYPE_2D_ARRAY:
 243       return A6XX_TEX_2D;
 244    case VK_IMAGE_VIEW_TYPE_3D:
 245       return A6XX_TEX_3D;
 246    case VK_IMAGE_VIEW_TYPE_CUBE:
 247    case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY:
 248       return A6XX_TEX_CUBE;
 249    }
 250 }
 251
 252 void
 253 tu_image_view_init(struct tu_image_view *iview,
 254                    struct tu_device *device,
 255                    const VkImageViewCreateInfo *pCreateInfo)
 256 {
 257    TU_FROM_HANDLE(tu_image, image, pCreateInfo->image);
 258    const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
 259
 260    switch (image->type) {
 261    case VK_IMAGE_TYPE_1D:
 262    case VK_IMAGE_TYPE_2D:
 263       assert(range->baseArrayLayer + tu_get_layerCount(image, range) <=
 264              image->layer_count);
 265       break;
 266    case VK_IMAGE_TYPE_3D:
 267       assert(range->baseArrayLayer + tu_get_layerCount(image, range) <=
 268              tu_minify(image->extent.depth, range->baseMipLevel));
 269       break;
 270    default:
 271       unreachable("bad VkImageType");
 272    }
 273
 274    iview->image = image;
 275    iview->type = pCreateInfo->viewType;
 276    iview->vk_format = pCreateInfo->format;
 277    iview->aspect_mask = pCreateInfo->subresourceRange.aspectMask;
 278
 279    if (iview->aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 280       iview->vk_format = vk_format_stencil_only(iview->vk_format);
 281    } else if (iview->aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 282       iview->vk_format = vk_format_depth_only(iview->vk_format);
 283    }
 284
 285    // should we minify?
 286    iview->extent = image->extent;
 287
 288    iview->base_layer = range->baseArrayLayer;
 289    iview->layer_count = tu_get_layerCount(image, range);
 290    iview->base_mip = range->baseMipLevel;
 291    iview->level_count = tu_get_levelCount(image, range);
 292
 293    memset(iview->descriptor, 0, sizeof(iview->descriptor));
 294
 295    const struct tu_native_format *fmt = tu6_get_native_format(iview->vk_format);
 296    struct tu_image_level *slice0 = &image->levels[iview->base_mip];
 297    uint64_t base_addr = image->bo->iova + iview->base_layer * image->layer_size + slice0->offset;
 298    uint32_t pitch = (slice0->pitch / vk_format_get_blockwidth(iview->vk_format)) *
 299                         vk_format_get_blocksize(iview->vk_format);
 300    enum a6xx_tile_mode tile_mode =
 301       image_level_linear(image, iview->base_mip) ? TILE6_LINEAR : image->tile_mode;
 302
 303    iview->descriptor[0] =
 304       A6XX_TEX_CONST_0_TILE_MODE(tile_mode) |
 305       COND(vk_format_is_srgb(iview->vk_format), A6XX_TEX_CONST_0_SRGB) |
 306       A6XX_TEX_CONST_0_FMT(fmt->tex) |
 307       A6XX_TEX_CONST_0_SAMPLES(0) |
 308       A6XX_TEX_CONST_0_SWAP(image->tile_mode ? WZYX : fmt->swap) |
 309       tu6_texswiz(&pCreateInfo->components, vk_format_description(iview->vk_format)->swizzle) |
 310       A6XX_TEX_CONST_0_MIPLVLS(iview->level_count - 1);
 311    iview->descriptor[1] =
 312       A6XX_TEX_CONST_1_WIDTH(u_minify(image->extent.width, iview->base_mip)) |
 313       A6XX_TEX_CONST_1_HEIGHT(u_minify(image->extent.height, iview->base_mip));
 314    iview->descriptor[2] =
 315       A6XX_TEX_CONST_2_FETCHSIZE(tu6_fetchsize(iview->vk_format)) |
 316       A6XX_TEX_CONST_2_PITCH(pitch) |
 317       A6XX_TEX_CONST_2_TYPE(tu6_tex_type(pCreateInfo->viewType));
 318    iview->descriptor[3] = 0;
 319    iview->descriptor[4] = base_addr;
 320    iview->descriptor[5] = base_addr >> 32;
 321
 322    if (pCreateInfo->viewType != VK_IMAGE_VIEW_TYPE_3D) {
 323       iview->descriptor[3] |= A6XX_TEX_CONST_3_ARRAY_PITCH(image->layer_size);
 324       iview->descriptor[5] |= A6XX_TEX_CONST_5_DEPTH(iview->layer_count);
 325    } else {
 326       iview->descriptor[3] |=
 327          A6XX_TEX_CONST_3_MIN_LAYERSZ(image->levels[image->level_count - 1].size) |
 328          A6XX_TEX_CONST_3_ARRAY_PITCH(slice0->size);
 329       iview->descriptor[5] |=
 330          A6XX_TEX_CONST_5_DEPTH(u_minify(image->extent.depth, iview->base_mip));
 331    }
 332 }
 333
 334 unsigned
 335 tu_image_queue_family_mask(const struct tu_image *image,
 336                            uint32_t family,
 337                            uint32_t queue_family)
 338 {
 339    if (!image->exclusive)
 340       return image->queue_family_mask;
 341    if (family == VK_QUEUE_FAMILY_EXTERNAL)
 342       return (1u << TU_MAX_QUEUE_FAMILIES) - 1u;
 343    if (family == VK_QUEUE_FAMILY_IGNORED)
 344       return 1u << queue_family;
 345    return 1u << family;
 346 }
 347
 348 VkResult
 349 tu_CreateImage(VkDevice device,
 350                const VkImageCreateInfo *pCreateInfo,
 351                const VkAllocationCallbacks *pAllocator,
 352                VkImage *pImage)
 353 {
 354 #ifdef ANDROID
 355    const VkNativeBufferANDROID *gralloc_info =
 356       vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
 357
 358    if (gralloc_info)
 359       return tu_image_from_gralloc(device, pCreateInfo, gralloc_info,
 360                                    pAllocator, pImage);
 361 #endif
 362
 363    const struct wsi_image_create_info *wsi_info =
 364       vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA);
 365    bool scanout = wsi_info && wsi_info->scanout;
 366
 367    return tu_image_create(device,
 368                           &(struct tu_image_create_info) {
 369                              .vk_info = pCreateInfo,
 370                              .scanout = scanout,
 371                           },
 372                           pAllocator, pImage);
 373 }
 374
 375 void
 376 tu_DestroyImage(VkDevice _device,
 377                 VkImage _image,
 378                 const VkAllocationCallbacks *pAllocator)
 379 {
 380    TU_FROM_HANDLE(tu_device, device, _device);
 381    TU_FROM_HANDLE(tu_image, image, _image);
 382
 383    if (!image)
 384       return;
 385
 386    if (image->owned_memory != VK_NULL_HANDLE)
 387       tu_FreeMemory(_device, image->owned_memory, pAllocator);
 388
 389    vk_free2(&device->alloc, pAllocator, image);
 390 }
 391
 392 void
 393 tu_GetImageSubresourceLayout(VkDevice _device,
 394                              VkImage _image,
 395                              const VkImageSubresource *pSubresource,
 396                              VkSubresourceLayout *pLayout)
 397 {
 398    TU_FROM_HANDLE(tu_image, image, _image);
 399
 400    const uint32_t layer_offset = image->layer_size * pSubresource->arrayLayer;
 401    const struct tu_image_level *level =
 402       image->levels + pSubresource->mipLevel;
 403
 404    pLayout->offset = layer_offset + level->offset;
 405    pLayout->size = level->size;
 406    pLayout->rowPitch =
 407       level->pitch * vk_format_get_blocksize(image->vk_format);
 408    pLayout->arrayPitch = image->layer_size;
 409    pLayout->depthPitch = level->size;
 410 }
 411
 412 VkResult
 413 tu_CreateImageView(VkDevice _device,
 414                    const VkImageViewCreateInfo *pCreateInfo,
 415                    const VkAllocationCallbacks *pAllocator,
 416                    VkImageView *pView)
 417 {
 418    TU_FROM_HANDLE(tu_device, device, _device);
 419    struct tu_image_view *view;
 420
 421    view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
 422                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 423    if (view == NULL)
 424       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 425
 426    tu_image_view_init(view, device, pCreateInfo);
 427
 428    *pView = tu_image_view_to_handle(view);
 429
 430    return VK_SUCCESS;
 431 }
 432
 433 void
 434 tu_DestroyImageView(VkDevice _device,
 435                     VkImageView _iview,
 436                     const VkAllocationCallbacks *pAllocator)
 437 {
 438    TU_FROM_HANDLE(tu_device, device, _device);
 439    TU_FROM_HANDLE(tu_image_view, iview, _iview);
 440
 441    if (!iview)
 442       return;
 443    vk_free2(&device->alloc, pAllocator, iview);
 444 }
 445
 446 void
 447 tu_buffer_view_init(struct tu_buffer_view *view,
 448                     struct tu_device *device,
 449                     const VkBufferViewCreateInfo *pCreateInfo)
 450 {
 451    TU_FROM_HANDLE(tu_buffer, buffer, pCreateInfo->buffer);
 452
 453    view->range = pCreateInfo->range == VK_WHOLE_SIZE
 454                     ? buffer->size - pCreateInfo->offset
 455                     : pCreateInfo->range;
 456    view->vk_format = pCreateInfo->format;
 457 }
 458
 459 VkResult
 460 tu_CreateBufferView(VkDevice _device,
 461                     const VkBufferViewCreateInfo *pCreateInfo,
 462                     const VkAllocationCallbacks *pAllocator,
 463                     VkBufferView *pView)
 464 {
 465    TU_FROM_HANDLE(tu_device, device, _device);
 466    struct tu_buffer_view *view;
 467
 468    view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
 469                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 470    if (!view)
 471       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 472
 473    tu_buffer_view_init(view, device, pCreateInfo);
 474
 475    *pView = tu_buffer_view_to_handle(view);
 476
 477    return VK_SUCCESS;
 478 }
 479
 480 void
 481 tu_DestroyBufferView(VkDevice _device,
 482                      VkBufferView bufferView,
 483                      const VkAllocationCallbacks *pAllocator)
 484 {
 485    TU_FROM_HANDLE(tu_device, device, _device);
 486    TU_FROM_HANDLE(tu_buffer_view, view, bufferView);
 487
 488    if (!view)
 489       return;
 490
 491    vk_free2(&device->alloc, pAllocator, view);
 492 }