src/intel/vulkan/anv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_meta.h"
  25
  26 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  27  * elements rather than texels. One element equals one texel or one block
  28  * if Image is uncompressed or compressed, respectively.
  29  */
  30 static struct VkExtent3D
  31 meta_region_extent_el(const struct anv_image *image,
  32                       const struct VkExtent3D *extent)
  33 {
  34    const struct isl_format_layout *isl_layout =
  35       anv_format_for_vk_format(image->vk_format)->isl_layout;
  36    return anv_sanitize_image_extent(image->type, (VkExtent3D) {
  37       .width  = DIV_ROUND_UP(extent->width , isl_layout->bw),
  38       .height = DIV_ROUND_UP(extent->height, isl_layout->bh),
  39       .depth  = DIV_ROUND_UP(extent->depth , isl_layout->bd),
  40    });
  41 }
  42
  43 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  44  * elements rather than texels. One element equals one texel or one block
  45  * if Image is uncompressed or compressed, respectively.
  46  */
  47 static struct VkOffset3D
  48 meta_region_offset_el(const struct anv_image *image,
  49                       const struct VkOffset3D *offset)
  50 {
  51    const struct isl_format_layout *isl_layout = image->format->isl_layout;
  52    return anv_sanitize_image_offset(image->type, (VkOffset3D) {
  53       .x = offset->x / isl_layout->bw,
  54       .y = offset->y / isl_layout->bh,
  55       .z = offset->z / isl_layout->bd,
  56    });
  57 }
  58
  59 static struct anv_meta_blit2d_surf
  60 blit_surf_for_image(const struct anv_image* image,
  61                     const struct isl_surf *img_isl_surf)
  62 {
  63    return (struct anv_meta_blit2d_surf) {
  64       .bo = image->bo,
  65       .tiling = img_isl_surf->tiling,
  66       .base_offset = image->offset,
  67       .bs = isl_format_get_layout(img_isl_surf->format)->bs,
  68       .pitch = isl_surf_get_row_pitch(img_isl_surf),
  69    };
  70 }
  71
  72 static void
  73 do_buffer_copy(struct anv_cmd_buffer *cmd_buffer,
  74                struct anv_bo *src, uint64_t src_offset,
  75                struct anv_bo *dest, uint64_t dest_offset,
  76                int width, int height, int bs)
  77 {
  78    struct anv_meta_blit2d_surf b_src = {
  79       .bo = src,
  80       .tiling = ISL_TILING_LINEAR,
  81       .base_offset = src_offset,
  82       .bs = bs,
  83       .pitch = width * bs,
  84    };
  85    struct anv_meta_blit2d_surf b_dst = {
  86       .bo = dest,
  87       .tiling = ISL_TILING_LINEAR,
  88       .base_offset = dest_offset,
  89       .bs = bs,
  90       .pitch = width * bs,
  91    };
  92    struct anv_meta_blit2d_rect rect = {
  93       .width = width,
  94       .height = height,
  95    };
  96    anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
  97 }
  98
  99 static void
 100 meta_copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
 101                           struct anv_buffer* buffer,
 102                           struct anv_image* image,
 103                           uint32_t regionCount,
 104                           const VkBufferImageCopy* pRegions,
 105                           bool forward)
 106 {
 107    struct anv_meta_saved_state saved_state;
 108
 109    /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 110     * VK_SAMPLE_COUNT_1_BIT."
 111     */
 112    assert(image->samples == 1);
 113
 114    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 115
 116    for (unsigned r = 0; r < regionCount; r++) {
 117
 118       /**
 119        * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 120        *    extent is the size in texels of the source image to copy in width,
 121        *    height and depth. 1D images use only x and width. 2D images use x, y,
 122        *    width and height. 3D images use x, y, z, width, height and depth.
 123        *
 124        *
 125        * Also, convert the offsets and extent from units of texels to units of
 126        * blocks - which is the highest resolution accessible in this command.
 127        */
 128       const VkOffset3D img_offset_el =
 129          meta_region_offset_el(image, &pRegions[r].imageOffset);
 130       const VkExtent3D bufferExtent = {
 131          .width = pRegions[r].bufferRowLength,
 132          .height = pRegions[r].bufferImageHeight,
 133       };
 134
 135       /* Start creating blit rect */
 136       const VkExtent3D buf_extent_el =
 137          meta_region_extent_el(image, &bufferExtent);
 138       const VkExtent3D img_extent_el =
 139          meta_region_extent_el(image, &pRegions[r].imageExtent);
 140       struct anv_meta_blit2d_rect rect = {
 141          .width = MAX2(buf_extent_el.width, img_extent_el.width),
 142          .height = MAX2(buf_extent_el.height, img_extent_el.height),
 143       };
 144
 145       /* Create blit surfaces */
 146       VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 147       const struct isl_surf *img_isl_surf =
 148          &anv_image_get_surface_for_aspect_mask(image, aspect)->isl;
 149       struct anv_meta_blit2d_surf img_bsurf =
 150          blit_surf_for_image(image, img_isl_surf);
 151       struct anv_meta_blit2d_surf buf_bsurf = {
 152          .bo = buffer->bo,
 153          .tiling = ISL_TILING_LINEAR,
 154          .base_offset = buffer->offset + pRegions[r].bufferOffset,
 155          .bs = forward ? image->format->isl_layout->bs : img_bsurf.bs,
 156          .pitch = rect.width * buf_bsurf.bs,
 157       };
 158
 159       /* Set direction-dependent variables */
 160       struct anv_meta_blit2d_surf *dst_bsurf = forward ? &img_bsurf : &buf_bsurf;
 161       struct anv_meta_blit2d_surf *src_bsurf = forward ? &buf_bsurf : &img_bsurf;
 162       uint32_t *x_offset = forward ? &rect.dst_x : &rect.src_x;
 163       uint32_t *y_offset = forward ? &rect.dst_y : &rect.src_y;
 164
 165       /* Loop through each 3D or array slice */
 166       unsigned num_slices_3d = img_extent_el.depth;
 167       unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 168       unsigned slice_3d = 0;
 169       unsigned slice_array = 0;
 170       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 171
 172          /* Finish creating blit rect */
 173          isl_surf_get_image_offset_el(img_isl_surf,
 174                                     pRegions[r].imageSubresource.mipLevel,
 175                                     pRegions[r].imageSubresource.baseArrayLayer
 176                                        + slice_array,
 177                                     img_offset_el.z + slice_3d,
 178                                     x_offset,
 179                                     y_offset);
 180          *x_offset += img_offset_el.x;
 181          *y_offset += img_offset_el.y;
 182
 183          /* Perform Blit */
 184          anv_meta_blit2d(cmd_buffer, src_bsurf, dst_bsurf, 1, &rect);
 185
 186          /* Once we've done the blit, all of the actual information about
 187           * the image is embedded in the command buffer so we can just
 188           * increment the offset directly in the image effectively
 189           * re-binding it to different backing memory.
 190           */
 191          buf_bsurf.base_offset += rect.width * rect.height * buf_bsurf.bs;
 192
 193          if (image->type == VK_IMAGE_TYPE_3D)
 194             slice_3d++;
 195          else
 196             slice_array++;
 197       }
 198    }
 199    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 200 }
 201
 202 void anv_CmdCopyBufferToImage(
 203     VkCommandBuffer                             commandBuffer,
 204     VkBuffer                                    srcBuffer,
 205     VkImage                                     destImage,
 206     VkImageLayout                               destImageLayout,
 207     uint32_t                                    regionCount,
 208     const VkBufferImageCopy*                    pRegions)
 209 {
 210    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 211    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 212    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 213
 214    meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image,
 215                              regionCount, pRegions, true);
 216 }
 217
 218 void anv_CmdCopyImageToBuffer(
 219     VkCommandBuffer                             commandBuffer,
 220     VkImage                                     srcImage,
 221     VkImageLayout                               srcImageLayout,
 222     VkBuffer                                    destBuffer,
 223     uint32_t                                    regionCount,
 224     const VkBufferImageCopy*                    pRegions)
 225 {
 226    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 227    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 228    ANV_FROM_HANDLE(anv_buffer, dst_buffer, destBuffer);
 229
 230    meta_copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
 231                              regionCount, pRegions, false);
 232 }
 233
 234 void anv_CmdCopyImage(
 235     VkCommandBuffer                             commandBuffer,
 236     VkImage                                     srcImage,
 237     VkImageLayout                               srcImageLayout,
 238     VkImage                                     destImage,
 239     VkImageLayout                               destImageLayout,
 240     uint32_t                                    regionCount,
 241     const VkImageCopy*                          pRegions)
 242 {
 243    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 244    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 245    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 246    struct anv_meta_saved_state saved_state;
 247
 248    /* From the Vulkan 1.0 spec:
 249     *
 250     *    vkCmdCopyImage can be used to copy image data between multisample
 251     *    images, but both images must have the same number of samples.
 252     */
 253    assert(src_image->samples == dest_image->samples);
 254
 255    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 256
 257    for (unsigned r = 0; r < regionCount; r++) {
 258       assert(pRegions[r].srcSubresource.aspectMask ==
 259              pRegions[r].dstSubresource.aspectMask);
 260
 261       VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask;
 262
 263       /* Create blit surfaces */
 264       struct isl_surf *src_isl_surf =
 265          &anv_image_get_surface_for_aspect_mask(src_image, aspect)->isl;
 266       struct isl_surf *dst_isl_surf =
 267          &anv_image_get_surface_for_aspect_mask(dest_image, aspect)->isl;
 268       struct anv_meta_blit2d_surf b_src =
 269          blit_surf_for_image(src_image, src_isl_surf);
 270       struct anv_meta_blit2d_surf b_dst =
 271          blit_surf_for_image(dest_image, dst_isl_surf);
 272
 273       /**
 274        * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 275        *    imageExtent is the size in texels of the image to copy in width, height
 276        *    and depth. 1D images use only x and width. 2D images use x, y, width
 277        *    and height. 3D images use x, y, z, width, height and depth.
 278        *
 279        * Also, convert the offsets and extent from units of texels to units of
 280        * blocks - which is the highest resolution accessible in this command.
 281        */
 282       const VkOffset3D dst_offset_el =
 283          meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 284       const VkOffset3D src_offset_el =
 285          meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 286       const VkExtent3D img_extent_el =
 287          meta_region_extent_el(src_image, &pRegions[r].extent);
 288
 289       /* Start creating blit rect */
 290       struct anv_meta_blit2d_rect rect = {
 291          .width = img_extent_el.width,
 292          .height = img_extent_el.height,
 293       };
 294
 295       /* Loop through each 3D or array slice */
 296       unsigned num_slices_3d = img_extent_el.depth;
 297       unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 298       unsigned slice_3d = 0;
 299       unsigned slice_array = 0;
 300       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 301
 302          /* Finish creating blit rect */
 303          isl_surf_get_image_offset_el(dst_isl_surf,
 304                                     pRegions[r].dstSubresource.mipLevel,
 305                                     pRegions[r].dstSubresource.baseArrayLayer
 306                                        + slice_array,
 307                                     dst_offset_el.z + slice_3d,
 308                                     &rect.dst_x,
 309                                     &rect.dst_y);
 310          isl_surf_get_image_offset_el(src_isl_surf,
 311                                     pRegions[r].srcSubresource.mipLevel,
 312                                     pRegions[r].srcSubresource.baseArrayLayer
 313                                        + slice_array,
 314                                     src_offset_el.z + slice_3d,
 315                                     &rect.src_x,
 316                                     &rect.src_y);
 317          rect.dst_x += dst_offset_el.x;
 318          rect.dst_y += dst_offset_el.y;
 319          rect.src_x += src_offset_el.x;
 320          rect.src_y += src_offset_el.y;
 321
 322          /* Perform Blit */
 323          anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
 324
 325          if (dest_image->type == VK_IMAGE_TYPE_3D)
 326             slice_3d++;
 327          else
 328             slice_array++;
 329       }
 330    }
 331
 332    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 333 }
 334
 335 void anv_CmdCopyBuffer(
 336     VkCommandBuffer                             commandBuffer,
 337     VkBuffer                                    srcBuffer,
 338     VkBuffer                                    destBuffer,
 339     uint32_t                                    regionCount,
 340     const VkBufferCopy*                         pRegions)
 341 {
 342    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 343    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 344    ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer);
 345
 346    struct anv_meta_saved_state saved_state;
 347
 348    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 349
 350    for (unsigned r = 0; r < regionCount; r++) {
 351       uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset;
 352       uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset;
 353       uint64_t copy_size = pRegions[r].size;
 354
 355       /* First, we compute the biggest format that can be used with the
 356        * given offsets and size.
 357        */
 358       int bs = 16;
 359
 360       int fs = ffs(src_offset) - 1;
 361       if (fs != -1)
 362          bs = MIN2(bs, 1 << fs);
 363       assert(src_offset % bs == 0);
 364
 365       fs = ffs(dest_offset) - 1;
 366       if (fs != -1)
 367          bs = MIN2(bs, 1 << fs);
 368       assert(dest_offset % bs == 0);
 369
 370       fs = ffs(pRegions[r].size) - 1;
 371       if (fs != -1)
 372          bs = MIN2(bs, 1 << fs);
 373       assert(pRegions[r].size % bs == 0);
 374
 375       /* This is maximum possible width/height our HW can handle */
 376       uint64_t max_surface_dim = 1 << 14;
 377
 378       /* First, we make a bunch of max-sized copies */
 379       uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
 380       while (copy_size >= max_copy_size) {
 381          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 382                         dest_buffer->bo, dest_offset,
 383                         max_surface_dim, max_surface_dim, bs);
 384          copy_size -= max_copy_size;
 385          src_offset += max_copy_size;
 386          dest_offset += max_copy_size;
 387       }
 388
 389       uint64_t height = copy_size / (max_surface_dim * bs);
 390       assert(height < max_surface_dim);
 391       if (height != 0) {
 392          uint64_t rect_copy_size = height * max_surface_dim * bs;
 393          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 394                         dest_buffer->bo, dest_offset,
 395                         max_surface_dim, height, bs);
 396          copy_size -= rect_copy_size;
 397          src_offset += rect_copy_size;
 398          dest_offset += rect_copy_size;
 399       }
 400
 401       if (copy_size != 0) {
 402          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 403                         dest_buffer->bo, dest_offset,
 404                         copy_size / bs, 1, bs);
 405       }
 406    }
 407
 408    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 409 }
 410
 411 void anv_CmdUpdateBuffer(
 412     VkCommandBuffer                             commandBuffer,
 413     VkBuffer                                    dstBuffer,
 414     VkDeviceSize                                dstOffset,
 415     VkDeviceSize                                dataSize,
 416     const uint32_t*                             pData)
 417 {
 418    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 419    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 420    struct anv_meta_saved_state saved_state;
 421
 422    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 423
 424    /* We can't quite grab a full block because the state stream needs a
 425     * little data at the top to build its linked list.
 426     */
 427    const uint32_t max_update_size =
 428       cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
 429
 430    assert(max_update_size < (1 << 14) * 4);
 431
 432    while (dataSize) {
 433       const uint32_t copy_size = MIN2(dataSize, max_update_size);
 434
 435       struct anv_state tmp_data =
 436          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
 437
 438       memcpy(tmp_data.map, pData, copy_size);
 439
 440       int bs;
 441       if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) {
 442          bs = 16;
 443       } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) {
 444          bs = 8;
 445       } else {
 446          assert((copy_size & 3) == 0 && (dstOffset & 3) == 0);
 447          bs = 4;
 448       }
 449
 450       do_buffer_copy(cmd_buffer,
 451                      &cmd_buffer->device->dynamic_state_block_pool.bo,
 452                      tmp_data.offset,
 453                      dst_buffer->bo, dst_buffer->offset + dstOffset,
 454                      copy_size / bs, 1, bs);
 455
 456       dataSize -= copy_size;
 457       dstOffset += copy_size;
 458       pData = (void *)pData + copy_size;
 459    }
 460
 461    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 462 }