src/intel/vulkan/anv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_meta.h"
  25
  26 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  27  * elements rather than texels. One element equals one texel or one block
  28  * if Image is uncompressed or compressed, respectively.
  29  */
  30 static struct VkExtent3D
  31 meta_region_extent_el(const VkFormat format,
  32                       const struct VkExtent3D *extent)
  33 {
  34    const struct isl_format_layout *isl_layout =
  35       anv_format_for_vk_format(format)->isl_layout;
  36    return (VkExtent3D) {
  37       .width  = DIV_ROUND_UP(extent->width , isl_layout->bw),
  38       .height = DIV_ROUND_UP(extent->height, isl_layout->bh),
  39       .depth  = DIV_ROUND_UP(extent->depth , isl_layout->bd),
  40    };
  41 }
  42
  43 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  44  * elements rather than texels. One element equals one texel or one block
  45  * if Image is uncompressed or compressed, respectively.
  46  */
  47 static struct VkOffset3D
  48 meta_region_offset_el(const struct anv_image *image,
  49                       const struct VkOffset3D *offset)
  50 {
  51    const struct isl_format_layout *isl_layout = image->format->isl_layout;
  52    return (VkOffset3D) {
  53       .x = offset->x / isl_layout->bw,
  54       .y = offset->y / isl_layout->bh,
  55       .z = offset->z / isl_layout->bd,
  56    };
  57 }
  58
  59 static struct anv_meta_blit2d_surf
  60 blit_surf_for_image(const struct anv_image* image,
  61                     const struct isl_surf *img_isl_surf)
  62 {
  63    return (struct anv_meta_blit2d_surf) {
  64       .bo = image->bo,
  65       .tiling = img_isl_surf->tiling,
  66       .base_offset = image->offset,
  67       .bs = isl_format_get_layout(img_isl_surf->format)->bs,
  68       .pitch = isl_surf_get_row_pitch(img_isl_surf),
  69    };
  70 }
  71
  72 static void
  73 do_buffer_copy(struct anv_cmd_buffer *cmd_buffer,
  74                struct anv_bo *src, uint64_t src_offset,
  75                struct anv_bo *dest, uint64_t dest_offset,
  76                int width, int height, int bs)
  77 {
  78    struct anv_meta_blit2d_surf b_src = {
  79       .bo = src,
  80       .tiling = ISL_TILING_LINEAR,
  81       .base_offset = src_offset,
  82       .bs = bs,
  83       .pitch = width * bs,
  84    };
  85    struct anv_meta_blit2d_surf b_dst = {
  86       .bo = dest,
  87       .tiling = ISL_TILING_LINEAR,
  88       .base_offset = dest_offset,
  89       .bs = bs,
  90       .pitch = width * bs,
  91    };
  92    struct anv_meta_blit2d_rect rect = {
  93       .width = width,
  94       .height = height,
  95    };
  96    anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
  97 }
  98
  99 static void
 100 meta_copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
 101                           struct anv_buffer* buffer,
 102                           struct anv_image* image,
 103                           uint32_t regionCount,
 104                           const VkBufferImageCopy* pRegions,
 105                           bool forward)
 106 {
 107    struct anv_meta_saved_state saved_state;
 108
 109    /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 110     * VK_SAMPLE_COUNT_1_BIT."
 111     */
 112    assert(image->samples == 1);
 113
 114    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 115
 116    for (unsigned r = 0; r < regionCount; r++) {
 117
 118       /* Start creating blit rect */
 119       const VkOffset3D img_offset_el =
 120          meta_region_offset_el(image, &pRegions[r].imageOffset);
 121       const VkExtent3D bufferExtent = {
 122          .width = pRegions[r].bufferRowLength,
 123          .height = pRegions[r].bufferImageHeight,
 124       };
 125       const VkExtent3D buf_extent_el =
 126          meta_region_extent_el(image->vk_format, &bufferExtent);
 127       const VkExtent3D img_extent_el =
 128          meta_region_extent_el(image->vk_format, &pRegions[r].imageExtent);
 129       struct anv_meta_blit2d_rect rect = {
 130          .width = MAX2(buf_extent_el.width, img_extent_el.width),
 131          .height = MAX2(buf_extent_el.height, img_extent_el.height),
 132       };
 133
 134       /* Create blit surfaces */
 135       VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 136       const struct isl_surf *img_isl_surf =
 137          &anv_image_get_surface_for_aspect_mask(image, aspect)->isl;
 138       struct anv_meta_blit2d_surf img_bsurf =
 139          blit_surf_for_image(image, img_isl_surf);
 140       struct anv_meta_blit2d_surf buf_bsurf = {
 141          .bo = buffer->bo,
 142          .tiling = ISL_TILING_LINEAR,
 143          .base_offset = buffer->offset + pRegions[r].bufferOffset,
 144          .bs = forward ? image->format->isl_layout->bs : img_bsurf.bs,
 145          .pitch = rect.width * buf_bsurf.bs,
 146       };
 147
 148       /* Set direction-dependent variables */
 149       struct anv_meta_blit2d_surf *dst_bsurf = forward ? &img_bsurf : &buf_bsurf;
 150       struct anv_meta_blit2d_surf *src_bsurf = forward ? &buf_bsurf : &img_bsurf;
 151       uint32_t *x_offset = forward ? &rect.dst_x : &rect.src_x;
 152       uint32_t *y_offset = forward ? &rect.dst_y : &rect.src_y;
 153
 154       /* Loop through each 3D or array slice */
 155       unsigned num_slices_3d = pRegions[r].imageExtent.depth;
 156       unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 157       unsigned slice_3d = 0;
 158       unsigned slice_array = 0;
 159       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 160
 161          /* Finish creating blit rect */
 162          isl_surf_get_image_offset_el(img_isl_surf,
 163                                     pRegions[r].imageSubresource.mipLevel,
 164                                     pRegions[r].imageSubresource.baseArrayLayer
 165                                        + slice_array,
 166                                     pRegions[r].imageOffset.z + slice_3d,
 167                                     x_offset,
 168                                     y_offset);
 169          *x_offset += img_offset_el.x;
 170          *y_offset += img_offset_el.y;
 171
 172          /* Perform Blit */
 173          anv_meta_blit2d(cmd_buffer, src_bsurf, dst_bsurf, 1, &rect);
 174
 175          /* Once we've done the blit, all of the actual information about
 176           * the image is embedded in the command buffer so we can just
 177           * increment the offset directly in the image effectively
 178           * re-binding it to different backing memory.
 179           */
 180          buf_bsurf.base_offset += rect.width * rect.height * buf_bsurf.bs;
 181
 182          if (image->type == VK_IMAGE_TYPE_3D)
 183             slice_3d++;
 184          else
 185             slice_array++;
 186       }
 187    }
 188    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 189 }
 190
 191 void anv_CmdCopyBufferToImage(
 192     VkCommandBuffer                             commandBuffer,
 193     VkBuffer                                    srcBuffer,
 194     VkImage                                     destImage,
 195     VkImageLayout                               destImageLayout,
 196     uint32_t                                    regionCount,
 197     const VkBufferImageCopy*                    pRegions)
 198 {
 199    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 200    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 201    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 202
 203    meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image,
 204                              regionCount, pRegions, true);
 205 }
 206
 207 void anv_CmdCopyImageToBuffer(
 208     VkCommandBuffer                             commandBuffer,
 209     VkImage                                     srcImage,
 210     VkImageLayout                               srcImageLayout,
 211     VkBuffer                                    destBuffer,
 212     uint32_t                                    regionCount,
 213     const VkBufferImageCopy*                    pRegions)
 214 {
 215    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 216    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 217    ANV_FROM_HANDLE(anv_buffer, dst_buffer, destBuffer);
 218
 219    meta_copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
 220                              regionCount, pRegions, false);
 221 }
 222
 223 void anv_CmdCopyImage(
 224     VkCommandBuffer                             commandBuffer,
 225     VkImage                                     srcImage,
 226     VkImageLayout                               srcImageLayout,
 227     VkImage                                     destImage,
 228     VkImageLayout                               destImageLayout,
 229     uint32_t                                    regionCount,
 230     const VkImageCopy*                          pRegions)
 231 {
 232    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 233    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 234    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 235    struct anv_meta_saved_state saved_state;
 236
 237    /* From the Vulkan 1.0 spec:
 238     *
 239     *    vkCmdCopyImage can be used to copy image data between multisample
 240     *    images, but both images must have the same number of samples.
 241     */
 242    assert(src_image->samples == dest_image->samples);
 243
 244    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 245
 246    for (unsigned r = 0; r < regionCount; r++) {
 247       assert(pRegions[r].srcSubresource.aspectMask ==
 248              pRegions[r].dstSubresource.aspectMask);
 249
 250       VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask;
 251
 252       /* Create blit surfaces */
 253       struct isl_surf *src_isl_surf =
 254          &anv_image_get_surface_for_aspect_mask(src_image, aspect)->isl;
 255       struct isl_surf *dst_isl_surf =
 256          &anv_image_get_surface_for_aspect_mask(dest_image, aspect)->isl;
 257       struct anv_meta_blit2d_surf b_src =
 258          blit_surf_for_image(src_image, src_isl_surf);
 259       struct anv_meta_blit2d_surf b_dst =
 260          blit_surf_for_image(dest_image, dst_isl_surf);
 261
 262       /* Start creating blit rect */
 263       const VkOffset3D dst_offset_el =
 264          meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 265       const VkOffset3D src_offset_el =
 266          meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 267       const VkExtent3D img_extent_el =
 268          meta_region_extent_el(src_image->vk_format, &pRegions[r].extent);
 269       struct anv_meta_blit2d_rect rect = {
 270          .width = img_extent_el.width,
 271          .height = img_extent_el.height,
 272       };
 273
 274       /* Loop through each 3D or array slice */
 275       unsigned num_slices_3d = pRegions[r].extent.depth;
 276       unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 277       unsigned slice_3d = 0;
 278       unsigned slice_array = 0;
 279       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 280
 281          /* Finish creating blit rect */
 282          isl_surf_get_image_offset_el(dst_isl_surf,
 283                                     pRegions[r].dstSubresource.mipLevel,
 284                                     pRegions[r].dstSubresource.baseArrayLayer
 285                                        + slice_array,
 286                                     pRegions[r].dstOffset.z + slice_3d,
 287                                     &rect.dst_x,
 288                                     &rect.dst_y);
 289          isl_surf_get_image_offset_el(src_isl_surf,
 290                                     pRegions[r].srcSubresource.mipLevel,
 291                                     pRegions[r].srcSubresource.baseArrayLayer
 292                                        + slice_array,
 293                                     pRegions[r].srcOffset.z + slice_3d,
 294                                     &rect.src_x,
 295                                     &rect.src_y);
 296          rect.dst_x += dst_offset_el.x;
 297          rect.dst_y += dst_offset_el.y;
 298          rect.src_x += src_offset_el.x;
 299          rect.src_y += src_offset_el.y;
 300
 301          /* Perform Blit */
 302          anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
 303
 304          if (dest_image->type == VK_IMAGE_TYPE_3D)
 305             slice_3d++;
 306          else
 307             slice_array++;
 308       }
 309    }
 310
 311    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 312 }
 313
 314 void anv_CmdCopyBuffer(
 315     VkCommandBuffer                             commandBuffer,
 316     VkBuffer                                    srcBuffer,
 317     VkBuffer                                    destBuffer,
 318     uint32_t                                    regionCount,
 319     const VkBufferCopy*                         pRegions)
 320 {
 321    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 322    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 323    ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer);
 324
 325    struct anv_meta_saved_state saved_state;
 326
 327    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 328
 329    for (unsigned r = 0; r < regionCount; r++) {
 330       uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset;
 331       uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset;
 332       uint64_t copy_size = pRegions[r].size;
 333
 334       /* First, we compute the biggest format that can be used with the
 335        * given offsets and size.
 336        */
 337       int bs = 16;
 338
 339       int fs = ffs(src_offset) - 1;
 340       if (fs != -1)
 341          bs = MIN2(bs, 1 << fs);
 342       assert(src_offset % bs == 0);
 343
 344       fs = ffs(dest_offset) - 1;
 345       if (fs != -1)
 346          bs = MIN2(bs, 1 << fs);
 347       assert(dest_offset % bs == 0);
 348
 349       fs = ffs(pRegions[r].size) - 1;
 350       if (fs != -1)
 351          bs = MIN2(bs, 1 << fs);
 352       assert(pRegions[r].size % bs == 0);
 353
 354       /* This is maximum possible width/height our HW can handle */
 355       uint64_t max_surface_dim = 1 << 14;
 356
 357       /* First, we make a bunch of max-sized copies */
 358       uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
 359       while (copy_size >= max_copy_size) {
 360          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 361                         dest_buffer->bo, dest_offset,
 362                         max_surface_dim, max_surface_dim, bs);
 363          copy_size -= max_copy_size;
 364          src_offset += max_copy_size;
 365          dest_offset += max_copy_size;
 366       }
 367
 368       uint64_t height = copy_size / (max_surface_dim * bs);
 369       assert(height < max_surface_dim);
 370       if (height != 0) {
 371          uint64_t rect_copy_size = height * max_surface_dim * bs;
 372          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 373                         dest_buffer->bo, dest_offset,
 374                         max_surface_dim, height, bs);
 375          copy_size -= rect_copy_size;
 376          src_offset += rect_copy_size;
 377          dest_offset += rect_copy_size;
 378       }
 379
 380       if (copy_size != 0) {
 381          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 382                         dest_buffer->bo, dest_offset,
 383                         copy_size / bs, 1, bs);
 384       }
 385    }
 386
 387    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 388 }
 389
 390 void anv_CmdUpdateBuffer(
 391     VkCommandBuffer                             commandBuffer,
 392     VkBuffer                                    dstBuffer,
 393     VkDeviceSize                                dstOffset,
 394     VkDeviceSize                                dataSize,
 395     const uint32_t*                             pData)
 396 {
 397    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 398    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 399    struct anv_meta_saved_state saved_state;
 400
 401    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 402
 403    /* We can't quite grab a full block because the state stream needs a
 404     * little data at the top to build its linked list.
 405     */
 406    const uint32_t max_update_size =
 407       cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
 408
 409    assert(max_update_size < (1 << 14) * 4);
 410
 411    while (dataSize) {
 412       const uint32_t copy_size = MIN2(dataSize, max_update_size);
 413
 414       struct anv_state tmp_data =
 415          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
 416
 417       memcpy(tmp_data.map, pData, copy_size);
 418
 419       int bs;
 420       if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) {
 421          bs = 16;
 422       } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) {
 423          bs = 8;
 424       } else {
 425          assert((copy_size & 3) == 0 && (dstOffset & 3) == 0);
 426          bs = 4;
 427       }
 428
 429       do_buffer_copy(cmd_buffer,
 430                      &cmd_buffer->device->dynamic_state_block_pool.bo,
 431                      tmp_data.offset,
 432                      dst_buffer->bo, dst_buffer->offset + dstOffset,
 433                      copy_size / bs, 1, bs);
 434
 435       dataSize -= copy_size;
 436       dstOffset += copy_size;
 437       pData = (void *)pData + copy_size;
 438    }
 439
 440    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 441 }