src/intel/vulkan/anv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_meta.h"
  25
  26 static VkExtent3D
  27 meta_image_block_size(const struct anv_image *image)
  28 {
  29    if (image->aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
  30       const struct isl_format_layout *isl_layout =
  31          isl_format_get_layout(image->color_surface.isl.format);
  32       return (VkExtent3D) { isl_layout->bw, isl_layout->bh, isl_layout->bd };
  33    } else {
  34       return (VkExtent3D) { 1, 1, 1 };
  35    }
  36 }
  37
  38 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  39  * elements rather than texels. One element equals one texel or one block
  40  * if Image is uncompressed or compressed, respectively.
  41  */
  42 static struct VkExtent3D
  43 meta_region_extent_el(const struct anv_image *image,
  44                       const struct VkExtent3D *extent)
  45 {
  46    const VkExtent3D block = meta_image_block_size(image);
  47    return anv_sanitize_image_extent(image->type, (VkExtent3D) {
  48       .width  = DIV_ROUND_UP(extent->width , block.width),
  49       .height = DIV_ROUND_UP(extent->height, block.height),
  50       .depth  = DIV_ROUND_UP(extent->depth , block.depth),
  51    });
  52 }
  53
  54 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  55  * elements rather than texels. One element equals one texel or one block
  56  * if Image is uncompressed or compressed, respectively.
  57  */
  58 static struct VkOffset3D
  59 meta_region_offset_el(const struct anv_image *image,
  60                       const struct VkOffset3D *offset)
  61 {
  62    const VkExtent3D block = meta_image_block_size(image);
  63    return anv_sanitize_image_offset(image->type, (VkOffset3D) {
  64       .x = offset->x / block.width,
  65       .y = offset->y / block.height,
  66       .z = offset->z / block.depth,
  67    });
  68 }
  69
  70 static struct anv_meta_blit2d_surf
  71 blit_surf_for_image(const struct anv_image* image,
  72                     const struct isl_surf *img_isl_surf)
  73 {
  74    return (struct anv_meta_blit2d_surf) {
  75       .bo = image->bo,
  76       .tiling = img_isl_surf->tiling,
  77       .base_offset = image->offset,
  78       .bs = isl_format_get_layout(img_isl_surf->format)->bs,
  79       .pitch = isl_surf_get_row_pitch(img_isl_surf),
  80    };
  81 }
  82
  83 static void
  84 do_buffer_copy(struct anv_cmd_buffer *cmd_buffer,
  85                struct anv_bo *src, uint64_t src_offset,
  86                struct anv_bo *dest, uint64_t dest_offset,
  87                int width, int height, int bs)
  88 {
  89    struct anv_meta_blit2d_surf b_src = {
  90       .bo = src,
  91       .tiling = ISL_TILING_LINEAR,
  92       .base_offset = src_offset,
  93       .bs = bs,
  94       .pitch = width * bs,
  95    };
  96    struct anv_meta_blit2d_surf b_dst = {
  97       .bo = dest,
  98       .tiling = ISL_TILING_LINEAR,
  99       .base_offset = dest_offset,
 100       .bs = bs,
 101       .pitch = width * bs,
 102    };
 103    struct anv_meta_blit2d_rect rect = {
 104       .width = width,
 105       .height = height,
 106    };
 107    anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
 108 }
 109
 110 static void
 111 meta_copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
 112                           struct anv_buffer* buffer,
 113                           struct anv_image* image,
 114                           uint32_t regionCount,
 115                           const VkBufferImageCopy* pRegions,
 116                           bool forward)
 117 {
 118    struct anv_meta_saved_state saved_state;
 119
 120    /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 121     * VK_SAMPLE_COUNT_1_BIT."
 122     */
 123    assert(image->samples == 1);
 124
 125    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 126
 127    for (unsigned r = 0; r < regionCount; r++) {
 128
 129       /**
 130        * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 131        *    extent is the size in texels of the source image to copy in width,
 132        *    height and depth. 1D images use only x and width. 2D images use x, y,
 133        *    width and height. 3D images use x, y, z, width, height and depth.
 134        *
 135        *
 136        * Also, convert the offsets and extent from units of texels to units of
 137        * blocks - which is the highest resolution accessible in this command.
 138        */
 139       const VkOffset3D img_offset_el =
 140          meta_region_offset_el(image, &pRegions[r].imageOffset);
 141       const VkExtent3D bufferExtent = {
 142          .width  = pRegions[r].bufferRowLength ?
 143                    pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 144          .height = pRegions[r].bufferImageHeight ?
 145                    pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 146       };
 147       const VkExtent3D buf_extent_el =
 148          meta_region_extent_el(image, &bufferExtent);
 149
 150       /* Start creating blit rect */
 151       const VkExtent3D img_extent_el =
 152          meta_region_extent_el(image, &pRegions[r].imageExtent);
 153       struct anv_meta_blit2d_rect rect = {
 154          .width = img_extent_el.width,
 155          .height =  img_extent_el.height,
 156       };
 157
 158       /* Create blit surfaces */
 159       VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 160       const struct isl_surf *img_isl_surf =
 161          &anv_image_get_surface_for_aspect_mask(image, aspect)->isl;
 162       struct anv_meta_blit2d_surf img_bsurf =
 163          blit_surf_for_image(image, img_isl_surf);
 164       enum isl_format buf_format = anv_get_isl_format(image->vk_format, aspect,
 165                                                       VK_IMAGE_TILING_LINEAR,
 166                                                       NULL);
 167       struct anv_meta_blit2d_surf buf_bsurf = {
 168          .bo = buffer->bo,
 169          .tiling = ISL_TILING_LINEAR,
 170          .base_offset = buffer->offset + pRegions[r].bufferOffset,
 171          .bs = isl_format_get_layout(buf_format)->bs,
 172          .pitch = buf_extent_el.width * buf_bsurf.bs,
 173       };
 174
 175       /* Set direction-dependent variables */
 176       struct anv_meta_blit2d_surf *dst_bsurf = forward ? &img_bsurf : &buf_bsurf;
 177       struct anv_meta_blit2d_surf *src_bsurf = forward ? &buf_bsurf : &img_bsurf;
 178       uint32_t *x_offset = forward ? &rect.dst_x : &rect.src_x;
 179       uint32_t *y_offset = forward ? &rect.dst_y : &rect.src_y;
 180
 181       /* Loop through each 3D or array slice */
 182       unsigned num_slices_3d = img_extent_el.depth;
 183       unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 184       unsigned slice_3d = 0;
 185       unsigned slice_array = 0;
 186       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 187
 188          /* Finish creating blit rect */
 189          isl_surf_get_image_offset_el(img_isl_surf,
 190                                     pRegions[r].imageSubresource.mipLevel,
 191                                     pRegions[r].imageSubresource.baseArrayLayer
 192                                        + slice_array,
 193                                     img_offset_el.z + slice_3d,
 194                                     x_offset,
 195                                     y_offset);
 196          *x_offset += img_offset_el.x;
 197          *y_offset += img_offset_el.y;
 198
 199          /* Perform Blit */
 200          anv_meta_blit2d(cmd_buffer, src_bsurf, dst_bsurf, 1, &rect);
 201
 202          /* Once we've done the blit, all of the actual information about
 203           * the image is embedded in the command buffer so we can just
 204           * increment the offset directly in the image effectively
 205           * re-binding it to different backing memory.
 206           */
 207          buf_bsurf.base_offset += buf_extent_el.width *
 208                                   buf_extent_el.height * buf_bsurf.bs;
 209
 210          if (image->type == VK_IMAGE_TYPE_3D)
 211             slice_3d++;
 212          else
 213             slice_array++;
 214       }
 215    }
 216    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 217 }
 218
 219 void anv_CmdCopyBufferToImage(
 220     VkCommandBuffer                             commandBuffer,
 221     VkBuffer                                    srcBuffer,
 222     VkImage                                     destImage,
 223     VkImageLayout                               destImageLayout,
 224     uint32_t                                    regionCount,
 225     const VkBufferImageCopy*                    pRegions)
 226 {
 227    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 228    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 229    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 230
 231    meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image,
 232                              regionCount, pRegions, true);
 233 }
 234
 235 void anv_CmdCopyImageToBuffer(
 236     VkCommandBuffer                             commandBuffer,
 237     VkImage                                     srcImage,
 238     VkImageLayout                               srcImageLayout,
 239     VkBuffer                                    destBuffer,
 240     uint32_t                                    regionCount,
 241     const VkBufferImageCopy*                    pRegions)
 242 {
 243    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 244    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 245    ANV_FROM_HANDLE(anv_buffer, dst_buffer, destBuffer);
 246
 247    meta_copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
 248                              regionCount, pRegions, false);
 249 }
 250
 251 void anv_CmdCopyImage(
 252     VkCommandBuffer                             commandBuffer,
 253     VkImage                                     srcImage,
 254     VkImageLayout                               srcImageLayout,
 255     VkImage                                     destImage,
 256     VkImageLayout                               destImageLayout,
 257     uint32_t                                    regionCount,
 258     const VkImageCopy*                          pRegions)
 259 {
 260    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 261    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 262    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 263    struct anv_meta_saved_state saved_state;
 264
 265    /* From the Vulkan 1.0 spec:
 266     *
 267     *    vkCmdCopyImage can be used to copy image data between multisample
 268     *    images, but both images must have the same number of samples.
 269     */
 270    assert(src_image->samples == dest_image->samples);
 271
 272    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 273
 274    for (unsigned r = 0; r < regionCount; r++) {
 275       assert(pRegions[r].srcSubresource.aspectMask ==
 276              pRegions[r].dstSubresource.aspectMask);
 277
 278       VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask;
 279
 280       /* Create blit surfaces */
 281       struct isl_surf *src_isl_surf =
 282          &anv_image_get_surface_for_aspect_mask(src_image, aspect)->isl;
 283       struct isl_surf *dst_isl_surf =
 284          &anv_image_get_surface_for_aspect_mask(dest_image, aspect)->isl;
 285       struct anv_meta_blit2d_surf b_src =
 286          blit_surf_for_image(src_image, src_isl_surf);
 287       struct anv_meta_blit2d_surf b_dst =
 288          blit_surf_for_image(dest_image, dst_isl_surf);
 289
 290       /**
 291        * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 292        *    imageExtent is the size in texels of the image to copy in width, height
 293        *    and depth. 1D images use only x and width. 2D images use x, y, width
 294        *    and height. 3D images use x, y, z, width, height and depth.
 295        *
 296        * Also, convert the offsets and extent from units of texels to units of
 297        * blocks - which is the highest resolution accessible in this command.
 298        */
 299       const VkOffset3D dst_offset_el =
 300          meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 301       const VkOffset3D src_offset_el =
 302          meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 303       const VkExtent3D img_extent_el =
 304          meta_region_extent_el(src_image, &pRegions[r].extent);
 305
 306       /* Start creating blit rect */
 307       struct anv_meta_blit2d_rect rect = {
 308          .width = img_extent_el.width,
 309          .height = img_extent_el.height,
 310       };
 311
 312       /* Loop through each 3D or array slice */
 313       unsigned num_slices_3d = img_extent_el.depth;
 314       unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 315       unsigned slice_3d = 0;
 316       unsigned slice_array = 0;
 317       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 318
 319          /* Finish creating blit rect */
 320          isl_surf_get_image_offset_el(dst_isl_surf,
 321                                     pRegions[r].dstSubresource.mipLevel,
 322                                     pRegions[r].dstSubresource.baseArrayLayer
 323                                        + slice_array,
 324                                     dst_offset_el.z + slice_3d,
 325                                     &rect.dst_x,
 326                                     &rect.dst_y);
 327          isl_surf_get_image_offset_el(src_isl_surf,
 328                                     pRegions[r].srcSubresource.mipLevel,
 329                                     pRegions[r].srcSubresource.baseArrayLayer
 330                                        + slice_array,
 331                                     src_offset_el.z + slice_3d,
 332                                     &rect.src_x,
 333                                     &rect.src_y);
 334          rect.dst_x += dst_offset_el.x;
 335          rect.dst_y += dst_offset_el.y;
 336          rect.src_x += src_offset_el.x;
 337          rect.src_y += src_offset_el.y;
 338
 339          /* Perform Blit */
 340          anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
 341
 342          if (dest_image->type == VK_IMAGE_TYPE_3D)
 343             slice_3d++;
 344          else
 345             slice_array++;
 346       }
 347    }
 348
 349    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 350 }
 351
 352 void anv_CmdCopyBuffer(
 353     VkCommandBuffer                             commandBuffer,
 354     VkBuffer                                    srcBuffer,
 355     VkBuffer                                    destBuffer,
 356     uint32_t                                    regionCount,
 357     const VkBufferCopy*                         pRegions)
 358 {
 359    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 360    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 361    ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer);
 362
 363    struct anv_meta_saved_state saved_state;
 364
 365    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 366
 367    for (unsigned r = 0; r < regionCount; r++) {
 368       uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset;
 369       uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset;
 370       uint64_t copy_size = pRegions[r].size;
 371
 372       /* First, we compute the biggest format that can be used with the
 373        * given offsets and size.
 374        */
 375       int bs = 16;
 376
 377       int fs = ffs(src_offset) - 1;
 378       if (fs != -1)
 379          bs = MIN2(bs, 1 << fs);
 380       assert(src_offset % bs == 0);
 381
 382       fs = ffs(dest_offset) - 1;
 383       if (fs != -1)
 384          bs = MIN2(bs, 1 << fs);
 385       assert(dest_offset % bs == 0);
 386
 387       fs = ffs(pRegions[r].size) - 1;
 388       if (fs != -1)
 389          bs = MIN2(bs, 1 << fs);
 390       assert(pRegions[r].size % bs == 0);
 391
 392       /* This is maximum possible width/height our HW can handle */
 393       uint64_t max_surface_dim = 1 << 14;
 394
 395       /* First, we make a bunch of max-sized copies */
 396       uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
 397       while (copy_size >= max_copy_size) {
 398          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 399                         dest_buffer->bo, dest_offset,
 400                         max_surface_dim, max_surface_dim, bs);
 401          copy_size -= max_copy_size;
 402          src_offset += max_copy_size;
 403          dest_offset += max_copy_size;
 404       }
 405
 406       uint64_t height = copy_size / (max_surface_dim * bs);
 407       assert(height < max_surface_dim);
 408       if (height != 0) {
 409          uint64_t rect_copy_size = height * max_surface_dim * bs;
 410          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 411                         dest_buffer->bo, dest_offset,
 412                         max_surface_dim, height, bs);
 413          copy_size -= rect_copy_size;
 414          src_offset += rect_copy_size;
 415          dest_offset += rect_copy_size;
 416       }
 417
 418       if (copy_size != 0) {
 419          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 420                         dest_buffer->bo, dest_offset,
 421                         copy_size / bs, 1, bs);
 422       }
 423    }
 424
 425    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 426 }
 427
 428 void anv_CmdUpdateBuffer(
 429     VkCommandBuffer                             commandBuffer,
 430     VkBuffer                                    dstBuffer,
 431     VkDeviceSize                                dstOffset,
 432     VkDeviceSize                                dataSize,
 433     const uint32_t*                             pData)
 434 {
 435    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 436    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 437    struct anv_meta_saved_state saved_state;
 438
 439    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 440
 441    /* We can't quite grab a full block because the state stream needs a
 442     * little data at the top to build its linked list.
 443     */
 444    const uint32_t max_update_size =
 445       cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
 446
 447    assert(max_update_size < (1 << 14) * 4);
 448
 449    while (dataSize) {
 450       const uint32_t copy_size = MIN2(dataSize, max_update_size);
 451
 452       struct anv_state tmp_data =
 453          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
 454
 455       memcpy(tmp_data.map, pData, copy_size);
 456
 457       int bs;
 458       if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) {
 459          bs = 16;
 460       } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) {
 461          bs = 8;
 462       } else {
 463          assert((copy_size & 3) == 0 && (dstOffset & 3) == 0);
 464          bs = 4;
 465       }
 466
 467       do_buffer_copy(cmd_buffer,
 468                      &cmd_buffer->device->dynamic_state_block_pool.bo,
 469                      tmp_data.offset,
 470                      dst_buffer->bo, dst_buffer->offset + dstOffset,
 471                      copy_size / bs, 1, bs);
 472
 473       dataSize -= copy_size;
 474       dstOffset += copy_size;
 475       pData = (void *)pData + copy_size;
 476    }
 477
 478    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 479 }