src/intel/vulkan/anv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_meta.h"
  25
  26 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  27  * elements rather than texels. One element equals one texel or one block
  28  * if Image is uncompressed or compressed, respectively.
  29  */
  30 static struct VkExtent3D
  31 meta_region_extent_el(const struct anv_image *image,
  32                       const struct VkExtent3D *extent)
  33 {
  34    const struct isl_format_layout *isl_layout =
  35       anv_format_for_vk_format(image->vk_format)->isl_layout;
  36    return anv_sanitize_image_extent(image->type, (VkExtent3D) {
  37       .width  = DIV_ROUND_UP(extent->width , isl_layout->bw),
  38       .height = DIV_ROUND_UP(extent->height, isl_layout->bh),
  39       .depth  = DIV_ROUND_UP(extent->depth , isl_layout->bd),
  40    });
  41 }
  42
  43 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  44  * elements rather than texels. One element equals one texel or one block
  45  * if Image is uncompressed or compressed, respectively.
  46  */
  47 static struct VkOffset3D
  48 meta_region_offset_el(const struct anv_image *image,
  49                       const struct VkOffset3D *offset)
  50 {
  51    const struct isl_format_layout *isl_layout = image->format->isl_layout;
  52    return anv_sanitize_image_offset(image->type, (VkOffset3D) {
  53       .x = offset->x / isl_layout->bw,
  54       .y = offset->y / isl_layout->bh,
  55       .z = offset->z / isl_layout->bd,
  56    });
  57 }
  58
  59 static struct anv_meta_blit2d_surf
  60 blit_surf_for_image(const struct anv_image* image,
  61                     const struct isl_surf *img_isl_surf)
  62 {
  63    return (struct anv_meta_blit2d_surf) {
  64       .bo = image->bo,
  65       .tiling = img_isl_surf->tiling,
  66       .base_offset = image->offset,
  67       .bs = isl_format_get_layout(img_isl_surf->format)->bs,
  68       .pitch = isl_surf_get_row_pitch(img_isl_surf),
  69    };
  70 }
  71
  72 static void
  73 do_buffer_copy(struct anv_cmd_buffer *cmd_buffer,
  74                struct anv_bo *src, uint64_t src_offset,
  75                struct anv_bo *dest, uint64_t dest_offset,
  76                int width, int height, int bs)
  77 {
  78    struct anv_meta_blit2d_surf b_src = {
  79       .bo = src,
  80       .tiling = ISL_TILING_LINEAR,
  81       .base_offset = src_offset,
  82       .bs = bs,
  83       .pitch = width * bs,
  84    };
  85    struct anv_meta_blit2d_surf b_dst = {
  86       .bo = dest,
  87       .tiling = ISL_TILING_LINEAR,
  88       .base_offset = dest_offset,
  89       .bs = bs,
  90       .pitch = width * bs,
  91    };
  92    struct anv_meta_blit2d_rect rect = {
  93       .width = width,
  94       .height = height,
  95    };
  96    anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
  97 }
  98
  99 static void
 100 meta_copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
 101                           struct anv_buffer* buffer,
 102                           struct anv_image* image,
 103                           uint32_t regionCount,
 104                           const VkBufferImageCopy* pRegions,
 105                           bool forward)
 106 {
 107    struct anv_meta_saved_state saved_state;
 108
 109    /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 110     * VK_SAMPLE_COUNT_1_BIT."
 111     */
 112    assert(image->samples == 1);
 113
 114    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 115
 116    for (unsigned r = 0; r < regionCount; r++) {
 117
 118       /**
 119        * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 120        *    extent is the size in texels of the source image to copy in width,
 121        *    height and depth. 1D images use only x and width. 2D images use x, y,
 122        *    width and height. 3D images use x, y, z, width, height and depth.
 123        *
 124        *
 125        * Also, convert the offsets and extent from units of texels to units of
 126        * blocks - which is the highest resolution accessible in this command.
 127        */
 128       const VkOffset3D img_offset_el =
 129          meta_region_offset_el(image, &pRegions[r].imageOffset);
 130       const VkExtent3D bufferExtent = {
 131          .width  = pRegions[r].bufferRowLength ?
 132                    pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 133          .height = pRegions[r].bufferImageHeight ?
 134                    pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 135       };
 136       const VkExtent3D buf_extent_el =
 137          meta_region_extent_el(image, &bufferExtent);
 138
 139       /* Start creating blit rect */
 140       const VkExtent3D img_extent_el =
 141          meta_region_extent_el(image, &pRegions[r].imageExtent);
 142       struct anv_meta_blit2d_rect rect = {
 143          .width = img_extent_el.width,
 144          .height =  img_extent_el.height,
 145       };
 146
 147       /* Create blit surfaces */
 148       VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 149       const struct isl_surf *img_isl_surf =
 150          &anv_image_get_surface_for_aspect_mask(image, aspect)->isl;
 151       struct anv_meta_blit2d_surf img_bsurf =
 152          blit_surf_for_image(image, img_isl_surf);
 153       struct anv_meta_blit2d_surf buf_bsurf = {
 154          .bo = buffer->bo,
 155          .tiling = ISL_TILING_LINEAR,
 156          .base_offset = buffer->offset + pRegions[r].bufferOffset,
 157          .bs = forward ? image->format->isl_layout->bs : img_bsurf.bs,
 158          .pitch = buf_extent_el.width * buf_bsurf.bs,
 159       };
 160
 161       /* Set direction-dependent variables */
 162       struct anv_meta_blit2d_surf *dst_bsurf = forward ? &img_bsurf : &buf_bsurf;
 163       struct anv_meta_blit2d_surf *src_bsurf = forward ? &buf_bsurf : &img_bsurf;
 164       uint32_t *x_offset = forward ? &rect.dst_x : &rect.src_x;
 165       uint32_t *y_offset = forward ? &rect.dst_y : &rect.src_y;
 166
 167       /* Loop through each 3D or array slice */
 168       unsigned num_slices_3d = img_extent_el.depth;
 169       unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 170       unsigned slice_3d = 0;
 171       unsigned slice_array = 0;
 172       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 173
 174          /* Finish creating blit rect */
 175          isl_surf_get_image_offset_el(img_isl_surf,
 176                                     pRegions[r].imageSubresource.mipLevel,
 177                                     pRegions[r].imageSubresource.baseArrayLayer
 178                                        + slice_array,
 179                                     img_offset_el.z + slice_3d,
 180                                     x_offset,
 181                                     y_offset);
 182          *x_offset += img_offset_el.x;
 183          *y_offset += img_offset_el.y;
 184
 185          /* Perform Blit */
 186          anv_meta_blit2d(cmd_buffer, src_bsurf, dst_bsurf, 1, &rect);
 187
 188          /* Once we've done the blit, all of the actual information about
 189           * the image is embedded in the command buffer so we can just
 190           * increment the offset directly in the image effectively
 191           * re-binding it to different backing memory.
 192           */
 193          buf_bsurf.base_offset += buf_extent_el.width *
 194                                   buf_extent_el.height * buf_bsurf.bs;
 195
 196          if (image->type == VK_IMAGE_TYPE_3D)
 197             slice_3d++;
 198          else
 199             slice_array++;
 200       }
 201    }
 202    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 203 }
 204
 205 void anv_CmdCopyBufferToImage(
 206     VkCommandBuffer                             commandBuffer,
 207     VkBuffer                                    srcBuffer,
 208     VkImage                                     destImage,
 209     VkImageLayout                               destImageLayout,
 210     uint32_t                                    regionCount,
 211     const VkBufferImageCopy*                    pRegions)
 212 {
 213    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 214    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 215    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 216
 217    meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image,
 218                              regionCount, pRegions, true);
 219 }
 220
 221 void anv_CmdCopyImageToBuffer(
 222     VkCommandBuffer                             commandBuffer,
 223     VkImage                                     srcImage,
 224     VkImageLayout                               srcImageLayout,
 225     VkBuffer                                    destBuffer,
 226     uint32_t                                    regionCount,
 227     const VkBufferImageCopy*                    pRegions)
 228 {
 229    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 230    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 231    ANV_FROM_HANDLE(anv_buffer, dst_buffer, destBuffer);
 232
 233    meta_copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
 234                              regionCount, pRegions, false);
 235 }
 236
 237 void anv_CmdCopyImage(
 238     VkCommandBuffer                             commandBuffer,
 239     VkImage                                     srcImage,
 240     VkImageLayout                               srcImageLayout,
 241     VkImage                                     destImage,
 242     VkImageLayout                               destImageLayout,
 243     uint32_t                                    regionCount,
 244     const VkImageCopy*                          pRegions)
 245 {
 246    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 247    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 248    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 249    struct anv_meta_saved_state saved_state;
 250
 251    /* From the Vulkan 1.0 spec:
 252     *
 253     *    vkCmdCopyImage can be used to copy image data between multisample
 254     *    images, but both images must have the same number of samples.
 255     */
 256    assert(src_image->samples == dest_image->samples);
 257
 258    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 259
 260    for (unsigned r = 0; r < regionCount; r++) {
 261       assert(pRegions[r].srcSubresource.aspectMask ==
 262              pRegions[r].dstSubresource.aspectMask);
 263
 264       VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask;
 265
 266       /* Create blit surfaces */
 267       struct isl_surf *src_isl_surf =
 268          &anv_image_get_surface_for_aspect_mask(src_image, aspect)->isl;
 269       struct isl_surf *dst_isl_surf =
 270          &anv_image_get_surface_for_aspect_mask(dest_image, aspect)->isl;
 271       struct anv_meta_blit2d_surf b_src =
 272          blit_surf_for_image(src_image, src_isl_surf);
 273       struct anv_meta_blit2d_surf b_dst =
 274          blit_surf_for_image(dest_image, dst_isl_surf);
 275
 276       /**
 277        * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 278        *    imageExtent is the size in texels of the image to copy in width, height
 279        *    and depth. 1D images use only x and width. 2D images use x, y, width
 280        *    and height. 3D images use x, y, z, width, height and depth.
 281        *
 282        * Also, convert the offsets and extent from units of texels to units of
 283        * blocks - which is the highest resolution accessible in this command.
 284        */
 285       const VkOffset3D dst_offset_el =
 286          meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 287       const VkOffset3D src_offset_el =
 288          meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 289       const VkExtent3D img_extent_el =
 290          meta_region_extent_el(src_image, &pRegions[r].extent);
 291
 292       /* Start creating blit rect */
 293       struct anv_meta_blit2d_rect rect = {
 294          .width = img_extent_el.width,
 295          .height = img_extent_el.height,
 296       };
 297
 298       /* Loop through each 3D or array slice */
 299       unsigned num_slices_3d = img_extent_el.depth;
 300       unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 301       unsigned slice_3d = 0;
 302       unsigned slice_array = 0;
 303       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 304
 305          /* Finish creating blit rect */
 306          isl_surf_get_image_offset_el(dst_isl_surf,
 307                                     pRegions[r].dstSubresource.mipLevel,
 308                                     pRegions[r].dstSubresource.baseArrayLayer
 309                                        + slice_array,
 310                                     dst_offset_el.z + slice_3d,
 311                                     &rect.dst_x,
 312                                     &rect.dst_y);
 313          isl_surf_get_image_offset_el(src_isl_surf,
 314                                     pRegions[r].srcSubresource.mipLevel,
 315                                     pRegions[r].srcSubresource.baseArrayLayer
 316                                        + slice_array,
 317                                     src_offset_el.z + slice_3d,
 318                                     &rect.src_x,
 319                                     &rect.src_y);
 320          rect.dst_x += dst_offset_el.x;
 321          rect.dst_y += dst_offset_el.y;
 322          rect.src_x += src_offset_el.x;
 323          rect.src_y += src_offset_el.y;
 324
 325          /* Perform Blit */
 326          anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect);
 327
 328          if (dest_image->type == VK_IMAGE_TYPE_3D)
 329             slice_3d++;
 330          else
 331             slice_array++;
 332       }
 333    }
 334
 335    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 336 }
 337
 338 void anv_CmdCopyBuffer(
 339     VkCommandBuffer                             commandBuffer,
 340     VkBuffer                                    srcBuffer,
 341     VkBuffer                                    destBuffer,
 342     uint32_t                                    regionCount,
 343     const VkBufferCopy*                         pRegions)
 344 {
 345    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 346    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 347    ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer);
 348
 349    struct anv_meta_saved_state saved_state;
 350
 351    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 352
 353    for (unsigned r = 0; r < regionCount; r++) {
 354       uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset;
 355       uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset;
 356       uint64_t copy_size = pRegions[r].size;
 357
 358       /* First, we compute the biggest format that can be used with the
 359        * given offsets and size.
 360        */
 361       int bs = 16;
 362
 363       int fs = ffs(src_offset) - 1;
 364       if (fs != -1)
 365          bs = MIN2(bs, 1 << fs);
 366       assert(src_offset % bs == 0);
 367
 368       fs = ffs(dest_offset) - 1;
 369       if (fs != -1)
 370          bs = MIN2(bs, 1 << fs);
 371       assert(dest_offset % bs == 0);
 372
 373       fs = ffs(pRegions[r].size) - 1;
 374       if (fs != -1)
 375          bs = MIN2(bs, 1 << fs);
 376       assert(pRegions[r].size % bs == 0);
 377
 378       /* This is maximum possible width/height our HW can handle */
 379       uint64_t max_surface_dim = 1 << 14;
 380
 381       /* First, we make a bunch of max-sized copies */
 382       uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
 383       while (copy_size >= max_copy_size) {
 384          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 385                         dest_buffer->bo, dest_offset,
 386                         max_surface_dim, max_surface_dim, bs);
 387          copy_size -= max_copy_size;
 388          src_offset += max_copy_size;
 389          dest_offset += max_copy_size;
 390       }
 391
 392       uint64_t height = copy_size / (max_surface_dim * bs);
 393       assert(height < max_surface_dim);
 394       if (height != 0) {
 395          uint64_t rect_copy_size = height * max_surface_dim * bs;
 396          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 397                         dest_buffer->bo, dest_offset,
 398                         max_surface_dim, height, bs);
 399          copy_size -= rect_copy_size;
 400          src_offset += rect_copy_size;
 401          dest_offset += rect_copy_size;
 402       }
 403
 404       if (copy_size != 0) {
 405          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 406                         dest_buffer->bo, dest_offset,
 407                         copy_size / bs, 1, bs);
 408       }
 409    }
 410
 411    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 412 }
 413
 414 void anv_CmdUpdateBuffer(
 415     VkCommandBuffer                             commandBuffer,
 416     VkBuffer                                    dstBuffer,
 417     VkDeviceSize                                dstOffset,
 418     VkDeviceSize                                dataSize,
 419     const uint32_t*                             pData)
 420 {
 421    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 422    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 423    struct anv_meta_saved_state saved_state;
 424
 425    anv_meta_begin_blit2d(cmd_buffer, &saved_state);
 426
 427    /* We can't quite grab a full block because the state stream needs a
 428     * little data at the top to build its linked list.
 429     */
 430    const uint32_t max_update_size =
 431       cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
 432
 433    assert(max_update_size < (1 << 14) * 4);
 434
 435    while (dataSize) {
 436       const uint32_t copy_size = MIN2(dataSize, max_update_size);
 437
 438       struct anv_state tmp_data =
 439          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
 440
 441       memcpy(tmp_data.map, pData, copy_size);
 442
 443       int bs;
 444       if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) {
 445          bs = 16;
 446       } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) {
 447          bs = 8;
 448       } else {
 449          assert((copy_size & 3) == 0 && (dstOffset & 3) == 0);
 450          bs = 4;
 451       }
 452
 453       do_buffer_copy(cmd_buffer,
 454                      &cmd_buffer->device->dynamic_state_block_pool.bo,
 455                      tmp_data.offset,
 456                      dst_buffer->bo, dst_buffer->offset + dstOffset,
 457                      copy_size / bs, 1, bs);
 458
 459       dataSize -= copy_size;
 460       dstOffset += copy_size;
 461       pData = (void *)pData + copy_size;
 462    }
 463
 464    anv_meta_end_blit2d(cmd_buffer, &saved_state);
 465 }