src/amd/vulkan/radv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "radv_meta.h"
  25 #include "vk_format.h"
  26
  27 static VkExtent3D
  28 meta_image_block_size(const struct radv_image *image)
  29 {
  30         const struct vk_format_description *desc = vk_format_description(image->vk_format);
  31         return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
  32 }
  33
  34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  35  * elements rather than texels. One element equals one texel or one block
  36  * if Image is uncompressed or compressed, respectively.
  37  */
  38 static struct VkExtent3D
  39 meta_region_extent_el(const struct radv_image *image,
  40                       const VkImageType imageType,
  41                       const struct VkExtent3D *extent)
  42 {
  43         const VkExtent3D block = meta_image_block_size(image);
  44         return radv_sanitize_image_extent(imageType, (VkExtent3D) {
  45                         .width  = DIV_ROUND_UP(extent->width , block.width),
  46                                 .height = DIV_ROUND_UP(extent->height, block.height),
  47                                 .depth  = DIV_ROUND_UP(extent->depth , block.depth),
  48                                 });
  49 }
  50
  51 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  52  * elements rather than texels. One element equals one texel or one block
  53  * if Image is uncompressed or compressed, respectively.
  54  */
  55 static struct VkOffset3D
  56 meta_region_offset_el(const struct radv_image *image,
  57                       const struct VkOffset3D *offset)
  58 {
  59         const VkExtent3D block = meta_image_block_size(image);
  60         return radv_sanitize_image_offset(image->type, (VkOffset3D) {
  61                         .x = offset->x / block.width,
  62                                 .y = offset->y / block.height,
  63                                 .z = offset->z / block.depth,
  64                                 });
  65 }
  66
  67 static VkFormat
  68 vk_format_for_size(int bs)
  69 {
  70         switch (bs) {
  71         case 1: return VK_FORMAT_R8_UINT;
  72         case 2: return VK_FORMAT_R8G8_UINT;
  73         case 4: return VK_FORMAT_R8G8B8A8_UINT;
  74         case 8: return VK_FORMAT_R16G16B16A16_UINT;
  75         case 16: return VK_FORMAT_R32G32B32A32_UINT;
  76         default:
  77                 unreachable("Invalid format block size");
  78         }
  79 }
  80
  81 static struct radv_meta_blit2d_surf
  82 blit_surf_for_image_level_layer(struct radv_image *image,
  83                                 VkImageLayout layout,
  84                                 const VkImageSubresourceLayers *subres)
  85 {
  86         VkFormat format = image->vk_format;
  87         if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
  88                 format = vk_format_depth_only(format);
  89         else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
  90                 format = vk_format_stencil_only(format);
  91
  92         if (!radv_image_has_dcc(image) &&
  93             !(radv_image_is_tc_compat_htile(image)))
  94                 format = vk_format_for_size(vk_format_get_blocksize(format));
  95
  96         return (struct radv_meta_blit2d_surf) {
  97                 .format = format,
  98                 .bs = vk_format_get_blocksize(format),
  99                 .level = subres->mipLevel,
 100                 .layer = subres->baseArrayLayer,
 101                 .image = image,
 102                 .aspect_mask = subres->aspectMask,
 103                 .current_layout = layout,
 104         };
 105 }
 106
 107 static void
 108 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
 109                           struct radv_buffer* buffer,
 110                           struct radv_image* image,
 111                           VkImageLayout layout,
 112                           uint32_t regionCount,
 113                           const VkBufferImageCopy* pRegions)
 114 {
 115         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 116         struct radv_meta_saved_state saved_state;
 117
 118         /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 119          * VK_SAMPLE_COUNT_1_BIT."
 120          */
 121         assert(image->info.samples == 1);
 122
 123         radv_meta_save(&saved_state, cmd_buffer,
 124                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 125                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 126                        RADV_META_SAVE_CONSTANTS |
 127                        RADV_META_SAVE_DESCRIPTORS);
 128
 129         for (unsigned r = 0; r < regionCount; r++) {
 130
 131                 /**
 132                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 133                  *    extent is the size in texels of the source image to copy in width,
 134                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 135                  *    width and height. 3D images use x, y, z, width, height and depth.
 136                  *
 137                  *
 138                  * Also, convert the offsets and extent from units of texels to units of
 139                  * blocks - which is the highest resolution accessible in this command.
 140                  */
 141                 const VkOffset3D img_offset_el =
 142                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 143                 const VkExtent3D bufferExtent = {
 144                         .width  = pRegions[r].bufferRowLength ?
 145                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 146                         .height = pRegions[r].bufferImageHeight ?
 147                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 148                 };
 149                 const VkExtent3D buf_extent_el =
 150                         meta_region_extent_el(image, image->type, &bufferExtent);
 151
 152                 /* Start creating blit rect */
 153                 const VkExtent3D img_extent_el =
 154                         meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
 155                 struct radv_meta_blit2d_rect rect = {
 156                         .width = img_extent_el.width,
 157                         .height =  img_extent_el.height,
 158                 };
 159
 160                 /* Create blit surfaces */
 161                 struct radv_meta_blit2d_surf img_bsurf =
 162                         blit_surf_for_image_level_layer(image,
 163                                                         layout,
 164                                                         &pRegions[r].imageSubresource);
 165
 166                 struct radv_meta_blit2d_buffer buf_bsurf = {
 167                         .bs = img_bsurf.bs,
 168                         .format = img_bsurf.format,
 169                         .buffer = buffer,
 170                         .offset = pRegions[r].bufferOffset,
 171                         .pitch = buf_extent_el.width,
 172                 };
 173
 174                 if (image->type == VK_IMAGE_TYPE_3D)
 175                         img_bsurf.layer = img_offset_el.z;
 176                 /* Loop through each 3D or array slice */
 177                 unsigned num_slices_3d = img_extent_el.depth;
 178                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 179                 unsigned slice_3d = 0;
 180                 unsigned slice_array = 0;
 181                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 182
 183                         rect.dst_x = img_offset_el.x;
 184                         rect.dst_y = img_offset_el.y;
 185
 186
 187                         /* Perform Blit */
 188                         if (cs)
 189                                 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
 190                         else
 191                                 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
 192
 193                         /* Once we've done the blit, all of the actual information about
 194                          * the image is embedded in the command buffer so we can just
 195                          * increment the offset directly in the image effectively
 196                          * re-binding it to different backing memory.
 197                          */
 198                         buf_bsurf.offset += buf_extent_el.width *
 199                                             buf_extent_el.height * buf_bsurf.bs;
 200                         img_bsurf.layer++;
 201                         if (image->type == VK_IMAGE_TYPE_3D)
 202                                 slice_3d++;
 203                         else
 204                                 slice_array++;
 205                 }
 206         }
 207
 208         radv_meta_restore(&saved_state, cmd_buffer);
 209 }
 210
 211 void radv_CmdCopyBufferToImage(
 212         VkCommandBuffer                             commandBuffer,
 213         VkBuffer                                    srcBuffer,
 214         VkImage                                     destImage,
 215         VkImageLayout                               destImageLayout,
 216         uint32_t                                    regionCount,
 217         const VkBufferImageCopy*                    pRegions)
 218 {
 219         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 220         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 221         RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
 222
 223         meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, destImageLayout,
 224                                   regionCount, pRegions);
 225 }
 226
 227 static void
 228 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 229                           struct radv_buffer* buffer,
 230                           struct radv_image* image,
 231                           VkImageLayout layout,
 232                           uint32_t regionCount,
 233                           const VkBufferImageCopy* pRegions)
 234 {
 235         struct radv_meta_saved_state saved_state;
 236
 237         radv_meta_save(&saved_state, cmd_buffer,
 238                        RADV_META_SAVE_COMPUTE_PIPELINE |
 239                        RADV_META_SAVE_CONSTANTS |
 240                        RADV_META_SAVE_DESCRIPTORS);
 241
 242         for (unsigned r = 0; r < regionCount; r++) {
 243
 244                 /**
 245                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 246                  *    extent is the size in texels of the source image to copy in width,
 247                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 248                  *    width and height. 3D images use x, y, z, width, height and depth.
 249                  *
 250                  *
 251                  * Also, convert the offsets and extent from units of texels to units of
 252                  * blocks - which is the highest resolution accessible in this command.
 253                  */
 254                 const VkOffset3D img_offset_el =
 255                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 256                 const VkExtent3D bufferExtent = {
 257                         .width  = pRegions[r].bufferRowLength ?
 258                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 259                         .height = pRegions[r].bufferImageHeight ?
 260                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 261                 };
 262                 const VkExtent3D buf_extent_el =
 263                         meta_region_extent_el(image, image->type, &bufferExtent);
 264
 265                 /* Start creating blit rect */
 266                 const VkExtent3D img_extent_el =
 267                         meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
 268                 struct radv_meta_blit2d_rect rect = {
 269                         .width = img_extent_el.width,
 270                         .height =  img_extent_el.height,
 271                 };
 272
 273                 /* Create blit surfaces */
 274                 struct radv_meta_blit2d_surf img_info =
 275                         blit_surf_for_image_level_layer(image,
 276                                                         layout,
 277                                                         &pRegions[r].imageSubresource);
 278
 279                 struct radv_meta_blit2d_buffer buf_info = {
 280                         .bs = img_info.bs,
 281                         .format = img_info.format,
 282                         .buffer = buffer,
 283                         .offset = pRegions[r].bufferOffset,
 284                         .pitch = buf_extent_el.width,
 285                 };
 286
 287                 if (image->type == VK_IMAGE_TYPE_3D)
 288                         img_info.layer = img_offset_el.z;
 289                 /* Loop through each 3D or array slice */
 290                 unsigned num_slices_3d = img_extent_el.depth;
 291                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 292                 unsigned slice_3d = 0;
 293                 unsigned slice_array = 0;
 294                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 295
 296                         rect.src_x = img_offset_el.x;
 297                         rect.src_y = img_offset_el.y;
 298
 299
 300                         /* Perform Blit */
 301                         radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
 302
 303                         buf_info.offset += buf_extent_el.width *
 304                                             buf_extent_el.height * buf_info.bs;
 305                         img_info.layer++;
 306                         if (image->type == VK_IMAGE_TYPE_3D)
 307                                 slice_3d++;
 308                         else
 309                                 slice_array++;
 310                 }
 311         }
 312
 313         radv_meta_restore(&saved_state, cmd_buffer);
 314 }
 315
 316 void radv_CmdCopyImageToBuffer(
 317         VkCommandBuffer                             commandBuffer,
 318         VkImage                                     srcImage,
 319         VkImageLayout                               srcImageLayout,
 320         VkBuffer                                    destBuffer,
 321         uint32_t                                    regionCount,
 322         const VkBufferImageCopy*                    pRegions)
 323 {
 324         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 325         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 326         RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
 327
 328         meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
 329                                   srcImageLayout,
 330                                   regionCount, pRegions);
 331 }
 332
 333 static void
 334 meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
 335                 struct radv_image *src_image,
 336                 VkImageLayout src_image_layout,
 337                 struct radv_image *dest_image,
 338                 VkImageLayout dest_image_layout,
 339                 uint32_t regionCount,
 340                 const VkImageCopy *pRegions)
 341 {
 342         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 343         struct radv_meta_saved_state saved_state;
 344
 345         /* From the Vulkan 1.0 spec:
 346          *
 347          *    vkCmdCopyImage can be used to copy image data between multisample
 348          *    images, but both images must have the same number of samples.
 349          */
 350         assert(src_image->info.samples == dest_image->info.samples);
 351
 352         radv_meta_save(&saved_state, cmd_buffer,
 353                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 354                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 355                        RADV_META_SAVE_CONSTANTS |
 356                        RADV_META_SAVE_DESCRIPTORS);
 357
 358         for (unsigned r = 0; r < regionCount; r++) {
 359                 assert(pRegions[r].srcSubresource.aspectMask ==
 360                        pRegions[r].dstSubresource.aspectMask);
 361
 362                 /* Create blit surfaces */
 363                 struct radv_meta_blit2d_surf b_src =
 364                         blit_surf_for_image_level_layer(src_image,
 365                                                         src_image_layout,
 366                                                         &pRegions[r].srcSubresource);
 367
 368                 struct radv_meta_blit2d_surf b_dst =
 369                         blit_surf_for_image_level_layer(dest_image,
 370                                                         dest_image_layout,
 371                                                         &pRegions[r].dstSubresource);
 372
 373                 uint32_t dst_queue_mask = radv_image_queue_family_mask(dest_image,
 374                                                                        cmd_buffer->queue_family_index,
 375                                                                        cmd_buffer->queue_family_index);
 376                 bool dst_compressed = radv_layout_dcc_compressed(dest_image, dest_image_layout, dst_queue_mask);
 377                 uint32_t src_queue_mask = radv_image_queue_family_mask(src_image,
 378                                                                        cmd_buffer->queue_family_index,
 379                                                                        cmd_buffer->queue_family_index);
 380                 bool src_compressed = radv_layout_dcc_compressed(src_image, src_image_layout, src_queue_mask);
 381
 382                 if (!src_compressed || radv_dcc_formats_compatible(b_src.format, b_dst.format)) {
 383                         b_src.format = b_dst.format;
 384                 } else if (!dst_compressed) {
 385                         b_dst.format = b_src.format;
 386                 } else {
 387                         radv_decompress_dcc(cmd_buffer, dest_image, &(VkImageSubresourceRange) {
 388                                                 .aspectMask = pRegions[r].dstSubresource.aspectMask,
 389                                                 .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
 390                                                 .levelCount = 1,
 391                                                 .baseArrayLayer = pRegions[r].dstSubresource.baseArrayLayer,
 392                                                 .layerCount = pRegions[r].dstSubresource.layerCount,
 393                                             });
 394                         b_dst.format = b_src.format;
 395                         b_dst.current_layout = VK_IMAGE_LAYOUT_GENERAL;
 396                 }
 397
 398
 399                 /**
 400                  * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 401                  *    imageExtent is the size in texels of the image to copy in width, height
 402                  *    and depth. 1D images use only x and width. 2D images use x, y, width
 403                  *    and height. 3D images use x, y, z, width, height and depth.
 404                  *
 405                  * Also, convert the offsets and extent from units of texels to units of
 406                  * blocks - which is the highest resolution accessible in this command.
 407                  */
 408                 const VkOffset3D dst_offset_el =
 409                         meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 410                 const VkOffset3D src_offset_el =
 411                         meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 412
 413                 /*
 414                  * From Vulkan 1.0.68, "Copying Data Between Images":
 415                  *    "When copying between compressed and uncompressed formats
 416                  *     the extent members represent the texel dimensions of the
 417                  *     source image and not the destination."
 418                  * However, we must use the destination image type to avoid
 419                  * clamping depth when copying multiple layers of a 2D image to
 420                  * a 3D image.
 421                  */
 422                 const VkExtent3D img_extent_el =
 423                         meta_region_extent_el(src_image, dest_image->type, &pRegions[r].extent);
 424
 425                 /* Start creating blit rect */
 426                 struct radv_meta_blit2d_rect rect = {
 427                         .width = img_extent_el.width,
 428                         .height = img_extent_el.height,
 429                 };
 430
 431                 if (src_image->type == VK_IMAGE_TYPE_3D)
 432                         b_src.layer = src_offset_el.z;
 433
 434                 if (dest_image->type == VK_IMAGE_TYPE_3D)
 435                         b_dst.layer = dst_offset_el.z;
 436
 437                 /* Loop through each 3D or array slice */
 438                 unsigned num_slices_3d = img_extent_el.depth;
 439                 unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 440                 unsigned slice_3d = 0;
 441                 unsigned slice_array = 0;
 442                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 443
 444                         /* Finish creating blit rect */
 445                         rect.dst_x = dst_offset_el.x;
 446                         rect.dst_y = dst_offset_el.y;
 447                         rect.src_x = src_offset_el.x;
 448                         rect.src_y = src_offset_el.y;
 449
 450                         /* Perform Blit */
 451                         if (cs)
 452                                 radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
 453                         else
 454                                 radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
 455
 456                         b_src.layer++;
 457                         b_dst.layer++;
 458                         if (dest_image->type == VK_IMAGE_TYPE_3D)
 459                                 slice_3d++;
 460                         else
 461                                 slice_array++;
 462                 }
 463         }
 464
 465         radv_meta_restore(&saved_state, cmd_buffer);
 466 }
 467
 468 void radv_CmdCopyImage(
 469         VkCommandBuffer                             commandBuffer,
 470         VkImage                                     srcImage,
 471         VkImageLayout                               srcImageLayout,
 472         VkImage                                     destImage,
 473         VkImageLayout                               destImageLayout,
 474         uint32_t                                    regionCount,
 475         const VkImageCopy*                          pRegions)
 476 {
 477         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 478         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 479         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 480
 481         meta_copy_image(cmd_buffer,
 482                         src_image, srcImageLayout,
 483                         dest_image, destImageLayout,
 484                         regionCount, pRegions);
 485 }
 486
 487 void radv_blit_to_prime_linear(struct radv_cmd_buffer *cmd_buffer,
 488                                struct radv_image *image,
 489                                struct radv_image *linear_image)
 490 {
 491         struct VkImageCopy image_copy = { 0 };
 492
 493         image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
 494         image_copy.srcSubresource.layerCount = 1;
 495
 496         image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
 497         image_copy.dstSubresource.layerCount = 1;
 498
 499         image_copy.extent.width = image->info.width;
 500         image_copy.extent.height = image->info.height;
 501         image_copy.extent.depth = 1;
 502
 503         meta_copy_image(cmd_buffer, image, VK_IMAGE_LAYOUT_GENERAL, linear_image,
 504                         VK_IMAGE_LAYOUT_GENERAL,
 505                         1, &image_copy);
 506 }