src/amd/vulkan/radv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "radv_meta.h"
  25 #include "vk_format.h"
  26
  27 static VkExtent3D
  28 meta_image_block_size(const struct radv_image *image)
  29 {
  30         const struct vk_format_description *desc = vk_format_description(image->vk_format);
  31         return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
  32 }
  33
  34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  35  * elements rather than texels. One element equals one texel or one block
  36  * if Image is uncompressed or compressed, respectively.
  37  */
  38 static struct VkExtent3D
  39 meta_region_extent_el(const struct radv_image *image,
  40                       const VkImageType imageType,
  41                       const struct VkExtent3D *extent)
  42 {
  43         const VkExtent3D block = meta_image_block_size(image);
  44         return radv_sanitize_image_extent(imageType, (VkExtent3D) {
  45                         .width  = DIV_ROUND_UP(extent->width , block.width),
  46                                 .height = DIV_ROUND_UP(extent->height, block.height),
  47                                 .depth  = DIV_ROUND_UP(extent->depth , block.depth),
  48                                 });
  49 }
  50
  51 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  52  * elements rather than texels. One element equals one texel or one block
  53  * if Image is uncompressed or compressed, respectively.
  54  */
  55 static struct VkOffset3D
  56 meta_region_offset_el(const struct radv_image *image,
  57                       const struct VkOffset3D *offset)
  58 {
  59         const VkExtent3D block = meta_image_block_size(image);
  60         return radv_sanitize_image_offset(image->type, (VkOffset3D) {
  61                         .x = offset->x / block.width,
  62                                 .y = offset->y / block.height,
  63                                 .z = offset->z / block.depth,
  64                                 });
  65 }
  66
  67 static VkFormat
  68 vk_format_for_size(int bs)
  69 {
  70         switch (bs) {
  71         case 1: return VK_FORMAT_R8_UINT;
  72         case 2: return VK_FORMAT_R8G8_UINT;
  73         case 4: return VK_FORMAT_R8G8B8A8_UINT;
  74         case 8: return VK_FORMAT_R16G16B16A16_UINT;
  75         case 12: return VK_FORMAT_R32G32B32_UINT;
  76         case 16: return VK_FORMAT_R32G32B32A32_UINT;
  77         default:
  78                 unreachable("Invalid format block size");
  79         }
  80 }
  81
  82 static struct radv_meta_blit2d_surf
  83 blit_surf_for_image_level_layer(struct radv_image *image,
  84                                 VkImageLayout layout,
  85                                 const VkImageSubresourceLayers *subres)
  86 {
  87         VkFormat format = image->vk_format;
  88         if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
  89                 format = vk_format_depth_only(format);
  90         else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
  91                 format = vk_format_stencil_only(format);
  92
  93         if (!radv_image_has_dcc(image) &&
  94             !(radv_image_is_tc_compat_htile(image)))
  95                 format = vk_format_for_size(vk_format_get_blocksize(format));
  96
  97         format = vk_format_no_srgb(format);
  98
  99         return (struct radv_meta_blit2d_surf) {
 100                 .format = format,
 101                 .bs = vk_format_get_blocksize(format),
 102                 .level = subres->mipLevel,
 103                 .layer = subres->baseArrayLayer,
 104                 .image = image,
 105                 .aspect_mask = subres->aspectMask,
 106                 .current_layout = layout,
 107         };
 108 }
 109
 110 static void
 111 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
 112                           struct radv_buffer* buffer,
 113                           struct radv_image* image,
 114                           VkImageLayout layout,
 115                           uint32_t regionCount,
 116                           const VkBufferImageCopy* pRegions)
 117 {
 118         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 119         struct radv_meta_saved_state saved_state;
 120         bool old_predicating;
 121
 122         /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 123          * VK_SAMPLE_COUNT_1_BIT."
 124          */
 125         assert(image->info.samples == 1);
 126
 127         radv_meta_save(&saved_state, cmd_buffer,
 128                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 129                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 130                        RADV_META_SAVE_CONSTANTS |
 131                        RADV_META_SAVE_DESCRIPTORS);
 132
 133         /* VK_EXT_conditional_rendering says that copy commands should not be
 134          * affected by conditional rendering.
 135          */
 136         old_predicating = cmd_buffer->state.predicating;
 137         cmd_buffer->state.predicating = false;
 138
 139         for (unsigned r = 0; r < regionCount; r++) {
 140
 141                 /**
 142                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 143                  *    extent is the size in texels of the source image to copy in width,
 144                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 145                  *    width and height. 3D images use x, y, z, width, height and depth.
 146                  *
 147                  *
 148                  * Also, convert the offsets and extent from units of texels to units of
 149                  * blocks - which is the highest resolution accessible in this command.
 150                  */
 151                 const VkOffset3D img_offset_el =
 152                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 153                 const VkExtent3D bufferExtent = {
 154                         .width  = pRegions[r].bufferRowLength ?
 155                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 156                         .height = pRegions[r].bufferImageHeight ?
 157                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 158                 };
 159                 const VkExtent3D buf_extent_el =
 160                         meta_region_extent_el(image, image->type, &bufferExtent);
 161
 162                 /* Start creating blit rect */
 163                 const VkExtent3D img_extent_el =
 164                         meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
 165                 struct radv_meta_blit2d_rect rect = {
 166                         .width = img_extent_el.width,
 167                         .height =  img_extent_el.height,
 168                 };
 169
 170                 /* Create blit surfaces */
 171                 struct radv_meta_blit2d_surf img_bsurf =
 172                         blit_surf_for_image_level_layer(image,
 173                                                         layout,
 174                                                         &pRegions[r].imageSubresource);
 175
 176                 struct radv_meta_blit2d_buffer buf_bsurf = {
 177                         .bs = img_bsurf.bs,
 178                         .format = img_bsurf.format,
 179                         .buffer = buffer,
 180                         .offset = pRegions[r].bufferOffset,
 181                         .pitch = buf_extent_el.width,
 182                 };
 183
 184                 if (image->type == VK_IMAGE_TYPE_3D)
 185                         img_bsurf.layer = img_offset_el.z;
 186                 /* Loop through each 3D or array slice */
 187                 unsigned num_slices_3d = img_extent_el.depth;
 188                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 189                 unsigned slice_3d = 0;
 190                 unsigned slice_array = 0;
 191                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 192
 193                         rect.dst_x = img_offset_el.x;
 194                         rect.dst_y = img_offset_el.y;
 195
 196
 197                         /* Perform Blit */
 198                         if (cs ||
 199                             (img_bsurf.image->vk_format == VK_FORMAT_R32G32B32_UINT ||
 200                              img_bsurf.image->vk_format == VK_FORMAT_R32G32B32_SINT ||
 201                              img_bsurf.image->vk_format == VK_FORMAT_R32G32B32_SFLOAT)) {
 202                                 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
 203                         } else {
 204                                 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
 205                         }
 206
 207                         /* Once we've done the blit, all of the actual information about
 208                          * the image is embedded in the command buffer so we can just
 209                          * increment the offset directly in the image effectively
 210                          * re-binding it to different backing memory.
 211                          */
 212                         buf_bsurf.offset += buf_extent_el.width *
 213                                             buf_extent_el.height * buf_bsurf.bs;
 214                         img_bsurf.layer++;
 215                         if (image->type == VK_IMAGE_TYPE_3D)
 216                                 slice_3d++;
 217                         else
 218                                 slice_array++;
 219                 }
 220         }
 221
 222         /* Restore conditional rendering. */
 223         cmd_buffer->state.predicating = old_predicating;
 224
 225         radv_meta_restore(&saved_state, cmd_buffer);
 226 }
 227
 228 void radv_CmdCopyBufferToImage(
 229         VkCommandBuffer                             commandBuffer,
 230         VkBuffer                                    srcBuffer,
 231         VkImage                                     destImage,
 232         VkImageLayout                               destImageLayout,
 233         uint32_t                                    regionCount,
 234         const VkBufferImageCopy*                    pRegions)
 235 {
 236         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 237         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 238         RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
 239
 240         meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, destImageLayout,
 241                                   regionCount, pRegions);
 242 }
 243
 244 static void
 245 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 246                           struct radv_buffer* buffer,
 247                           struct radv_image* image,
 248                           VkImageLayout layout,
 249                           uint32_t regionCount,
 250                           const VkBufferImageCopy* pRegions)
 251 {
 252         struct radv_meta_saved_state saved_state;
 253         bool old_predicating;
 254
 255         radv_meta_save(&saved_state, cmd_buffer,
 256                        RADV_META_SAVE_COMPUTE_PIPELINE |
 257                        RADV_META_SAVE_CONSTANTS |
 258                        RADV_META_SAVE_DESCRIPTORS);
 259
 260         /* VK_EXT_conditional_rendering says that copy commands should not be
 261          * affected by conditional rendering.
 262          */
 263         old_predicating = cmd_buffer->state.predicating;
 264         cmd_buffer->state.predicating = false;
 265
 266         for (unsigned r = 0; r < regionCount; r++) {
 267
 268                 /**
 269                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 270                  *    extent is the size in texels of the source image to copy in width,
 271                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 272                  *    width and height. 3D images use x, y, z, width, height and depth.
 273                  *
 274                  *
 275                  * Also, convert the offsets and extent from units of texels to units of
 276                  * blocks - which is the highest resolution accessible in this command.
 277                  */
 278                 const VkOffset3D img_offset_el =
 279                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 280                 const VkExtent3D bufferExtent = {
 281                         .width  = pRegions[r].bufferRowLength ?
 282                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 283                         .height = pRegions[r].bufferImageHeight ?
 284                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 285                 };
 286                 const VkExtent3D buf_extent_el =
 287                         meta_region_extent_el(image, image->type, &bufferExtent);
 288
 289                 /* Start creating blit rect */
 290                 const VkExtent3D img_extent_el =
 291                         meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
 292                 struct radv_meta_blit2d_rect rect = {
 293                         .width = img_extent_el.width,
 294                         .height =  img_extent_el.height,
 295                 };
 296
 297                 /* Create blit surfaces */
 298                 struct radv_meta_blit2d_surf img_info =
 299                         blit_surf_for_image_level_layer(image,
 300                                                         layout,
 301                                                         &pRegions[r].imageSubresource);
 302
 303                 struct radv_meta_blit2d_buffer buf_info = {
 304                         .bs = img_info.bs,
 305                         .format = img_info.format,
 306                         .buffer = buffer,
 307                         .offset = pRegions[r].bufferOffset,
 308                         .pitch = buf_extent_el.width,
 309                 };
 310
 311                 if (image->type == VK_IMAGE_TYPE_3D)
 312                         img_info.layer = img_offset_el.z;
 313                 /* Loop through each 3D or array slice */
 314                 unsigned num_slices_3d = img_extent_el.depth;
 315                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 316                 unsigned slice_3d = 0;
 317                 unsigned slice_array = 0;
 318                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 319
 320                         rect.src_x = img_offset_el.x;
 321                         rect.src_y = img_offset_el.y;
 322
 323
 324                         /* Perform Blit */
 325                         radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
 326
 327                         buf_info.offset += buf_extent_el.width *
 328                                             buf_extent_el.height * buf_info.bs;
 329                         img_info.layer++;
 330                         if (image->type == VK_IMAGE_TYPE_3D)
 331                                 slice_3d++;
 332                         else
 333                                 slice_array++;
 334                 }
 335         }
 336
 337         /* Restore conditional rendering. */
 338         cmd_buffer->state.predicating = old_predicating;
 339
 340         radv_meta_restore(&saved_state, cmd_buffer);
 341 }
 342
 343 void radv_CmdCopyImageToBuffer(
 344         VkCommandBuffer                             commandBuffer,
 345         VkImage                                     srcImage,
 346         VkImageLayout                               srcImageLayout,
 347         VkBuffer                                    destBuffer,
 348         uint32_t                                    regionCount,
 349         const VkBufferImageCopy*                    pRegions)
 350 {
 351         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 352         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 353         RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
 354
 355         meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
 356                                   srcImageLayout,
 357                                   regionCount, pRegions);
 358 }
 359
 360 static void
 361 meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
 362                 struct radv_image *src_image,
 363                 VkImageLayout src_image_layout,
 364                 struct radv_image *dest_image,
 365                 VkImageLayout dest_image_layout,
 366                 uint32_t regionCount,
 367                 const VkImageCopy *pRegions)
 368 {
 369         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 370         struct radv_meta_saved_state saved_state;
 371         bool old_predicating;
 372
 373         /* From the Vulkan 1.0 spec:
 374          *
 375          *    vkCmdCopyImage can be used to copy image data between multisample
 376          *    images, but both images must have the same number of samples.
 377          */
 378         assert(src_image->info.samples == dest_image->info.samples);
 379
 380         radv_meta_save(&saved_state, cmd_buffer,
 381                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 382                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 383                        RADV_META_SAVE_CONSTANTS |
 384                        RADV_META_SAVE_DESCRIPTORS);
 385
 386         /* VK_EXT_conditional_rendering says that copy commands should not be
 387          * affected by conditional rendering.
 388          */
 389         old_predicating = cmd_buffer->state.predicating;
 390         cmd_buffer->state.predicating = false;
 391
 392         for (unsigned r = 0; r < regionCount; r++) {
 393                 assert(pRegions[r].srcSubresource.aspectMask ==
 394                        pRegions[r].dstSubresource.aspectMask);
 395
 396                 /* Create blit surfaces */
 397                 struct radv_meta_blit2d_surf b_src =
 398                         blit_surf_for_image_level_layer(src_image,
 399                                                         src_image_layout,
 400                                                         &pRegions[r].srcSubresource);
 401
 402                 struct radv_meta_blit2d_surf b_dst =
 403                         blit_surf_for_image_level_layer(dest_image,
 404                                                         dest_image_layout,
 405                                                         &pRegions[r].dstSubresource);
 406
 407                 uint32_t dst_queue_mask = radv_image_queue_family_mask(dest_image,
 408                                                                        cmd_buffer->queue_family_index,
 409                                                                        cmd_buffer->queue_family_index);
 410                 bool dst_compressed = radv_layout_dcc_compressed(dest_image, dest_image_layout, dst_queue_mask);
 411                 uint32_t src_queue_mask = radv_image_queue_family_mask(src_image,
 412                                                                        cmd_buffer->queue_family_index,
 413                                                                        cmd_buffer->queue_family_index);
 414                 bool src_compressed = radv_layout_dcc_compressed(src_image, src_image_layout, src_queue_mask);
 415
 416                 if (!src_compressed || radv_dcc_formats_compatible(b_src.format, b_dst.format)) {
 417                         b_src.format = b_dst.format;
 418                 } else if (!dst_compressed) {
 419                         b_dst.format = b_src.format;
 420                 } else {
 421                         radv_decompress_dcc(cmd_buffer, dest_image, &(VkImageSubresourceRange) {
 422                                                 .aspectMask = pRegions[r].dstSubresource.aspectMask,
 423                                                 .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
 424                                                 .levelCount = 1,
 425                                                 .baseArrayLayer = pRegions[r].dstSubresource.baseArrayLayer,
 426                                                 .layerCount = pRegions[r].dstSubresource.layerCount,
 427                                             });
 428                         b_dst.format = b_src.format;
 429                         b_dst.current_layout = VK_IMAGE_LAYOUT_GENERAL;
 430                 }
 431
 432
 433                 /**
 434                  * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 435                  *    imageExtent is the size in texels of the image to copy in width, height
 436                  *    and depth. 1D images use only x and width. 2D images use x, y, width
 437                  *    and height. 3D images use x, y, z, width, height and depth.
 438                  *
 439                  * Also, convert the offsets and extent from units of texels to units of
 440                  * blocks - which is the highest resolution accessible in this command.
 441                  */
 442                 const VkOffset3D dst_offset_el =
 443                         meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 444                 const VkOffset3D src_offset_el =
 445                         meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 446
 447                 /*
 448                  * From Vulkan 1.0.68, "Copying Data Between Images":
 449                  *    "When copying between compressed and uncompressed formats
 450                  *     the extent members represent the texel dimensions of the
 451                  *     source image and not the destination."
 452                  * However, we must use the destination image type to avoid
 453                  * clamping depth when copying multiple layers of a 2D image to
 454                  * a 3D image.
 455                  */
 456                 const VkExtent3D img_extent_el =
 457                         meta_region_extent_el(src_image, dest_image->type, &pRegions[r].extent);
 458
 459                 /* Start creating blit rect */
 460                 struct radv_meta_blit2d_rect rect = {
 461                         .width = img_extent_el.width,
 462                         .height = img_extent_el.height,
 463                 };
 464
 465                 if (src_image->type == VK_IMAGE_TYPE_3D)
 466                         b_src.layer = src_offset_el.z;
 467
 468                 if (dest_image->type == VK_IMAGE_TYPE_3D)
 469                         b_dst.layer = dst_offset_el.z;
 470
 471                 /* Loop through each 3D or array slice */
 472                 unsigned num_slices_3d = img_extent_el.depth;
 473                 unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 474                 unsigned slice_3d = 0;
 475                 unsigned slice_array = 0;
 476                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 477
 478                         /* Finish creating blit rect */
 479                         rect.dst_x = dst_offset_el.x;
 480                         rect.dst_y = dst_offset_el.y;
 481                         rect.src_x = src_offset_el.x;
 482                         rect.src_y = src_offset_el.y;
 483
 484                         /* Perform Blit */
 485                         if (cs)
 486                                 radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
 487                         else
 488                                 radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
 489
 490                         b_src.layer++;
 491                         b_dst.layer++;
 492                         if (dest_image->type == VK_IMAGE_TYPE_3D)
 493                                 slice_3d++;
 494                         else
 495                                 slice_array++;
 496                 }
 497         }
 498
 499         /* Restore conditional rendering. */
 500         cmd_buffer->state.predicating = old_predicating;
 501
 502         radv_meta_restore(&saved_state, cmd_buffer);
 503 }
 504
 505 void radv_CmdCopyImage(
 506         VkCommandBuffer                             commandBuffer,
 507         VkImage                                     srcImage,
 508         VkImageLayout                               srcImageLayout,
 509         VkImage                                     destImage,
 510         VkImageLayout                               destImageLayout,
 511         uint32_t                                    regionCount,
 512         const VkImageCopy*                          pRegions)
 513 {
 514         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 515         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 516         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 517
 518         meta_copy_image(cmd_buffer,
 519                         src_image, srcImageLayout,
 520                         dest_image, destImageLayout,
 521                         regionCount, pRegions);
 522 }