src/amd/vulkan/radv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "radv_meta.h"
  25 #include "vk_format.h"
  26
  27 static VkExtent3D
  28 meta_image_block_size(const struct radv_image *image)
  29 {
  30         const struct vk_format_description *desc = vk_format_description(image->vk_format);
  31         return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
  32 }
  33
  34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  35  * elements rather than texels. One element equals one texel or one block
  36  * if Image is uncompressed or compressed, respectively.
  37  */
  38 static struct VkExtent3D
  39 meta_region_extent_el(const struct radv_image *image,
  40                       const VkImageType imageType,
  41                       const struct VkExtent3D *extent)
  42 {
  43         const VkExtent3D block = meta_image_block_size(image);
  44         return radv_sanitize_image_extent(imageType, (VkExtent3D) {
  45                         .width  = DIV_ROUND_UP(extent->width , block.width),
  46                                 .height = DIV_ROUND_UP(extent->height, block.height),
  47                                 .depth  = DIV_ROUND_UP(extent->depth , block.depth),
  48                                 });
  49 }
  50
  51 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  52  * elements rather than texels. One element equals one texel or one block
  53  * if Image is uncompressed or compressed, respectively.
  54  */
  55 static struct VkOffset3D
  56 meta_region_offset_el(const struct radv_image *image,
  57                       const struct VkOffset3D *offset)
  58 {
  59         const VkExtent3D block = meta_image_block_size(image);
  60         return radv_sanitize_image_offset(image->type, (VkOffset3D) {
  61                         .x = offset->x / block.width,
  62                                 .y = offset->y / block.height,
  63                                 .z = offset->z / block.depth,
  64                                 });
  65 }
  66
  67 static VkFormat
  68 vk_format_for_size(int bs)
  69 {
  70         switch (bs) {
  71         case 1: return VK_FORMAT_R8_UINT;
  72         case 2: return VK_FORMAT_R8G8_UINT;
  73         case 4: return VK_FORMAT_R8G8B8A8_UINT;
  74         case 8: return VK_FORMAT_R16G16B16A16_UINT;
  75         case 12: return VK_FORMAT_R32G32B32_UINT;
  76         case 16: return VK_FORMAT_R32G32B32A32_UINT;
  77         default:
  78                 unreachable("Invalid format block size");
  79         }
  80 }
  81
  82 static struct radv_meta_blit2d_surf
  83 blit_surf_for_image_level_layer(struct radv_image *image,
  84                                 VkImageLayout layout,
  85                                 const VkImageSubresourceLayers *subres,
  86                                 VkImageAspectFlags aspect_mask)
  87 {
  88         VkFormat format = radv_get_aspect_format(image, aspect_mask);
  89
  90         if (!radv_dcc_enabled(image, subres->mipLevel) &&
  91             !(radv_image_is_tc_compat_htile(image)))
  92                 format = vk_format_for_size(vk_format_get_blocksize(format));
  93
  94         format = vk_format_no_srgb(format);
  95
  96         return (struct radv_meta_blit2d_surf) {
  97                 .format = format,
  98                 .bs = vk_format_get_blocksize(format),
  99                 .level = subres->mipLevel,
 100                 .layer = subres->baseArrayLayer,
 101                 .image = image,
 102                 .aspect_mask = aspect_mask,
 103                 .current_layout = layout,
 104         };
 105 }
 106
 107 static bool
 108 image_is_renderable(struct radv_device *device, struct radv_image *image)
 109 {
 110         if (image->vk_format == VK_FORMAT_R32G32B32_UINT ||
 111             image->vk_format == VK_FORMAT_R32G32B32_SINT ||
 112             image->vk_format == VK_FORMAT_R32G32B32_SFLOAT)
 113                 return false;
 114
 115         if (device->physical_device->rad_info.chip_class >= GFX9 &&
 116             image->type == VK_IMAGE_TYPE_3D &&
 117             vk_format_get_blocksizebits(image->vk_format) == 128 &&
 118             vk_format_is_compressed(image->vk_format))
 119                 return false;
 120         return true;
 121 }
 122
 123 static void
 124 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
 125                           struct radv_buffer* buffer,
 126                           struct radv_image* image,
 127                           VkImageLayout layout,
 128                           uint32_t regionCount,
 129                           const VkBufferImageCopy* pRegions)
 130 {
 131         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 132         struct radv_meta_saved_state saved_state;
 133         bool old_predicating;
 134
 135         /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 136          * VK_SAMPLE_COUNT_1_BIT."
 137          */
 138         assert(image->info.samples == 1);
 139
 140         radv_meta_save(&saved_state, cmd_buffer,
 141                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 142                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 143                        RADV_META_SAVE_CONSTANTS |
 144                        RADV_META_SAVE_DESCRIPTORS);
 145
 146         /* VK_EXT_conditional_rendering says that copy commands should not be
 147          * affected by conditional rendering.
 148          */
 149         old_predicating = cmd_buffer->state.predicating;
 150         cmd_buffer->state.predicating = false;
 151
 152         for (unsigned r = 0; r < regionCount; r++) {
 153
 154                 /**
 155                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 156                  *    extent is the size in texels of the source image to copy in width,
 157                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 158                  *    width and height. 3D images use x, y, z, width, height and depth.
 159                  *
 160                  *
 161                  * Also, convert the offsets and extent from units of texels to units of
 162                  * blocks - which is the highest resolution accessible in this command.
 163                  */
 164                 const VkOffset3D img_offset_el =
 165                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 166                 const VkExtent3D bufferExtent = {
 167                         .width  = pRegions[r].bufferRowLength ?
 168                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 169                         .height = pRegions[r].bufferImageHeight ?
 170                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 171                 };
 172                 const VkExtent3D buf_extent_el =
 173                         meta_region_extent_el(image, image->type, &bufferExtent);
 174
 175                 /* Start creating blit rect */
 176                 const VkExtent3D img_extent_el =
 177                         meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
 178                 struct radv_meta_blit2d_rect rect = {
 179                         .width = img_extent_el.width,
 180                         .height =  img_extent_el.height,
 181                 };
 182
 183                 /* Create blit surfaces */
 184                 struct radv_meta_blit2d_surf img_bsurf =
 185                         blit_surf_for_image_level_layer(image,
 186                                                         layout,
 187                                                         &pRegions[r].imageSubresource,
 188                                                         pRegions[r].imageSubresource.aspectMask);
 189
 190                 if (!radv_is_buffer_format_supported(img_bsurf.format, NULL)) {
 191                         uint32_t queue_mask = radv_image_queue_family_mask(image,
 192                                                                            cmd_buffer->queue_family_index,
 193                                                                            cmd_buffer->queue_family_index);
 194                         bool compressed = radv_layout_dcc_compressed(cmd_buffer->device, image, layout, false, queue_mask);
 195                         if (compressed) {
 196                                 radv_decompress_dcc(cmd_buffer, image, &(VkImageSubresourceRange) {
 197                                                                 .aspectMask = pRegions[r].imageSubresource.aspectMask,
 198                                                                 .baseMipLevel = pRegions[r].imageSubresource.mipLevel,
 199                                                                 .levelCount = 1,
 200                                                                 .baseArrayLayer = pRegions[r].imageSubresource.baseArrayLayer,
 201                                                                 .layerCount = pRegions[r].imageSubresource.layerCount,
 202                                                         });
 203                         }
 204                         img_bsurf.format = vk_format_for_size(vk_format_get_blocksize(img_bsurf.format));
 205                         img_bsurf.current_layout = VK_IMAGE_LAYOUT_GENERAL;
 206                 }
 207
 208                 struct radv_meta_blit2d_buffer buf_bsurf = {
 209                         .bs = img_bsurf.bs,
 210                         .format = img_bsurf.format,
 211                         .buffer = buffer,
 212                         .offset = pRegions[r].bufferOffset,
 213                         .pitch = buf_extent_el.width,
 214                 };
 215
 216                 if (image->type == VK_IMAGE_TYPE_3D)
 217                         img_bsurf.layer = img_offset_el.z;
 218                 /* Loop through each 3D or array slice */
 219                 unsigned num_slices_3d = img_extent_el.depth;
 220                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 221                 unsigned slice_3d = 0;
 222                 unsigned slice_array = 0;
 223                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 224
 225                         rect.dst_x = img_offset_el.x;
 226                         rect.dst_y = img_offset_el.y;
 227
 228
 229                         /* Perform Blit */
 230                         if (cs ||
 231                             !image_is_renderable(cmd_buffer->device, img_bsurf.image)) {
 232                                 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
 233                         } else {
 234                                 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
 235                         }
 236
 237                         /* Once we've done the blit, all of the actual information about
 238                          * the image is embedded in the command buffer so we can just
 239                          * increment the offset directly in the image effectively
 240                          * re-binding it to different backing memory.
 241                          */
 242                         buf_bsurf.offset += buf_extent_el.width *
 243                                             buf_extent_el.height * buf_bsurf.bs;
 244                         img_bsurf.layer++;
 245                         if (image->type == VK_IMAGE_TYPE_3D)
 246                                 slice_3d++;
 247                         else
 248                                 slice_array++;
 249                 }
 250         }
 251
 252         /* Restore conditional rendering. */
 253         cmd_buffer->state.predicating = old_predicating;
 254
 255         radv_meta_restore(&saved_state, cmd_buffer);
 256 }
 257
 258 void radv_CmdCopyBufferToImage(
 259         VkCommandBuffer                             commandBuffer,
 260         VkBuffer                                    srcBuffer,
 261         VkImage                                     destImage,
 262         VkImageLayout                               destImageLayout,
 263         uint32_t                                    regionCount,
 264         const VkBufferImageCopy*                    pRegions)
 265 {
 266         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 267         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 268         RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
 269
 270         meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, destImageLayout,
 271                                   regionCount, pRegions);
 272 }
 273
 274 static void
 275 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 276                           struct radv_buffer* buffer,
 277                           struct radv_image* image,
 278                           VkImageLayout layout,
 279                           uint32_t regionCount,
 280                           const VkBufferImageCopy* pRegions)
 281 {
 282         struct radv_meta_saved_state saved_state;
 283         bool old_predicating;
 284
 285         radv_meta_save(&saved_state, cmd_buffer,
 286                        RADV_META_SAVE_COMPUTE_PIPELINE |
 287                        RADV_META_SAVE_CONSTANTS |
 288                        RADV_META_SAVE_DESCRIPTORS);
 289
 290         /* VK_EXT_conditional_rendering says that copy commands should not be
 291          * affected by conditional rendering.
 292          */
 293         old_predicating = cmd_buffer->state.predicating;
 294         cmd_buffer->state.predicating = false;
 295
 296         for (unsigned r = 0; r < regionCount; r++) {
 297
 298                 /**
 299                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 300                  *    extent is the size in texels of the source image to copy in width,
 301                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 302                  *    width and height. 3D images use x, y, z, width, height and depth.
 303                  *
 304                  *
 305                  * Also, convert the offsets and extent from units of texels to units of
 306                  * blocks - which is the highest resolution accessible in this command.
 307                  */
 308                 const VkOffset3D img_offset_el =
 309                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 310                 const VkExtent3D bufferExtent = {
 311                         .width  = pRegions[r].bufferRowLength ?
 312                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 313                         .height = pRegions[r].bufferImageHeight ?
 314                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 315                 };
 316                 const VkExtent3D buf_extent_el =
 317                         meta_region_extent_el(image, image->type, &bufferExtent);
 318
 319                 /* Start creating blit rect */
 320                 const VkExtent3D img_extent_el =
 321                         meta_region_extent_el(image, image->type, &pRegions[r].imageExtent);
 322                 struct radv_meta_blit2d_rect rect = {
 323                         .width = img_extent_el.width,
 324                         .height =  img_extent_el.height,
 325                 };
 326
 327                 /* Create blit surfaces */
 328                 struct radv_meta_blit2d_surf img_info =
 329                         blit_surf_for_image_level_layer(image,
 330                                                         layout,
 331                                                         &pRegions[r].imageSubresource,
 332                                                         pRegions[r].imageSubresource.aspectMask);
 333
 334                 if (!radv_is_buffer_format_supported(img_info.format, NULL)) {
 335                         uint32_t queue_mask = radv_image_queue_family_mask(image,
 336                                                                            cmd_buffer->queue_family_index,
 337                                                                            cmd_buffer->queue_family_index);
 338                         bool compressed = radv_layout_dcc_compressed(cmd_buffer->device, image, layout, false, queue_mask);
 339                         if (compressed) {
 340                                 radv_decompress_dcc(cmd_buffer, image, &(VkImageSubresourceRange) {
 341                                                                 .aspectMask = pRegions[r].imageSubresource.aspectMask,
 342                                                                 .baseMipLevel = pRegions[r].imageSubresource.mipLevel,
 343                                                                 .levelCount = 1,
 344                                                                 .baseArrayLayer = pRegions[r].imageSubresource.baseArrayLayer,
 345                                                                 .layerCount = pRegions[r].imageSubresource.layerCount,
 346                                                         });
 347                         }
 348                         img_info.format = vk_format_for_size(vk_format_get_blocksize(img_info.format));
 349                         img_info.current_layout = VK_IMAGE_LAYOUT_GENERAL;
 350                 }
 351
 352                 struct radv_meta_blit2d_buffer buf_info = {
 353                         .bs = img_info.bs,
 354                         .format = img_info.format,
 355                         .buffer = buffer,
 356                         .offset = pRegions[r].bufferOffset,
 357                         .pitch = buf_extent_el.width,
 358                 };
 359
 360                 if (image->type == VK_IMAGE_TYPE_3D)
 361                         img_info.layer = img_offset_el.z;
 362                 /* Loop through each 3D or array slice */
 363                 unsigned num_slices_3d = img_extent_el.depth;
 364                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 365                 unsigned slice_3d = 0;
 366                 unsigned slice_array = 0;
 367                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 368
 369                         rect.src_x = img_offset_el.x;
 370                         rect.src_y = img_offset_el.y;
 371
 372
 373                         /* Perform Blit */
 374                         radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
 375
 376                         buf_info.offset += buf_extent_el.width *
 377                                             buf_extent_el.height * buf_info.bs;
 378                         img_info.layer++;
 379                         if (image->type == VK_IMAGE_TYPE_3D)
 380                                 slice_3d++;
 381                         else
 382                                 slice_array++;
 383                 }
 384         }
 385
 386         /* Restore conditional rendering. */
 387         cmd_buffer->state.predicating = old_predicating;
 388
 389         radv_meta_restore(&saved_state, cmd_buffer);
 390 }
 391
 392 void radv_CmdCopyImageToBuffer(
 393         VkCommandBuffer                             commandBuffer,
 394         VkImage                                     srcImage,
 395         VkImageLayout                               srcImageLayout,
 396         VkBuffer                                    destBuffer,
 397         uint32_t                                    regionCount,
 398         const VkBufferImageCopy*                    pRegions)
 399 {
 400         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 401         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 402         RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
 403
 404         meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
 405                                   srcImageLayout,
 406                                   regionCount, pRegions);
 407 }
 408
 409 static void
 410 meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
 411                 struct radv_image *src_image,
 412                 VkImageLayout src_image_layout,
 413                 struct radv_image *dest_image,
 414                 VkImageLayout dest_image_layout,
 415                 uint32_t regionCount,
 416                 const VkImageCopy *pRegions)
 417 {
 418         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 419         struct radv_meta_saved_state saved_state;
 420         bool old_predicating;
 421
 422         /* From the Vulkan 1.0 spec:
 423          *
 424          *    vkCmdCopyImage can be used to copy image data between multisample
 425          *    images, but both images must have the same number of samples.
 426          */
 427         assert(src_image->info.samples == dest_image->info.samples);
 428
 429         radv_meta_save(&saved_state, cmd_buffer,
 430                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 431                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 432                        RADV_META_SAVE_CONSTANTS |
 433                        RADV_META_SAVE_DESCRIPTORS);
 434
 435         /* VK_EXT_conditional_rendering says that copy commands should not be
 436          * affected by conditional rendering.
 437          */
 438         old_predicating = cmd_buffer->state.predicating;
 439         cmd_buffer->state.predicating = false;
 440
 441         for (unsigned r = 0; r < regionCount; r++) {
 442                 VkImageAspectFlags src_aspects[3] = {VK_IMAGE_ASPECT_PLANE_0_BIT, VK_IMAGE_ASPECT_PLANE_1_BIT, VK_IMAGE_ASPECT_PLANE_2_BIT};
 443                 VkImageAspectFlags dst_aspects[3] = {VK_IMAGE_ASPECT_PLANE_0_BIT, VK_IMAGE_ASPECT_PLANE_1_BIT, VK_IMAGE_ASPECT_PLANE_2_BIT};
 444                 unsigned aspect_count = pRegions[r].srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT ? src_image->plane_count : 1;
 445                 if (pRegions[r].srcSubresource.aspectMask != VK_IMAGE_ASPECT_COLOR_BIT)
 446                         src_aspects[0] = pRegions[r].srcSubresource.aspectMask;
 447                 if (pRegions[r].dstSubresource.aspectMask != VK_IMAGE_ASPECT_COLOR_BIT)
 448                         dst_aspects[0] = pRegions[r].dstSubresource.aspectMask;
 449
 450                 for (unsigned a = 0; a < aspect_count; ++a) {
 451                         /* Create blit surfaces */
 452                         struct radv_meta_blit2d_surf b_src =
 453                                 blit_surf_for_image_level_layer(src_image,
 454                                                                 src_image_layout,
 455                                                                 &pRegions[r].srcSubresource,
 456                                                                 src_aspects[a]);
 457
 458                         struct radv_meta_blit2d_surf b_dst =
 459                                 blit_surf_for_image_level_layer(dest_image,
 460                                                                 dest_image_layout,
 461                                                                 &pRegions[r].dstSubresource,
 462                                                                 dst_aspects[a]);
 463
 464                         uint32_t dst_queue_mask = radv_image_queue_family_mask(dest_image,
 465                                                                                cmd_buffer->queue_family_index,
 466                                                                                cmd_buffer->queue_family_index);
 467                         bool dst_compressed = radv_layout_dcc_compressed(cmd_buffer->device, dest_image, dest_image_layout, false, dst_queue_mask);
 468                         uint32_t src_queue_mask = radv_image_queue_family_mask(src_image,
 469                                                                                cmd_buffer->queue_family_index,
 470                                                                                cmd_buffer->queue_family_index);
 471                         bool src_compressed = radv_layout_dcc_compressed(cmd_buffer->device, src_image, src_image_layout, false, src_queue_mask);
 472
 473                         if (!src_compressed || radv_dcc_formats_compatible(b_src.format, b_dst.format)) {
 474                                 b_src.format = b_dst.format;
 475                         } else if (!dst_compressed) {
 476                                 b_dst.format = b_src.format;
 477                         } else {
 478                                 radv_decompress_dcc(cmd_buffer, dest_image, &(VkImageSubresourceRange) {
 479                                                         .aspectMask = dst_aspects[a],
 480                                                         .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
 481                                                         .levelCount = 1,
 482                                                         .baseArrayLayer = pRegions[r].dstSubresource.baseArrayLayer,
 483                                                         .layerCount = pRegions[r].dstSubresource.layerCount,
 484                                         });
 485                                 b_dst.format = b_src.format;
 486                                 b_dst.current_layout = VK_IMAGE_LAYOUT_GENERAL;
 487                         }
 488
 489
 490                         /**
 491                          * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 492                          *    imageExtent is the size in texels of the image to copy in width, height
 493                          *    and depth. 1D images use only x and width. 2D images use x, y, width
 494                          *    and height. 3D images use x, y, z, width, height and depth.
 495                          *
 496                          * Also, convert the offsets and extent from units of texels to units of
 497                          * blocks - which is the highest resolution accessible in this command.
 498                          */
 499                         const VkOffset3D dst_offset_el =
 500                                 meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 501                         const VkOffset3D src_offset_el =
 502                                 meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 503
 504                         /*
 505                          * From Vulkan 1.0.68, "Copying Data Between Images":
 506                          *    "When copying between compressed and uncompressed formats
 507                          *     the extent members represent the texel dimensions of the
 508                          *     source image and not the destination."
 509                          * However, we must use the destination image type to avoid
 510                          * clamping depth when copying multiple layers of a 2D image to
 511                          * a 3D image.
 512                          */
 513                         const VkExtent3D img_extent_el =
 514                                 meta_region_extent_el(src_image, dest_image->type, &pRegions[r].extent);
 515
 516                         /* Start creating blit rect */
 517                         struct radv_meta_blit2d_rect rect = {
 518                                 .width = img_extent_el.width,
 519                                 .height = img_extent_el.height,
 520                         };
 521
 522                         if (src_image->type == VK_IMAGE_TYPE_3D)
 523                                 b_src.layer = src_offset_el.z;
 524
 525                         if (dest_image->type == VK_IMAGE_TYPE_3D)
 526                                 b_dst.layer = dst_offset_el.z;
 527
 528                         /* Loop through each 3D or array slice */
 529                         unsigned num_slices_3d = img_extent_el.depth;
 530                         unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 531                         unsigned slice_3d = 0;
 532                         unsigned slice_array = 0;
 533                         while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 534
 535                                 /* Finish creating blit rect */
 536                                 rect.dst_x = dst_offset_el.x;
 537                                 rect.dst_y = dst_offset_el.y;
 538                                 rect.src_x = src_offset_el.x;
 539                                 rect.src_y = src_offset_el.y;
 540
 541                                 /* Perform Blit */
 542                                 if (cs ||
 543                                     !image_is_renderable(cmd_buffer->device, b_dst.image)) {
 544                                         radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
 545                                 } else {
 546                                         radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
 547                                 }
 548
 549                                 b_src.layer++;
 550                                 b_dst.layer++;
 551                                 if (dest_image->type == VK_IMAGE_TYPE_3D)
 552                                         slice_3d++;
 553                                 else
 554                                         slice_array++;
 555                         }
 556                 }
 557         }
 558
 559         /* Restore conditional rendering. */
 560         cmd_buffer->state.predicating = old_predicating;
 561
 562         radv_meta_restore(&saved_state, cmd_buffer);
 563 }
 564
 565 void radv_CmdCopyImage(
 566         VkCommandBuffer                             commandBuffer,
 567         VkImage                                     srcImage,
 568         VkImageLayout                               srcImageLayout,
 569         VkImage                                     destImage,
 570         VkImageLayout                               destImageLayout,
 571         uint32_t                                    regionCount,
 572         const VkImageCopy*                          pRegions)
 573 {
 574         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 575         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 576         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 577
 578         meta_copy_image(cmd_buffer,
 579                         src_image, srcImageLayout,
 580                         dest_image, destImageLayout,
 581                         regionCount, pRegions);
 582 }