src/amd/vulkan/radv_meta_copy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "radv_meta.h"
  25 #include "vk_format.h"
  26
  27 static VkExtent3D
  28 meta_image_block_size(const struct radv_image *image)
  29 {
  30         const struct vk_format_description *desc = vk_format_description(image->vk_format);
  31         return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
  32 }
  33
  34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
  35  * elements rather than texels. One element equals one texel or one block
  36  * if Image is uncompressed or compressed, respectively.
  37  */
  38 static struct VkExtent3D
  39 meta_region_extent_el(const struct radv_image *image,
  40                       const struct VkExtent3D *extent)
  41 {
  42         const VkExtent3D block = meta_image_block_size(image);
  43         return radv_sanitize_image_extent(image->type, (VkExtent3D) {
  44                         .width  = DIV_ROUND_UP(extent->width , block.width),
  45                                 .height = DIV_ROUND_UP(extent->height, block.height),
  46                                 .depth  = DIV_ROUND_UP(extent->depth , block.depth),
  47                                 });
  48 }
  49
  50 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
  51  * elements rather than texels. One element equals one texel or one block
  52  * if Image is uncompressed or compressed, respectively.
  53  */
  54 static struct VkOffset3D
  55 meta_region_offset_el(const struct radv_image *image,
  56                       const struct VkOffset3D *offset)
  57 {
  58         const VkExtent3D block = meta_image_block_size(image);
  59         return radv_sanitize_image_offset(image->type, (VkOffset3D) {
  60                         .x = offset->x / block.width,
  61                                 .y = offset->y / block.height,
  62                                 .z = offset->z / block.depth,
  63                                 });
  64 }
  65
  66 static VkFormat
  67 vk_format_for_size(int bs)
  68 {
  69         switch (bs) {
  70         case 1: return VK_FORMAT_R8_UINT;
  71         case 2: return VK_FORMAT_R8G8_UINT;
  72         case 4: return VK_FORMAT_R8G8B8A8_UINT;
  73         case 8: return VK_FORMAT_R16G16B16A16_UINT;
  74         case 16: return VK_FORMAT_R32G32B32A32_UINT;
  75         default:
  76                 unreachable("Invalid format block size");
  77         }
  78 }
  79
  80 static struct radv_meta_blit2d_surf
  81 blit_surf_for_image_level_layer(struct radv_image *image,
  82                                 const VkImageSubresourceLayers *subres)
  83 {
  84         VkFormat format = image->vk_format;
  85         if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
  86                 format = vk_format_depth_only(format);
  87         else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
  88                 format = vk_format_stencil_only(format);
  89
  90         if (!image->surface.dcc_size)
  91                 format = vk_format_for_size(vk_format_get_blocksize(format));
  92
  93         return (struct radv_meta_blit2d_surf) {
  94                 .format = format,
  95                 .bs = vk_format_get_blocksize(format),
  96                 .level = subres->mipLevel,
  97                 .layer = subres->baseArrayLayer,
  98                 .image = image,
  99                 .aspect_mask = subres->aspectMask,
 100         };
 101 }
 102
 103 static void
 104 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
 105                           struct radv_buffer* buffer,
 106                           struct radv_image* image,
 107                           uint32_t regionCount,
 108                           const VkBufferImageCopy* pRegions)
 109 {
 110         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 111         struct radv_meta_saved_state saved_state;
 112
 113         /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 114          * VK_SAMPLE_COUNT_1_BIT."
 115          */
 116         assert(image->info.samples == 1);
 117
 118         radv_meta_save(&saved_state, cmd_buffer,
 119                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 120                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 121                        RADV_META_SAVE_CONSTANTS |
 122                        RADV_META_SAVE_DESCRIPTORS);
 123
 124         for (unsigned r = 0; r < regionCount; r++) {
 125
 126                 /**
 127                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 128                  *    extent is the size in texels of the source image to copy in width,
 129                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 130                  *    width and height. 3D images use x, y, z, width, height and depth.
 131                  *
 132                  *
 133                  * Also, convert the offsets and extent from units of texels to units of
 134                  * blocks - which is the highest resolution accessible in this command.
 135                  */
 136                 const VkOffset3D img_offset_el =
 137                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 138                 const VkExtent3D bufferExtent = {
 139                         .width  = pRegions[r].bufferRowLength ?
 140                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 141                         .height = pRegions[r].bufferImageHeight ?
 142                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 143                 };
 144                 const VkExtent3D buf_extent_el =
 145                         meta_region_extent_el(image, &bufferExtent);
 146
 147                 /* Start creating blit rect */
 148                 const VkExtent3D img_extent_el =
 149                         meta_region_extent_el(image, &pRegions[r].imageExtent);
 150                 struct radv_meta_blit2d_rect rect = {
 151                         .width = img_extent_el.width,
 152                         .height =  img_extent_el.height,
 153                 };
 154
 155                 /* Create blit surfaces */
 156                 struct radv_meta_blit2d_surf img_bsurf =
 157                         blit_surf_for_image_level_layer(image,
 158                                                         &pRegions[r].imageSubresource);
 159
 160                 struct radv_meta_blit2d_buffer buf_bsurf = {
 161                         .bs = img_bsurf.bs,
 162                         .format = img_bsurf.format,
 163                         .buffer = buffer,
 164                         .offset = pRegions[r].bufferOffset,
 165                         .pitch = buf_extent_el.width,
 166                 };
 167
 168                 if (image->type == VK_IMAGE_TYPE_3D)
 169                         img_bsurf.layer = img_offset_el.z;
 170                 /* Loop through each 3D or array slice */
 171                 unsigned num_slices_3d = img_extent_el.depth;
 172                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 173                 unsigned slice_3d = 0;
 174                 unsigned slice_array = 0;
 175                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 176
 177                         rect.dst_x = img_offset_el.x;
 178                         rect.dst_y = img_offset_el.y;
 179
 180
 181                         /* Perform Blit */
 182                         if (cs)
 183                                 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
 184                         else
 185                                 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
 186
 187                         /* Once we've done the blit, all of the actual information about
 188                          * the image is embedded in the command buffer so we can just
 189                          * increment the offset directly in the image effectively
 190                          * re-binding it to different backing memory.
 191                          */
 192                         buf_bsurf.offset += buf_extent_el.width *
 193                                             buf_extent_el.height * buf_bsurf.bs;
 194                         img_bsurf.layer++;
 195                         if (image->type == VK_IMAGE_TYPE_3D)
 196                                 slice_3d++;
 197                         else
 198                                 slice_array++;
 199                 }
 200         }
 201
 202         radv_meta_restore(&saved_state, cmd_buffer);
 203 }
 204
 205 void radv_CmdCopyBufferToImage(
 206         VkCommandBuffer                             commandBuffer,
 207         VkBuffer                                    srcBuffer,
 208         VkImage                                     destImage,
 209         VkImageLayout                               destImageLayout,
 210         uint32_t                                    regionCount,
 211         const VkBufferImageCopy*                    pRegions)
 212 {
 213         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 214         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 215         RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
 216
 217         meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image,
 218                                   regionCount, pRegions);
 219 }
 220
 221 static void
 222 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 223                           struct radv_buffer* buffer,
 224                           struct radv_image* image,
 225                           uint32_t regionCount,
 226                           const VkBufferImageCopy* pRegions)
 227 {
 228         struct radv_meta_saved_state saved_state;
 229
 230         radv_meta_save(&saved_state, cmd_buffer,
 231                        RADV_META_SAVE_COMPUTE_PIPELINE |
 232                        RADV_META_SAVE_CONSTANTS |
 233                        RADV_META_SAVE_DESCRIPTORS);
 234
 235         for (unsigned r = 0; r < regionCount; r++) {
 236
 237                 /**
 238                  * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
 239                  *    extent is the size in texels of the source image to copy in width,
 240                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 241                  *    width and height. 3D images use x, y, z, width, height and depth.
 242                  *
 243                  *
 244                  * Also, convert the offsets and extent from units of texels to units of
 245                  * blocks - which is the highest resolution accessible in this command.
 246                  */
 247                 const VkOffset3D img_offset_el =
 248                         meta_region_offset_el(image, &pRegions[r].imageOffset);
 249                 const VkExtent3D bufferExtent = {
 250                         .width  = pRegions[r].bufferRowLength ?
 251                         pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
 252                         .height = pRegions[r].bufferImageHeight ?
 253                         pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
 254                 };
 255                 const VkExtent3D buf_extent_el =
 256                         meta_region_extent_el(image, &bufferExtent);
 257
 258                 /* Start creating blit rect */
 259                 const VkExtent3D img_extent_el =
 260                         meta_region_extent_el(image, &pRegions[r].imageExtent);
 261                 struct radv_meta_blit2d_rect rect = {
 262                         .width = img_extent_el.width,
 263                         .height =  img_extent_el.height,
 264                 };
 265
 266                 /* Create blit surfaces */
 267                 struct radv_meta_blit2d_surf img_info =
 268                         blit_surf_for_image_level_layer(image,
 269                                                         &pRegions[r].imageSubresource);
 270
 271                 struct radv_meta_blit2d_buffer buf_info = {
 272                         .bs = img_info.bs,
 273                         .format = img_info.format,
 274                         .buffer = buffer,
 275                         .offset = pRegions[r].bufferOffset,
 276                         .pitch = buf_extent_el.width,
 277                 };
 278
 279                 if (image->type == VK_IMAGE_TYPE_3D)
 280                         img_info.layer = img_offset_el.z;
 281                 /* Loop through each 3D or array slice */
 282                 unsigned num_slices_3d = img_extent_el.depth;
 283                 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 284                 unsigned slice_3d = 0;
 285                 unsigned slice_array = 0;
 286                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 287
 288                         rect.src_x = img_offset_el.x;
 289                         rect.src_y = img_offset_el.y;
 290
 291
 292                         /* Perform Blit */
 293                         radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
 294
 295                         buf_info.offset += buf_extent_el.width *
 296                                             buf_extent_el.height * buf_info.bs;
 297                         img_info.layer++;
 298                         if (image->type == VK_IMAGE_TYPE_3D)
 299                                 slice_3d++;
 300                         else
 301                                 slice_array++;
 302                 }
 303         }
 304
 305         radv_meta_restore(&saved_state, cmd_buffer);
 306 }
 307
 308 void radv_CmdCopyImageToBuffer(
 309         VkCommandBuffer                             commandBuffer,
 310         VkImage                                     srcImage,
 311         VkImageLayout                               srcImageLayout,
 312         VkBuffer                                    destBuffer,
 313         uint32_t                                    regionCount,
 314         const VkBufferImageCopy*                    pRegions)
 315 {
 316         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 317         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 318         RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
 319
 320         meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
 321                                   regionCount, pRegions);
 322 }
 323
 324 static void
 325 meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
 326                 struct radv_image *src_image,
 327                 struct radv_image *dest_image,
 328                 uint32_t regionCount,
 329                 const VkImageCopy *pRegions)
 330 {
 331         bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
 332         struct radv_meta_saved_state saved_state;
 333
 334         /* From the Vulkan 1.0 spec:
 335          *
 336          *    vkCmdCopyImage can be used to copy image data between multisample
 337          *    images, but both images must have the same number of samples.
 338          */
 339         assert(src_image->info.samples == dest_image->info.samples);
 340
 341         radv_meta_save(&saved_state, cmd_buffer,
 342                        (cs ? RADV_META_SAVE_COMPUTE_PIPELINE :
 343                         RADV_META_SAVE_GRAPHICS_PIPELINE) |
 344                        RADV_META_SAVE_CONSTANTS |
 345                        RADV_META_SAVE_DESCRIPTORS);
 346
 347         for (unsigned r = 0; r < regionCount; r++) {
 348                 assert(pRegions[r].srcSubresource.aspectMask ==
 349                        pRegions[r].dstSubresource.aspectMask);
 350
 351                 /* Create blit surfaces */
 352                 struct radv_meta_blit2d_surf b_src =
 353                         blit_surf_for_image_level_layer(src_image,
 354                                                         &pRegions[r].srcSubresource);
 355
 356                 struct radv_meta_blit2d_surf b_dst =
 357                         blit_surf_for_image_level_layer(dest_image,
 358                                                         &pRegions[r].dstSubresource);
 359
 360                 /* for DCC */
 361                 b_src.format = b_dst.format;
 362
 363                 /**
 364                  * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
 365                  *    imageExtent is the size in texels of the image to copy in width, height
 366                  *    and depth. 1D images use only x and width. 2D images use x, y, width
 367                  *    and height. 3D images use x, y, z, width, height and depth.
 368                  *
 369                  * Also, convert the offsets and extent from units of texels to units of
 370                  * blocks - which is the highest resolution accessible in this command.
 371                  */
 372                 const VkOffset3D dst_offset_el =
 373                         meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
 374                 const VkOffset3D src_offset_el =
 375                         meta_region_offset_el(src_image, &pRegions[r].srcOffset);
 376                 const VkExtent3D img_extent_el =
 377                         meta_region_extent_el(dest_image, &pRegions[r].extent);
 378
 379                 /* Start creating blit rect */
 380                 struct radv_meta_blit2d_rect rect = {
 381                         .width = img_extent_el.width,
 382                         .height = img_extent_el.height,
 383                 };
 384
 385                 if (dest_image->type == VK_IMAGE_TYPE_3D)
 386                         b_dst.layer = dst_offset_el.z;
 387
 388                 /* Loop through each 3D or array slice */
 389                 unsigned num_slices_3d = img_extent_el.depth;
 390                 unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
 391                 unsigned slice_3d = 0;
 392                 unsigned slice_array = 0;
 393                 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 394
 395                         /* Finish creating blit rect */
 396                         rect.dst_x = dst_offset_el.x;
 397                         rect.dst_y = dst_offset_el.y;
 398                         rect.src_x = src_offset_el.x;
 399                         rect.src_y = src_offset_el.y;
 400
 401                         /* Perform Blit */
 402                         if (cs)
 403                                 radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
 404                         else
 405                                 radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
 406
 407                         b_src.layer++;
 408                         b_dst.layer++;
 409                         if (dest_image->type == VK_IMAGE_TYPE_3D)
 410                                 slice_3d++;
 411                         else
 412                                 slice_array++;
 413                 }
 414         }
 415
 416         radv_meta_restore(&saved_state, cmd_buffer);
 417 }
 418
 419 void radv_CmdCopyImage(
 420         VkCommandBuffer                             commandBuffer,
 421         VkImage                                     srcImage,
 422         VkImageLayout                               srcImageLayout,
 423         VkImage                                     destImage,
 424         VkImageLayout                               destImageLayout,
 425         uint32_t                                    regionCount,
 426         const VkImageCopy*                          pRegions)
 427 {
 428         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 429         RADV_FROM_HANDLE(radv_image, src_image, srcImage);
 430         RADV_FROM_HANDLE(radv_image, dest_image, destImage);
 431
 432         meta_copy_image(cmd_buffer, src_image, dest_image,
 433                         regionCount, pRegions);
 434 }
 435
 436 void radv_blit_to_prime_linear(struct radv_cmd_buffer *cmd_buffer,
 437                                struct radv_image *image,
 438                                struct radv_image *linear_image)
 439 {
 440         struct VkImageCopy image_copy = { 0 };
 441
 442         image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
 443         image_copy.srcSubresource.layerCount = 1;
 444
 445         image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
 446         image_copy.dstSubresource.layerCount = 1;
 447
 448         image_copy.extent.width = image->info.width;
 449         image_copy.extent.height = image->info.height;
 450         image_copy.extent.depth = 1;
 451
 452         meta_copy_image(cmd_buffer, image, linear_image,
 453                         1, &image_copy);
 454 }