src/amd/vulkan/radv_meta_resolve.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26
  27 #include "radv_meta.h"
  28 #include "radv_private.h"
  29 #include "vk_format.h"
  30 #include "nir/nir_builder.h"
  31 #include "sid.h"
  32
  33 /* emit 0, 0, 0, 1 */
  34 static nir_shader *
  35 build_nir_fs(void)
  36 {
  37         const struct glsl_type *vec4 = glsl_vec4_type();
  38         nir_builder b;
  39         nir_variable *f_color; /* vec4, fragment output color */
  40
  41         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
  42         b.shader->info.name = ralloc_asprintf(b.shader,
  43                                                "meta_resolve_fs");
  44
  45         f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4,
  46                                       "f_color");
  47         f_color->data.location = FRAG_RESULT_DATA0;
  48         nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf);
  49
  50         return b.shader;
  51 }
  52
  53 static VkResult
  54 create_pass(struct radv_device *device, VkFormat vk_format, VkRenderPass *pass)
  55 {
  56         VkResult result;
  57         VkDevice device_h = radv_device_to_handle(device);
  58         const VkAllocationCallbacks *alloc = &device->meta_state.alloc;
  59         VkAttachmentDescription attachments[2];
  60         int i;
  61
  62         for (i = 0; i < 2; i++) {
  63                 attachments[i].format = vk_format;
  64                 attachments[i].samples = 1;
  65                 attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
  66                 attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
  67         }
  68         attachments[0].initialLayout = VK_IMAGE_LAYOUT_GENERAL;
  69         attachments[0].finalLayout = VK_IMAGE_LAYOUT_GENERAL;
  70         attachments[1].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
  71         attachments[1].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
  72
  73         result = radv_CreateRenderPass(device_h,
  74                                        &(VkRenderPassCreateInfo) {
  75                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
  76                                                        .attachmentCount = 2,
  77                                                        .pAttachments = attachments,
  78                                                        .subpassCount = 1,
  79                                                                 .pSubpasses = &(VkSubpassDescription) {
  80                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
  81                                                        .inputAttachmentCount = 0,
  82                                                        .colorAttachmentCount = 2,
  83                                                        .pColorAttachments = (VkAttachmentReference[]) {
  84                                                                {
  85                                                                        .attachment = 0,
  86                                                                        .layout = VK_IMAGE_LAYOUT_GENERAL,
  87                                                                },
  88                                                                {
  89                                                                        .attachment = 1,
  90                                                                        .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
  91                                                                },
  92                                                        },
  93                                                        .pResolveAttachments = NULL,
  94                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
  95                                                                .attachment = VK_ATTACHMENT_UNUSED,
  96                                                        },
  97                                                        .preserveAttachmentCount = 0,
  98                                                        .pPreserveAttachments = NULL,
  99                                                },
 100                                                         .dependencyCount = 2,
 101                                                         .pDependencies = (VkSubpassDependency[]) {
 102                                                                 {
 103                                                                         .srcSubpass = VK_SUBPASS_EXTERNAL,
 104                                                                         .dstSubpass = 0,
 105                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 106                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 107                                                                         .srcAccessMask = 0,
 108                                                                         .dstAccessMask = 0,
 109                                                                         .dependencyFlags = 0
 110                                                                 },
 111                                                                 {
 112                                                                         .srcSubpass = 0,
 113                                                                         .dstSubpass = VK_SUBPASS_EXTERNAL,
 114                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 115                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 116                                                                         .srcAccessMask = 0,
 117                                                                         .dstAccessMask = 0,
 118                                                                         .dependencyFlags = 0
 119                                                                 }
 120                                                         },
 121                                                                          },
 122                                        alloc,
 123                                        pass);
 124
 125         return result;
 126 }
 127
 128 static VkResult
 129 create_pipeline(struct radv_device *device,
 130                  VkShaderModule vs_module_h,
 131                  VkPipeline *pipeline,
 132                  VkRenderPass pass)
 133 {
 134         VkResult result;
 135         VkDevice device_h = radv_device_to_handle(device);
 136
 137         struct radv_shader_module fs_module = {
 138                 .nir = build_nir_fs(),
 139         };
 140
 141         if (!fs_module.nir) {
 142                 /* XXX: Need more accurate error */
 143                 result = VK_ERROR_OUT_OF_HOST_MEMORY;
 144                 goto cleanup;
 145         }
 146
 147         VkPipelineLayoutCreateInfo pl_create_info = {
 148                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 149                 .setLayoutCount = 0,
 150                 .pSetLayouts = NULL,
 151                 .pushConstantRangeCount = 0,
 152                 .pPushConstantRanges = NULL,
 153         };
 154
 155         if (!device->meta_state.resolve.p_layout) {
 156                 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 157                                                    &pl_create_info,
 158                                                    &device->meta_state.alloc,
 159                                                    &device->meta_state.resolve.p_layout);
 160                 if (result != VK_SUCCESS)
 161                         goto cleanup;
 162         }
 163
 164         result = radv_graphics_pipeline_create(device_h,
 165                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 166                                                &(VkGraphicsPipelineCreateInfo) {
 167                                                        .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 168                                                        .stageCount = 2,
 169                                                        .pStages = (VkPipelineShaderStageCreateInfo[]) {
 170                                                        {
 171                                                                .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 172                                                                .stage = VK_SHADER_STAGE_VERTEX_BIT,
 173                                                                .module = vs_module_h,
 174                                                                .pName = "main",
 175                                                        },
 176                                                        {
 177                                                                .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 178                                                                .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 179                                                                .module = radv_shader_module_to_handle(&fs_module),
 180                                                                .pName = "main",
 181                                                        },
 182                                                },
 183                                                .pVertexInputState = &(VkPipelineVertexInputStateCreateInfo) {
 184                                                        .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 185                                                        .vertexBindingDescriptionCount = 0,
 186                                                        .vertexAttributeDescriptionCount = 0,
 187                                                },
 188                                                .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 189                                                        .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 190                                                        .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 191                                                        .primitiveRestartEnable = false,
 192                                                },
 193                                                .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 194                                                        .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 195                                                        .viewportCount = 1,
 196                                                        .scissorCount = 1,
 197                                                },
 198                                                .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 199                                                        .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 200                                                        .depthClampEnable = false,
 201                                                        .rasterizerDiscardEnable = false,
 202                                                        .polygonMode = VK_POLYGON_MODE_FILL,
 203                                                        .cullMode = VK_CULL_MODE_NONE,
 204                                                        .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
 205                                                },
 206                                                .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
 207                                                        .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 208                                                        .rasterizationSamples = 1,
 209                                                        .sampleShadingEnable = false,
 210                                                        .pSampleMask = NULL,
 211                                                        .alphaToCoverageEnable = false,
 212                                                        .alphaToOneEnable = false,
 213                                                },
 214                                                .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 215                                                        .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 216                                                        .logicOpEnable = false,
 217                                                        .attachmentCount = 2,
 218                                                        .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 219                                                                {
 220                                                                .colorWriteMask = VK_COLOR_COMPONENT_R_BIT |
 221                                                                                VK_COLOR_COMPONENT_G_BIT |
 222                                                                                VK_COLOR_COMPONENT_B_BIT |
 223                                                                                VK_COLOR_COMPONENT_A_BIT,
 224                                                                },
 225                                                                {
 226                                                                .colorWriteMask = 0,
 227
 228                                                                }
 229                                                        },
 230                                                 },
 231                                                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 232                                                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 233                                                         .dynamicStateCount = 2,
 234                                                         .pDynamicStates = (VkDynamicState[]) {
 235                                                                 VK_DYNAMIC_STATE_VIEWPORT,
 236                                                                 VK_DYNAMIC_STATE_SCISSOR,
 237                                                         },
 238                                                 },
 239                                                 .layout = device->meta_state.resolve.p_layout,
 240                                                 .renderPass = pass,
 241                                                                                                                                        .subpass = 0,
 242                                                                                                                                        },
 243                                                &(struct radv_graphics_pipeline_create_info) {
 244                                                        .use_rectlist = true,
 245                                                        .custom_blend_mode = V_028808_CB_RESOLVE,
 246                                                                },
 247                                                &device->meta_state.alloc, pipeline);
 248         if (result != VK_SUCCESS)
 249                 goto cleanup;
 250
 251         goto cleanup;
 252
 253 cleanup:
 254         ralloc_free(fs_module.nir);
 255         return result;
 256 }
 257
 258 void
 259 radv_device_finish_meta_resolve_state(struct radv_device *device)
 260 {
 261         struct radv_meta_state *state = &device->meta_state;
 262
 263         for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) {
 264                 radv_DestroyRenderPass(radv_device_to_handle(device),
 265                                        state->resolve.pass[j], &state->alloc);
 266                 radv_DestroyPipeline(radv_device_to_handle(device),
 267                                      state->resolve.pipeline[j], &state->alloc);
 268         }
 269         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 270                                    state->resolve.p_layout, &state->alloc);
 271
 272 }
 273
 274 VkResult
 275 radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand)
 276 {
 277         if (on_demand)
 278                 return VK_SUCCESS;
 279
 280         VkResult res = VK_SUCCESS;
 281         struct radv_meta_state *state = &device->meta_state;
 282         struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
 283         if (!vs_module.nir) {
 284                 /* XXX: Need more accurate error */
 285                 res = VK_ERROR_OUT_OF_HOST_MEMORY;
 286                 goto fail;
 287         }
 288
 289         for (uint32_t i = 0; i < NUM_META_FS_KEYS; ++i) {
 290                 VkFormat format = radv_fs_key_format_exemplars[i];
 291                 unsigned fs_key = radv_format_meta_fs_key(format);
 292                 res = create_pass(device, format, &state->resolve.pass[fs_key]);
 293                 if (res != VK_SUCCESS)
 294                         goto fail;
 295
 296                 VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
 297                 res = create_pipeline(device, vs_module_h,
 298                                       &state->resolve.pipeline[fs_key], state->resolve.pass[fs_key]);
 299                 if (res != VK_SUCCESS)
 300                         goto fail;
 301         }
 302
 303         goto cleanup;
 304
 305 fail:
 306         radv_device_finish_meta_resolve_state(device);
 307
 308 cleanup:
 309         ralloc_free(vs_module.nir);
 310
 311         return res;
 312 }
 313
 314 static void
 315 emit_resolve(struct radv_cmd_buffer *cmd_buffer,
 316              VkFormat vk_format,
 317              const VkOffset2D *dest_offset,
 318              const VkExtent2D *resolve_extent)
 319 {
 320         struct radv_device *device = cmd_buffer->device;
 321         VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
 322         unsigned fs_key = radv_format_meta_fs_key(vk_format);
 323
 324         cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
 325
 326         radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
 327                              device->meta_state.resolve.pipeline[fs_key]);
 328
 329         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 330                 .x = dest_offset->x,
 331                 .y = dest_offset->y,
 332                 .width = resolve_extent->width,
 333                 .height = resolve_extent->height,
 334                 .minDepth = 0.0f,
 335                 .maxDepth = 1.0f
 336         });
 337
 338         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 339                 .offset = *dest_offset,
 340                 .extent = *resolve_extent,
 341         });
 342
 343         radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
 344         cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
 345 }
 346
 347 enum radv_resolve_method {
 348         RESOLVE_HW,
 349         RESOLVE_COMPUTE,
 350         RESOLVE_FRAGMENT,
 351 };
 352
 353 static void radv_pick_resolve_method_images(struct radv_device *device,
 354                                             struct radv_image *src_image,
 355                                             VkFormat src_format,
 356                                             struct radv_image *dest_image,
 357                                             VkImageLayout dest_image_layout,
 358                                             bool dest_render_loop,
 359                                             struct radv_cmd_buffer *cmd_buffer,
 360                                             enum radv_resolve_method *method)
 361
 362 {
 363         uint32_t queue_mask = radv_image_queue_family_mask(dest_image,
 364                                                            cmd_buffer->queue_family_index,
 365                                                            cmd_buffer->queue_family_index);
 366
 367         if (vk_format_is_color(src_format)) {
 368                 if (src_format == VK_FORMAT_R16G16_UNORM ||
 369                     src_format == VK_FORMAT_R16G16_SNORM)
 370                         *method = RESOLVE_COMPUTE;
 371                 else if (vk_format_is_int(src_format))
 372                         *method = RESOLVE_COMPUTE;
 373                 else if (src_image->info.array_size > 1 ||
 374                          dest_image->info.array_size > 1)
 375                         *method = RESOLVE_COMPUTE;
 376
 377                 if (radv_layout_dcc_compressed(device, dest_image, dest_image_layout,
 378                                                dest_render_loop, queue_mask)) {
 379                         *method = RESOLVE_FRAGMENT;
 380                 } else if (dest_image->planes[0].surface.micro_tile_mode !=
 381                            src_image->planes[0].surface.micro_tile_mode) {
 382                         *method = RESOLVE_COMPUTE;
 383                 }
 384         } else {
 385                 if (src_image->info.array_size > 1 ||
 386                     dest_image->info.array_size > 1)
 387                         *method = RESOLVE_COMPUTE;
 388                 else
 389                         *method = RESOLVE_FRAGMENT;
 390         }
 391 }
 392
 393 static VkResult
 394 build_resolve_pipeline(struct radv_device *device,
 395                        unsigned fs_key)
 396 {
 397         VkResult result = VK_SUCCESS;
 398
 399         if (device->meta_state.resolve.pipeline[fs_key])
 400                 return result;
 401
 402         mtx_lock(&device->meta_state.mtx);
 403         if (device->meta_state.resolve.pipeline[fs_key]) {
 404                 mtx_unlock(&device->meta_state.mtx);
 405                 return result;
 406         }
 407
 408         struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
 409
 410         result = create_pass(device, radv_fs_key_format_exemplars[fs_key], &device->meta_state.resolve.pass[fs_key]);
 411         if (result != VK_SUCCESS)
 412                 goto fail;
 413
 414         VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
 415         result = create_pipeline(device, vs_module_h, &device->meta_state.resolve.pipeline[fs_key], device->meta_state.resolve.pass[fs_key]);
 416
 417 fail:
 418         ralloc_free(vs_module.nir);
 419         mtx_unlock(&device->meta_state.mtx);
 420         return result;
 421 }
 422
 423 void radv_CmdResolveImage(
 424         VkCommandBuffer                             cmd_buffer_h,
 425         VkImage                                     src_image_h,
 426         VkImageLayout                               src_image_layout,
 427         VkImage                                     dest_image_h,
 428         VkImageLayout                               dest_image_layout,
 429         uint32_t                                    region_count,
 430         const VkImageResolve*                       regions)
 431 {
 432         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, cmd_buffer_h);
 433         RADV_FROM_HANDLE(radv_image, src_image, src_image_h);
 434         RADV_FROM_HANDLE(radv_image, dest_image, dest_image_h);
 435         struct radv_device *device = cmd_buffer->device;
 436         struct radv_meta_saved_state saved_state;
 437         VkDevice device_h = radv_device_to_handle(device);
 438         enum radv_resolve_method resolve_method = RESOLVE_HW;
 439         /* we can use the hw resolve only for single full resolves */
 440         if (region_count == 1) {
 441                 if (regions[0].srcOffset.x ||
 442                     regions[0].srcOffset.y ||
 443                     regions[0].srcOffset.z)
 444                         resolve_method = RESOLVE_COMPUTE;
 445                 if (regions[0].dstOffset.x ||
 446                     regions[0].dstOffset.y ||
 447                     regions[0].dstOffset.z)
 448                         resolve_method = RESOLVE_COMPUTE;
 449
 450                 if (regions[0].extent.width != src_image->info.width ||
 451                     regions[0].extent.height != src_image->info.height ||
 452                     regions[0].extent.depth != src_image->info.depth)
 453                         resolve_method = RESOLVE_COMPUTE;
 454         } else
 455                 resolve_method = RESOLVE_COMPUTE;
 456
 457         radv_pick_resolve_method_images(cmd_buffer->device, src_image,
 458                                         src_image->vk_format, dest_image,
 459                                         dest_image_layout, false, cmd_buffer,
 460                                         &resolve_method);
 461
 462         if (resolve_method == RESOLVE_FRAGMENT) {
 463                 radv_meta_resolve_fragment_image(cmd_buffer,
 464                                                  src_image,
 465                                                  src_image_layout,
 466                                                  dest_image,
 467                                                  dest_image_layout,
 468                                                  region_count, regions);
 469                 return;
 470         }
 471
 472         if (resolve_method == RESOLVE_COMPUTE) {
 473                 radv_meta_resolve_compute_image(cmd_buffer,
 474                                                 src_image,
 475                                                 src_image->vk_format,
 476                                                 src_image_layout,
 477                                                 dest_image,
 478                                                 dest_image->vk_format,
 479                                                 dest_image_layout,
 480                                                 region_count, regions);
 481                 return;
 482         }
 483
 484         radv_meta_save(&saved_state, cmd_buffer,
 485                        RADV_META_SAVE_GRAPHICS_PIPELINE);
 486
 487         assert(src_image->info.samples > 1);
 488         if (src_image->info.samples <= 1) {
 489                 /* this causes GPU hangs if we get past here */
 490                 fprintf(stderr, "radv: Illegal resolve operation (src not multisampled), will hang GPU.");
 491                 return;
 492         }
 493         assert(dest_image->info.samples == 1);
 494
 495         if (src_image->info.array_size > 1)
 496                 radv_finishme("vkCmdResolveImage: multisample array images");
 497
 498         unsigned fs_key = radv_format_meta_fs_key(dest_image->vk_format);
 499         for (uint32_t r = 0; r < region_count; ++r) {
 500                 const VkImageResolve *region = &regions[r];
 501
 502                 /* From the Vulkan 1.0 spec:
 503                  *
 504                  *    - The aspectMask member of srcSubresource and dstSubresource must
 505                  *      only contain VK_IMAGE_ASPECT_COLOR_BIT
 506                  *
 507                  *    - The layerCount member of srcSubresource and dstSubresource must
 508                  *      match
 509                  */
 510                 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
 511                 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
 512                 assert(region->srcSubresource.layerCount ==
 513                        region->dstSubresource.layerCount);
 514
 515                 const uint32_t src_base_layer =
 516                         radv_meta_get_iview_layer(src_image, &region->srcSubresource,
 517                                                   &region->srcOffset);
 518
 519                 const uint32_t dest_base_layer =
 520                         radv_meta_get_iview_layer(dest_image, &region->dstSubresource,
 521                                                   &region->dstOffset);
 522
 523                 /**
 524                  * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images
 525                  *
 526                  *    extent is the size in texels of the source image to resolve in width,
 527                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 528                  *    width and height. 3D images use x, y, z, width, height and depth.
 529                  *
 530                  *    srcOffset and dstOffset select the initial x, y, and z offsets in
 531                  *    texels of the sub-regions of the source and destination image data.
 532                  *    extent is the size in texels of the source image to resolve in width,
 533                  *    height and depth. 1D images use only x and width. 2D images use x, y,
 534                  *    width and height. 3D images use x, y, z, width, height and depth.
 535                  */
 536                 const struct VkExtent3D extent =
 537                         radv_sanitize_image_extent(src_image->type, region->extent);
 538                 const struct VkOffset3D dstOffset =
 539                         radv_sanitize_image_offset(dest_image->type, region->dstOffset);
 540
 541                 if (radv_dcc_enabled(dest_image, region->dstSubresource.mipLevel)) {
 542                         VkImageSubresourceRange range = {
 543                                 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 544                                 .baseMipLevel = region->dstSubresource.mipLevel,
 545                                 .levelCount = 1,
 546                                 .baseArrayLayer = dest_base_layer,
 547                                 .layerCount = region->dstSubresource.layerCount,
 548                         };
 549
 550                         radv_initialize_dcc(cmd_buffer, dest_image, &range, 0xffffffff);
 551                 }
 552
 553                 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount;
 554                      ++layer) {
 555
 556                         VkResult ret = build_resolve_pipeline(device, fs_key);
 557                         if (ret != VK_SUCCESS) {
 558                                 cmd_buffer->record_result = ret;
 559                                 break;
 560                         }
 561
 562                         struct radv_image_view src_iview;
 563                         radv_image_view_init(&src_iview, cmd_buffer->device,
 564                                              &(VkImageViewCreateInfo) {
 565                                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 566                                                              .image = src_image_h,
 567                                                              .viewType = radv_meta_get_view_type(src_image),
 568                                                              .format = src_image->vk_format,
 569                                                              .subresourceRange = {
 570                                                              .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 571                                                              .baseMipLevel = region->srcSubresource.mipLevel,
 572                                                              .levelCount = 1,
 573                                                              .baseArrayLayer = src_base_layer + layer,
 574                                                              .layerCount = 1,
 575                                                      },
 576                                              }, NULL);
 577
 578                         struct radv_image_view dest_iview;
 579                         radv_image_view_init(&dest_iview, cmd_buffer->device,
 580                                              &(VkImageViewCreateInfo) {
 581                                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 582                                                              .image = dest_image_h,
 583                                                              .viewType = radv_meta_get_view_type(dest_image),
 584                                                              .format = dest_image->vk_format,
 585                                                              .subresourceRange = {
 586                                                              .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 587                                                              .baseMipLevel = region->dstSubresource.mipLevel,
 588                                                              .levelCount = 1,
 589                                                              .baseArrayLayer = dest_base_layer + layer,
 590                                                              .layerCount = 1,
 591                                                      },
 592                                               }, NULL);
 593
 594                         VkFramebuffer fb_h;
 595                         radv_CreateFramebuffer(device_h,
 596                                                &(VkFramebufferCreateInfo) {
 597                                                        .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
 598                                                                .attachmentCount = 2,
 599                                                                .pAttachments = (VkImageView[]) {
 600                                                                radv_image_view_to_handle(&src_iview),
 601                                                                radv_image_view_to_handle(&dest_iview),
 602                                                        },
 603                                                        .width = radv_minify(dest_image->info.width,
 604                                                                             region->dstSubresource.mipLevel),
 605                                                        .height = radv_minify(dest_image->info.height,
 606                                                                               region->dstSubresource.mipLevel),
 607                                                        .layers = 1
 608                                                },
 609                                                &cmd_buffer->pool->alloc,
 610                                                &fb_h);
 611
 612                         radv_cmd_buffer_begin_render_pass(cmd_buffer,
 613                                                           &(VkRenderPassBeginInfo) {
 614                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 615                                                                 .renderPass = device->meta_state.resolve.pass[fs_key],
 616                                                                 .framebuffer = fb_h,
 617                                                                 .renderArea = {
 618                                                                         .offset = {
 619                                                                                 dstOffset.x,
 620                                                                                 dstOffset.y,
 621                                                                         },
 622                                                                         .extent = {
 623                                                                                 extent.width,
 624                                                                                 extent.height,
 625                                                                       }
 626                                                                 },
 627                                                                 .clearValueCount = 0,
 628                                                                 .pClearValues = NULL,
 629                                                       });
 630
 631                         radv_cmd_buffer_set_subpass(cmd_buffer,
 632                                                     &cmd_buffer->state.pass->subpasses[0]);
 633
 634                         emit_resolve(cmd_buffer,
 635                                      dest_iview.vk_format,
 636                                      &(VkOffset2D) {
 637                                              .x = dstOffset.x,
 638                                              .y = dstOffset.y,
 639                                      },
 640                                      &(VkExtent2D) {
 641                                              .width = extent.width,
 642                                              .height = extent.height,
 643                                      });
 644
 645                         radv_cmd_buffer_end_render_pass(cmd_buffer);
 646
 647                         radv_DestroyFramebuffer(device_h, fb_h,
 648                                                 &cmd_buffer->pool->alloc);
 649                 }
 650         }
 651
 652         radv_meta_restore(&saved_state, cmd_buffer);
 653 }
 654
 655 /**
 656  * Emit any needed resolves for the current subpass.
 657  */
 658 void
 659 radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer)
 660 {
 661         struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
 662         const struct radv_subpass *subpass = cmd_buffer->state.subpass;
 663         struct radv_meta_saved_state saved_state;
 664         enum radv_resolve_method resolve_method = RESOLVE_HW;
 665
 666         if (subpass->ds_resolve_attachment) {
 667                 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
 668                 struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment;
 669                 struct radv_image_view *src_iview =
 670                         cmd_buffer->state.attachments[src_att.attachment].iview;
 671                 struct radv_image_view *dst_iview =
 672                         cmd_buffer->state.attachments[dst_att.attachment].iview;
 673
 674                 /* Make sure to not clear the depth/stencil attachment after resolves. */
 675                 cmd_buffer->state.attachments[dst_att.attachment].pending_clear_aspects = 0;
 676
 677                 radv_pick_resolve_method_images(cmd_buffer->device,
 678                                                 src_iview->image,
 679                                                 src_iview->vk_format,
 680                                                 dst_iview->image,
 681                                                 dst_att.layout,
 682                                                 dst_att.in_render_loop,
 683                                                 cmd_buffer,
 684                                                 &resolve_method);
 685
 686                 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) &&
 687                     subpass->depth_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
 688                         if (resolve_method == RESOLVE_FRAGMENT) {
 689                                 radv_depth_stencil_resolve_subpass_fs(cmd_buffer,
 690                                                                       VK_IMAGE_ASPECT_DEPTH_BIT,
 691                                                                       subpass->depth_resolve_mode);
 692                         } else {
 693                                 assert(resolve_method == RESOLVE_COMPUTE);
 694                                 radv_depth_stencil_resolve_subpass_cs(cmd_buffer,
 695                                                                       VK_IMAGE_ASPECT_DEPTH_BIT,
 696                                                                       subpass->depth_resolve_mode);
 697                         }
 698                 }
 699
 700                 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) &&
 701                     subpass->stencil_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
 702                         if (resolve_method == RESOLVE_FRAGMENT) {
 703                                 radv_depth_stencil_resolve_subpass_fs(cmd_buffer,
 704                                                                       VK_IMAGE_ASPECT_STENCIL_BIT,
 705                                                                       subpass->stencil_resolve_mode);
 706                         } else {
 707                                 assert(resolve_method == RESOLVE_COMPUTE);
 708                                 radv_depth_stencil_resolve_subpass_cs(cmd_buffer,
 709                                                                       VK_IMAGE_ASPECT_STENCIL_BIT,
 710                                                                       subpass->stencil_resolve_mode);
 711                         }
 712                 }
 713         }
 714
 715         if (!subpass->has_color_resolve)
 716                 return;
 717
 718         for (uint32_t i = 0; i < subpass->color_count; ++i) {
 719                 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
 720                 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
 721
 722                 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
 723                         continue;
 724
 725                 /* Make sure to not clear color attachments after resolves. */
 726                 cmd_buffer->state.attachments[dest_att.attachment].pending_clear_aspects = 0;
 727
 728                 struct radv_image *dst_img = cmd_buffer->state.attachments[dest_att.attachment].iview->image;
 729                 struct radv_image_view *src_iview= cmd_buffer->state.attachments[src_att.attachment].iview;
 730                 struct radv_image *src_img = src_iview->image;
 731
 732                 radv_pick_resolve_method_images(cmd_buffer->device, src_img,
 733                                                 src_iview->vk_format, dst_img,
 734                                                 dest_att.layout,
 735                                                 dest_att.in_render_loop,
 736                                                 cmd_buffer, &resolve_method);
 737
 738                 if (resolve_method == RESOLVE_FRAGMENT) {
 739                         break;
 740                 }
 741         }
 742
 743         if (resolve_method == RESOLVE_COMPUTE) {
 744                 radv_cmd_buffer_resolve_subpass_cs(cmd_buffer);
 745                 return;
 746         } else if (resolve_method == RESOLVE_FRAGMENT) {
 747                 radv_cmd_buffer_resolve_subpass_fs(cmd_buffer);
 748                 return;
 749         }
 750
 751         radv_meta_save(&saved_state, cmd_buffer,
 752                        RADV_META_SAVE_GRAPHICS_PIPELINE);
 753
 754         for (uint32_t i = 0; i < subpass->color_count; ++i) {
 755                 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
 756                 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
 757
 758                 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
 759                         continue;
 760
 761                 struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview;
 762                 struct radv_image *dst_img = dest_iview->image;
 763
 764                 if (radv_dcc_enabled(dst_img, dest_iview->base_mip)) {
 765                         VkImageSubresourceRange range = {
 766                                 .aspectMask = dest_iview->aspect_mask,
 767                                 .baseMipLevel = dest_iview->base_mip,
 768                                 .levelCount = dest_iview->level_count,
 769                                 .baseArrayLayer = dest_iview->base_layer,
 770                                 .layerCount = dest_iview->layer_count,
 771                         };
 772
 773                         radv_initialize_dcc(cmd_buffer, dst_img, &range, 0xffffffff);
 774                         cmd_buffer->state.attachments[dest_att.attachment].current_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 775                 }
 776
 777                 struct radv_subpass resolve_subpass = {
 778                         .color_count = 2,
 779                         .color_attachments = (struct radv_subpass_attachment[]) { src_att, dest_att },
 780                         .depth_stencil_attachment = NULL,
 781                 };
 782
 783                 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
 784
 785                 VkResult ret = build_resolve_pipeline(cmd_buffer->device, radv_format_meta_fs_key(dest_iview->vk_format));
 786                 if (ret != VK_SUCCESS) {
 787                         cmd_buffer->record_result = ret;
 788                         continue;
 789                 }
 790
 791                 emit_resolve(cmd_buffer,
 792                              dest_iview->vk_format,
 793                              &(VkOffset2D) { 0, 0 },
 794                              &(VkExtent2D) { fb->width, fb->height });
 795         }
 796
 797         radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
 798
 799         radv_meta_restore(&saved_state, cmd_buffer);
 800 }
 801
 802 /**
 803  * Decompress CMask/FMask before resolving a multisampled source image inside a
 804  * subpass.
 805  */
 806 void
 807 radv_decompress_resolve_subpass_src(struct radv_cmd_buffer *cmd_buffer)
 808 {
 809         const struct radv_subpass *subpass = cmd_buffer->state.subpass;
 810         struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
 811         uint32_t layer_count = fb->layers;
 812
 813         if (subpass->view_mask)
 814                 layer_count = util_last_bit(subpass->view_mask);
 815
 816         for (uint32_t i = 0; i < subpass->color_count; ++i) {
 817                 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
 818                 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
 819
 820                 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
 821                         continue;
 822
 823                 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
 824                 struct radv_image *src_image = src_iview->image;
 825
 826                 VkImageResolve region = {};
 827                 region.srcSubresource.aspectMask = src_iview->aspect_mask;
 828                 region.srcSubresource.mipLevel = 0;
 829                 region.srcSubresource.baseArrayLayer = src_iview->base_layer;
 830                 region.srcSubresource.layerCount = layer_count;
 831
 832                 radv_decompress_resolve_src(cmd_buffer, src_image,
 833                                             src_att.layout, 1, &region);
 834         }
 835
 836         if (subpass->ds_resolve_attachment) {
 837                 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
 838                 struct radv_image_view *src_iview = fb->attachments[src_att.attachment];
 839                 struct radv_image *src_image = src_iview->image;
 840
 841                 VkImageResolve region = {};
 842                 region.srcSubresource.aspectMask = src_iview->aspect_mask;
 843                 region.srcSubresource.mipLevel = 0;
 844                 region.srcSubresource.baseArrayLayer = src_iview->base_layer;
 845                 region.srcSubresource.layerCount = layer_count;
 846
 847                 radv_decompress_resolve_src(cmd_buffer, src_image,
 848                                             src_att.layout, 1, &region);
 849         }
 850 }
 851
 852 static struct radv_sample_locations_state *
 853 radv_get_resolve_sample_locations(struct radv_cmd_buffer *cmd_buffer)
 854 {
 855         struct radv_cmd_state *state = &cmd_buffer->state;
 856         uint32_t subpass_id = radv_get_subpass_id(cmd_buffer);
 857
 858         for (uint32_t i = 0; i < state->num_subpass_sample_locs; i++) {
 859                 if (state->subpass_sample_locs[i].subpass_idx == subpass_id)
 860                         return &state->subpass_sample_locs[i].sample_location;
 861         }
 862
 863         return NULL;
 864 }
 865
 866 /**
 867  * Decompress CMask/FMask before resolving a multisampled source image.
 868  */
 869 void
 870 radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer,
 871                             struct radv_image *src_image,
 872                             VkImageLayout src_image_layout,
 873                             uint32_t region_count,
 874                             const VkImageResolve *regions)
 875 {
 876         for (uint32_t r = 0; r < region_count; ++r) {
 877                 const VkImageResolve *region = &regions[r];
 878                 const uint32_t src_base_layer =
 879                         radv_meta_get_iview_layer(src_image, &region->srcSubresource,
 880                                                   &region->srcOffset);
 881
 882                 VkImageMemoryBarrier barrier = {};
 883                 barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
 884                 barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
 885                 barrier.oldLayout = src_image_layout;
 886                 barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
 887                 barrier.image = radv_image_to_handle(src_image);
 888                 barrier.subresourceRange = (VkImageSubresourceRange) {
 889                         .aspectMask = region->srcSubresource.aspectMask,
 890                         .baseMipLevel = region->srcSubresource.mipLevel,
 891                         .levelCount = 1,
 892                         .baseArrayLayer = src_base_layer,
 893                         .layerCount = region->srcSubresource.layerCount,
 894                 };
 895
 896                 if (src_image->flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT) {
 897                         /* If the depth/stencil image uses different sample
 898                          * locations, we need them during HTILE decompressions.
 899                          */
 900                         struct radv_sample_locations_state *sample_locs =
 901                                 radv_get_resolve_sample_locations(cmd_buffer);
 902
 903                         barrier.pNext = &(VkSampleLocationsInfoEXT) {
 904                                 .sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT,
 905                                 .sampleLocationsPerPixel = sample_locs->per_pixel,
 906                                 .sampleLocationGridSize = sample_locs->grid_size,
 907                                 .sampleLocationsCount = sample_locs->count,
 908                                 .pSampleLocations = sample_locs->locations,
 909                         };
 910                 }
 911
 912                 radv_CmdPipelineBarrier(radv_cmd_buffer_to_handle(cmd_buffer),
 913                                         VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 914                                         VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 915                                         false, 0, NULL, 0, NULL, 1, &barrier);
 916         }
 917 }