src/amd/vulkan/radv_meta_resolve_fs.c

   1 /*
   2  * Copyright © 2016 Dave Airlie
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24
  25 #include <assert.h>
  26 #include <stdbool.h>
  27
  28 #include "radv_meta.h"
  29 #include "radv_private.h"
  30 #include "nir/nir_builder.h"
  31 #include "sid.h"
  32 #include "vk_format.h"
  33
  34 static nir_shader *
  35 build_nir_vertex_shader(void)
  36 {
  37         const struct glsl_type *vec4 = glsl_vec4_type();
  38         nir_builder b;
  39
  40         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
  41         b.shader->info.name = ralloc_strdup(b.shader, "meta_resolve_vs");
  42
  43         nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
  44                                                     vec4, "gl_Position");
  45         pos_out->data.location = VARYING_SLOT_POS;
  46
  47         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
  48
  49         nir_store_var(&b, pos_out, outvec, 0xf);
  50         return b.shader;
  51 }
  52
  53 static nir_shader *
  54 build_resolve_fragment_shader(struct radv_device *dev, bool is_integer, int samples)
  55 {
  56         nir_builder b;
  57         char name[64];
  58         const struct glsl_type *vec4 = glsl_vec4_type();
  59         const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_MS,
  60                                                                  false,
  61                                                                  false,
  62                                                                  GLSL_TYPE_FLOAT);
  63
  64         snprintf(name, 64, "meta_resolve_fs-%d-%s", samples, is_integer ? "int" : "float");
  65         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
  66         b.shader->info.name = ralloc_strdup(b.shader, name);
  67
  68         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
  69                                                       sampler_type, "s_tex");
  70         input_img->data.descriptor_set = 0;
  71         input_img->data.binding = 0;
  72
  73         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
  74                                                       vec4, "f_color");
  75         color_out->data.location = FRAG_RESULT_DATA0;
  76
  77         nir_ssa_def *pos_in = nir_channels(&b, nir_load_frag_coord(&b), 0x3);
  78         nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  79         nir_intrinsic_set_base(src_offset, 0);
  80         nir_intrinsic_set_range(src_offset, 8);
  81         src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
  82         src_offset->num_components = 2;
  83         nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
  84         nir_builder_instr_insert(&b, &src_offset->instr);
  85
  86         nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
  87
  88         nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, &src_offset->dest.ssa), 0x3);
  89         nir_variable *color = nir_local_variable_create(b.impl, glsl_vec4_type(), "color");
  90
  91         radv_meta_build_resolve_shader_core(&b, is_integer, samples, input_img,
  92                                             color, img_coord);
  93
  94         nir_ssa_def *outval = nir_load_var(&b, color);
  95         nir_store_var(&b, color_out, outval, 0xf);
  96         return b.shader;
  97 }
  98
  99
 100 static VkResult
 101 create_layout(struct radv_device *device)
 102 {
 103         VkResult result;
 104         /*
 105          * one descriptors for the image being sampled
 106          */
 107         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 108                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 109                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 110                 .bindingCount = 1,
 111                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 112                         {
 113                                 .binding = 0,
 114                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 115                                 .descriptorCount = 1,
 116                                 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
 117                                 .pImmutableSamplers = NULL
 118                         },
 119                 }
 120         };
 121
 122         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 123                                                 &ds_create_info,
 124                                                 &device->meta_state.alloc,
 125                                                 &device->meta_state.resolve_fragment.ds_layout);
 126         if (result != VK_SUCCESS)
 127                 goto fail;
 128
 129
 130         VkPipelineLayoutCreateInfo pl_create_info = {
 131                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 132                 .setLayoutCount = 1,
 133                 .pSetLayouts = &device->meta_state.resolve_fragment.ds_layout,
 134                 .pushConstantRangeCount = 1,
 135                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8},
 136         };
 137
 138         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 139                                           &pl_create_info,
 140                                           &device->meta_state.alloc,
 141                                           &device->meta_state.resolve_fragment.p_layout);
 142         if (result != VK_SUCCESS)
 143                 goto fail;
 144         return VK_SUCCESS;
 145 fail:
 146         return result;
 147 }
 148
 149 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
 150         .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 151         .vertexBindingDescriptionCount = 0,
 152         .vertexAttributeDescriptionCount = 0,
 153 };
 154
 155 static VkResult
 156 create_resolve_pipeline(struct radv_device *device,
 157                         int samples_log2,
 158                         VkFormat format)
 159 {
 160         mtx_lock(&device->meta_state.mtx);
 161
 162         unsigned fs_key = radv_format_meta_fs_key(format);
 163         VkPipeline *pipeline = &device->meta_state.resolve_fragment.rc[samples_log2].pipeline[fs_key];
 164         if (*pipeline) {
 165                 mtx_unlock(&device->meta_state.mtx);
 166                 return VK_SUCCESS;
 167         }
 168
 169         VkResult result;
 170         bool is_integer = false;
 171         uint32_t samples = 1 << samples_log2;
 172         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 173         vi_create_info = &normal_vi_create_info;
 174         if (vk_format_is_int(format))
 175                 is_integer = true;
 176
 177         struct radv_shader_module fs = { .nir = NULL };
 178         fs.nir = build_resolve_fragment_shader(device, is_integer, samples);
 179         struct radv_shader_module vs = {
 180                 .nir = build_nir_vertex_shader(),
 181         };
 182
 183         VkRenderPass *rp = &device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][0];
 184
 185         assert(!*rp);
 186
 187         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 188                 {
 189                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 190                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 191                         .module = radv_shader_module_to_handle(&vs),
 192                         .pName = "main",
 193                         .pSpecializationInfo = NULL
 194                 }, {
 195                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 196                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 197                         .module = radv_shader_module_to_handle(&fs),
 198                         .pName = "main",
 199                         .pSpecializationInfo = NULL
 200                 },
 201         };
 202
 203
 204         for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
 205                 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
 206                 result = radv_CreateRenderPass(radv_device_to_handle(device),
 207                                         &(VkRenderPassCreateInfo) {
 208                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 209                                                 .attachmentCount = 1,
 210                                                 .pAttachments = &(VkAttachmentDescription) {
 211                                                         .format = format,
 212                                                         .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 213                                                         .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 214                                                         .initialLayout = layout,
 215                                                         .finalLayout = layout,
 216                                                 },
 217                                                 .subpassCount = 1,
 218                                                 .pSubpasses = &(VkSubpassDescription) {
 219                                                         .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 220                                                         .inputAttachmentCount = 0,
 221                                                         .colorAttachmentCount = 1,
 222                                                         .pColorAttachments = &(VkAttachmentReference) {
 223                                                                 .attachment = 0,
 224                                                                 .layout = layout,
 225                                                         },
 226                                                 .pResolveAttachments = NULL,
 227                                                 .pDepthStencilAttachment = &(VkAttachmentReference) {
 228                                                         .attachment = VK_ATTACHMENT_UNUSED,
 229                                                         .layout = VK_IMAGE_LAYOUT_GENERAL,
 230                                                 },
 231                                                 .preserveAttachmentCount = 0,
 232                                                 .pPreserveAttachments = NULL,
 233                                         },
 234                                         .dependencyCount = 0,
 235                                 }, &device->meta_state.alloc, rp + dst_layout);
 236         }
 237
 238
 239         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 240                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 241                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 242                 .pStages = pipeline_shader_stages,
 243                 .pVertexInputState = vi_create_info,
 244                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 245                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 246                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 247                         .primitiveRestartEnable = false,
 248                 },
 249                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 250                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 251                         .viewportCount = 1,
 252                         .scissorCount = 1,
 253                 },
 254                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 255                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 256                         .rasterizerDiscardEnable = false,
 257                         .polygonMode = VK_POLYGON_MODE_FILL,
 258                         .cullMode = VK_CULL_MODE_NONE,
 259                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
 260                 },
 261                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
 262                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 263                         .rasterizationSamples = 1,
 264                         .sampleShadingEnable = false,
 265                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
 266                 },
 267                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 268                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 269                         .attachmentCount = 1,
 270                         .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 271                                 { .colorWriteMask =
 272                                   VK_COLOR_COMPONENT_A_BIT |
 273                                   VK_COLOR_COMPONENT_R_BIT |
 274                                   VK_COLOR_COMPONENT_G_BIT |
 275                                   VK_COLOR_COMPONENT_B_BIT },
 276                         }
 277                 },
 278                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 279                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 280                         .dynamicStateCount = 9,
 281                         .pDynamicStates = (VkDynamicState[]) {
 282                                 VK_DYNAMIC_STATE_VIEWPORT,
 283                                 VK_DYNAMIC_STATE_SCISSOR,
 284                                 VK_DYNAMIC_STATE_LINE_WIDTH,
 285                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
 286                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
 287                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
 288                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
 289                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
 290                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
 291                         },
 292                 },
 293                 .flags = 0,
 294                 .layout = device->meta_state.resolve_fragment.p_layout,
 295                 .renderPass = *rp,
 296                 .subpass = 0,
 297         };
 298
 299         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
 300                 .use_rectlist = true
 301         };
 302
 303         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
 304                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 305                                                &vk_pipeline_info, &radv_pipeline_info,
 306                                                &device->meta_state.alloc,
 307                                                pipeline);
 308         ralloc_free(vs.nir);
 309         ralloc_free(fs.nir);
 310
 311         mtx_unlock(&device->meta_state.mtx);
 312         return result;
 313 }
 314
 315 enum {
 316         DEPTH_RESOLVE,
 317         STENCIL_RESOLVE
 318 };
 319
 320 static const char *
 321 get_resolve_mode_str(VkResolveModeFlagBitsKHR resolve_mode)
 322 {
 323         switch (resolve_mode) {
 324         case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
 325                 return "zero";
 326         case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
 327                 return "average";
 328         case VK_RESOLVE_MODE_MIN_BIT_KHR:
 329                 return "min";
 330         case VK_RESOLVE_MODE_MAX_BIT_KHR:
 331                 return "max";
 332         default:
 333                 unreachable("invalid resolve mode");
 334         }
 335 }
 336
 337 static nir_shader *
 338 build_depth_stencil_resolve_fragment_shader(struct radv_device *dev, int samples,
 339                                             int index,
 340                                             VkResolveModeFlagBitsKHR resolve_mode)
 341 {
 342         nir_builder b;
 343         char name[64];
 344         const struct glsl_type *vec4 = glsl_vec4_type();
 345         const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 346                                                                  false,
 347                                                                  false,
 348                                                                  GLSL_TYPE_FLOAT);
 349
 350         snprintf(name, 64, "meta_resolve_fs_%s-%s-%d",
 351                  index == DEPTH_RESOLVE ? "depth" : "stencil",
 352                  get_resolve_mode_str(resolve_mode), samples);
 353
 354         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 355         b.shader->info.name = ralloc_strdup(b.shader, name);
 356
 357         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 358                                                       sampler_type, "s_tex");
 359         input_img->data.descriptor_set = 0;
 360         input_img->data.binding = 0;
 361
 362         nir_variable *fs_out = nir_variable_create(b.shader,
 363                                                    nir_var_shader_out, vec4,
 364                                                    "f_out");
 365         fs_out->data.location =
 366                 index == DEPTH_RESOLVE ? FRAG_RESULT_DEPTH : FRAG_RESULT_STENCIL;
 367
 368         nir_ssa_def *pos_in = nir_channels(&b, nir_load_frag_coord(&b), 0x3);
 369
 370         nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 371         nir_intrinsic_set_base(src_offset, 0);
 372         nir_intrinsic_set_range(src_offset, 8);
 373         src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 374         src_offset->num_components = 2;
 375         nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
 376         nir_builder_instr_insert(&b, &src_offset->instr);
 377
 378         nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
 379
 380         nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, &src_offset->dest.ssa), 0x3);
 381
 382         nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
 383
 384         nir_alu_type type = index == DEPTH_RESOLVE ? nir_type_float : nir_type_uint;
 385
 386         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
 387         tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
 388         tex->op = nir_texop_txf_ms;
 389         tex->src[0].src_type = nir_tex_src_coord;
 390         tex->src[0].src = nir_src_for_ssa(img_coord);
 391         tex->src[1].src_type = nir_tex_src_ms_index;
 392         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 393         tex->src[2].src_type = nir_tex_src_texture_deref;
 394         tex->src[2].src = nir_src_for_ssa(input_img_deref);
 395         tex->dest_type = type;
 396         tex->is_array = false;
 397         tex->coord_components = 2;
 398
 399         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 400         nir_builder_instr_insert(&b, &tex->instr);
 401
 402         nir_ssa_def *outval = &tex->dest.ssa;
 403
 404         if (resolve_mode != VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR) {
 405                 for (int i = 1; i < samples; i++) {
 406                         nir_tex_instr *tex_add = nir_tex_instr_create(b.shader, 3);
 407                         tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
 408                         tex_add->op = nir_texop_txf_ms;
 409                         tex_add->src[0].src_type = nir_tex_src_coord;
 410                         tex_add->src[0].src = nir_src_for_ssa(img_coord);
 411                         tex_add->src[1].src_type = nir_tex_src_ms_index;
 412                         tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(&b, i));
 413                         tex_add->src[2].src_type = nir_tex_src_texture_deref;
 414                         tex_add->src[2].src = nir_src_for_ssa(input_img_deref);
 415                         tex_add->dest_type = type;
 416                         tex_add->is_array = false;
 417                         tex_add->coord_components = 2;
 418
 419                         nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
 420                         nir_builder_instr_insert(&b, &tex_add->instr);
 421
 422                         switch (resolve_mode) {
 423                         case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
 424                                 assert(index == DEPTH_RESOLVE);
 425                                 outval = nir_fadd(&b, outval, &tex_add->dest.ssa);
 426                                 break;
 427                         case VK_RESOLVE_MODE_MIN_BIT_KHR:
 428                                 if (index == DEPTH_RESOLVE)
 429                                         outval = nir_fmin(&b, outval, &tex_add->dest.ssa);
 430                                 else
 431                                         outval = nir_umin(&b, outval, &tex_add->dest.ssa);
 432                                 break;
 433                         case VK_RESOLVE_MODE_MAX_BIT_KHR:
 434                                 if (index == DEPTH_RESOLVE)
 435                                         outval = nir_fmax(&b, outval, &tex_add->dest.ssa);
 436                                 else
 437                                         outval = nir_umax(&b, outval, &tex_add->dest.ssa);
 438                                 break;
 439                         default:
 440                                 unreachable("invalid resolve mode");
 441                         }
 442                 }
 443
 444                 if (resolve_mode == VK_RESOLVE_MODE_AVERAGE_BIT_KHR)
 445                         outval = nir_fdiv(&b, outval, nir_imm_float(&b, samples));
 446         }
 447
 448         nir_store_var(&b, fs_out, outval, 0x1);
 449
 450         return b.shader;
 451 }
 452
 453 static VkResult
 454 create_depth_stencil_resolve_pipeline(struct radv_device *device,
 455                                       int samples_log2,
 456                                       int index,
 457                                       VkResolveModeFlagBitsKHR resolve_mode)
 458 {
 459         VkRenderPass *render_pass;
 460         VkPipeline *pipeline;
 461         VkFormat src_format;
 462         VkResult result;
 463
 464         mtx_lock(&device->meta_state.mtx);
 465
 466         switch (resolve_mode) {
 467         case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
 468                 if (index == DEPTH_RESOLVE)
 469                         pipeline = &device->meta_state.resolve_fragment.depth_zero_pipeline;
 470                 else
 471                         pipeline = &device->meta_state.resolve_fragment.stencil_zero_pipeline;
 472                 break;
 473         case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
 474                 assert(index == DEPTH_RESOLVE);
 475                 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].average_pipeline;
 476                 break;
 477         case VK_RESOLVE_MODE_MIN_BIT_KHR:
 478                 if (index == DEPTH_RESOLVE)
 479                         pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].min_pipeline;
 480                 else
 481                         pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].min_pipeline;
 482                 break;
 483         case VK_RESOLVE_MODE_MAX_BIT_KHR:
 484                 if (index == DEPTH_RESOLVE)
 485                         pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].max_pipeline;
 486                 else
 487                         pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].max_pipeline;
 488                 break;
 489         default:
 490                 unreachable("invalid resolve mode");
 491         }
 492
 493         if (*pipeline) {
 494                 mtx_unlock(&device->meta_state.mtx);
 495                 return VK_SUCCESS;
 496         }
 497
 498         struct radv_shader_module fs = { .nir = NULL };
 499         struct radv_shader_module vs = { .nir = NULL };
 500         uint32_t samples = 1 << samples_log2;
 501
 502         vs.nir = build_nir_vertex_shader();
 503         fs.nir = build_depth_stencil_resolve_fragment_shader(device, samples,
 504                                                              index, resolve_mode);
 505
 506         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 507                 {
 508                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 509                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 510                         .module = radv_shader_module_to_handle(&vs),
 511                         .pName = "main",
 512                         .pSpecializationInfo = NULL
 513                 }, {
 514                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 515                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 516                         .module = radv_shader_module_to_handle(&fs),
 517                         .pName = "main",
 518                         .pSpecializationInfo = NULL
 519                 },
 520         };
 521
 522         if (index == DEPTH_RESOLVE) {
 523                 src_format = VK_FORMAT_D32_SFLOAT;
 524                 render_pass = &device->meta_state.resolve_fragment.depth_render_pass;
 525         } else {
 526                 render_pass = &device->meta_state.resolve_fragment.stencil_render_pass;
 527                 src_format = VK_FORMAT_S8_UINT;
 528         }
 529
 530         if (!*render_pass) {
 531                 result = radv_CreateRenderPass(radv_device_to_handle(device),
 532                                                &(VkRenderPassCreateInfo) {
 533                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 534                                                         .attachmentCount = 1,
 535                                                         .pAttachments = &(VkAttachmentDescription) {
 536                                                                 .format = src_format,
 537                                                                 .loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
 538                                                                 .storeOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
 539                                                                 .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 540                                                                 .stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE,
 541                                                                 .initialLayout = VK_IMAGE_LAYOUT_GENERAL,
 542                                                                 .finalLayout = VK_IMAGE_LAYOUT_GENERAL,
 543                                                         },
 544                                                         .subpassCount = 1,
 545                                                         .pSubpasses = &(VkSubpassDescription) {
 546                                                                 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 547                                                                 .inputAttachmentCount = 0,
 548                                                                 .colorAttachmentCount = 0,
 549                                                                 .pColorAttachments = NULL,
 550                                                         .pResolveAttachments = NULL,
 551                                                         .pDepthStencilAttachment = &(VkAttachmentReference) {
 552                                                                 .attachment = 0,
 553                                                                 .layout = VK_IMAGE_LAYOUT_GENERAL,
 554                                                         },
 555                                                         .preserveAttachmentCount = 0,
 556                                                         .pPreserveAttachments = NULL,
 557                                                 },
 558                                                 .dependencyCount = 0,
 559                                         }, &device->meta_state.alloc, render_pass);
 560         }
 561
 562         VkStencilOp stencil_op =
 563                 index == DEPTH_RESOLVE ? VK_STENCIL_OP_KEEP : VK_STENCIL_OP_REPLACE;
 564
 565         VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {
 566                 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
 567                 .depthTestEnable = true,
 568                 .depthWriteEnable = index == DEPTH_RESOLVE,
 569                 .stencilTestEnable = index == STENCIL_RESOLVE,
 570                 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
 571                 .front = {
 572                         .failOp = stencil_op,
 573                         .passOp = stencil_op,
 574                         .depthFailOp = stencil_op,
 575                         .compareOp = VK_COMPARE_OP_ALWAYS,
 576                 },
 577                 .back = {
 578                         .failOp = stencil_op,
 579                         .passOp = stencil_op,
 580                         .depthFailOp = stencil_op,
 581                         .compareOp = VK_COMPARE_OP_ALWAYS,
 582                 }
 583         };
 584
 585         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 586         vi_create_info = &normal_vi_create_info;
 587
 588         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 589                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 590                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 591                 .pStages = pipeline_shader_stages,
 592                 .pVertexInputState = vi_create_info,
 593                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 594                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 595                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 596                         .primitiveRestartEnable = false,
 597                 },
 598                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 599                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 600                         .viewportCount = 1,
 601                         .scissorCount = 1,
 602                 },
 603                 .pDepthStencilState = &depth_stencil_state,
 604                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 605                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 606                         .rasterizerDiscardEnable = false,
 607                         .polygonMode = VK_POLYGON_MODE_FILL,
 608                         .cullMode = VK_CULL_MODE_NONE,
 609                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
 610                 },
 611                 .pMultisampleState = NULL,
 612                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 613                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 614                         .attachmentCount = 0,
 615                         .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 616                                 { .colorWriteMask =
 617                                   VK_COLOR_COMPONENT_A_BIT |
 618                                   VK_COLOR_COMPONENT_R_BIT |
 619                                   VK_COLOR_COMPONENT_G_BIT |
 620                                   VK_COLOR_COMPONENT_B_BIT },
 621                         }
 622                 },
 623                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 624                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 625                         .dynamicStateCount = 9,
 626                         .pDynamicStates = (VkDynamicState[]) {
 627                                 VK_DYNAMIC_STATE_VIEWPORT,
 628                                 VK_DYNAMIC_STATE_SCISSOR,
 629                                 VK_DYNAMIC_STATE_LINE_WIDTH,
 630                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
 631                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
 632                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
 633                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
 634                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
 635                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
 636                         },
 637                 },
 638                 .flags = 0,
 639                 .layout = device->meta_state.resolve_fragment.p_layout,
 640                 .renderPass = *render_pass,
 641                 .subpass = 0,
 642         };
 643
 644         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
 645                 .use_rectlist = true
 646         };
 647
 648         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
 649                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 650                                                &vk_pipeline_info, &radv_pipeline_info,
 651                                                &device->meta_state.alloc,
 652                                                pipeline);
 653
 654         ralloc_free(vs.nir);
 655         ralloc_free(fs.nir);
 656
 657         mtx_unlock(&device->meta_state.mtx);
 658         return result;
 659 }
 660
 661 VkResult
 662 radv_device_init_meta_resolve_fragment_state(struct radv_device *device, bool on_demand)
 663 {
 664         VkResult res;
 665
 666         res = create_layout(device);
 667         if (res != VK_SUCCESS)
 668                 goto fail;
 669
 670         if (on_demand)
 671                 return VK_SUCCESS;
 672
 673         for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
 674                 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 675                         res = create_resolve_pipeline(device, i, radv_fs_key_format_exemplars[j]);
 676                         if (res != VK_SUCCESS)
 677                                 goto fail;
 678                 }
 679
 680                 res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
 681                                                             VK_RESOLVE_MODE_AVERAGE_BIT_KHR);
 682                 if (res != VK_SUCCESS)
 683                         goto fail;
 684
 685                 res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
 686                                                             VK_RESOLVE_MODE_MIN_BIT_KHR);
 687                 if (res != VK_SUCCESS)
 688                         goto fail;
 689
 690                 res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
 691                                                             VK_RESOLVE_MODE_MAX_BIT_KHR);
 692                 if (res != VK_SUCCESS)
 693                         goto fail;
 694
 695                 res = create_depth_stencil_resolve_pipeline(device, i, STENCIL_RESOLVE,
 696                                                             VK_RESOLVE_MODE_MIN_BIT_KHR);
 697                 if (res != VK_SUCCESS)
 698                         goto fail;
 699
 700                 res = create_depth_stencil_resolve_pipeline(device, i, STENCIL_RESOLVE,
 701                                                             VK_RESOLVE_MODE_MAX_BIT_KHR);
 702                 if (res != VK_SUCCESS)
 703                         goto fail;
 704         }
 705
 706         res = create_depth_stencil_resolve_pipeline(device, 0, DEPTH_RESOLVE,
 707                                                     VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR);
 708         if (res != VK_SUCCESS)
 709                 goto fail;
 710
 711         res = create_depth_stencil_resolve_pipeline(device, 0, STENCIL_RESOLVE,
 712                                                     VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR);
 713         if (res != VK_SUCCESS)
 714                 goto fail;
 715
 716         return VK_SUCCESS;
 717 fail:
 718         radv_device_finish_meta_resolve_fragment_state(device);
 719         return res;
 720 }
 721
 722 void
 723 radv_device_finish_meta_resolve_fragment_state(struct radv_device *device)
 724 {
 725         struct radv_meta_state *state = &device->meta_state;
 726         for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
 727                 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 728                         for(unsigned k =0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
 729                                 radv_DestroyRenderPass(radv_device_to_handle(device),
 730                                                        state->resolve_fragment.rc[i].render_pass[j][k],
 731                                                        &state->alloc);
 732                         }
 733                         radv_DestroyPipeline(radv_device_to_handle(device),
 734                                              state->resolve_fragment.rc[i].pipeline[j],
 735                                              &state->alloc);
 736                 }
 737
 738                 radv_DestroyPipeline(radv_device_to_handle(device),
 739                                      state->resolve_fragment.depth[i].average_pipeline,
 740                                      &state->alloc);
 741
 742                 radv_DestroyPipeline(radv_device_to_handle(device),
 743                                      state->resolve_fragment.depth[i].max_pipeline,
 744                                      &state->alloc);
 745
 746                 radv_DestroyPipeline(radv_device_to_handle(device),
 747                                      state->resolve_fragment.depth[i].min_pipeline,
 748                                      &state->alloc);
 749
 750                 radv_DestroyPipeline(radv_device_to_handle(device),
 751                                      state->resolve_fragment.stencil[i].max_pipeline,
 752                                      &state->alloc);
 753
 754                 radv_DestroyPipeline(radv_device_to_handle(device),
 755                                      state->resolve_fragment.stencil[i].min_pipeline,
 756                                      &state->alloc);
 757         }
 758
 759         radv_DestroyRenderPass(radv_device_to_handle(device),
 760                                state->resolve_fragment.depth_render_pass,
 761                                &state->alloc);
 762         radv_DestroyRenderPass(radv_device_to_handle(device),
 763                                state->resolve_fragment.stencil_render_pass,
 764                                &state->alloc);
 765
 766         radv_DestroyPipeline(radv_device_to_handle(device),
 767                              state->resolve_fragment.depth_zero_pipeline,
 768                              &state->alloc);
 769         radv_DestroyPipeline(radv_device_to_handle(device),
 770                              state->resolve_fragment.stencil_zero_pipeline,
 771                              &state->alloc);
 772
 773         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 774                                         state->resolve_fragment.ds_layout,
 775                                         &state->alloc);
 776         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 777                                    state->resolve_fragment.p_layout,
 778                                    &state->alloc);
 779 }
 780
 781 static VkPipeline *
 782 radv_get_resolve_pipeline(struct radv_cmd_buffer *cmd_buffer,
 783                           struct radv_image_view *src_iview,
 784                           struct radv_image_view *dst_iview)
 785 {
 786         struct radv_device *device = cmd_buffer->device;
 787         unsigned fs_key = radv_format_meta_fs_key(dst_iview->vk_format);
 788         const uint32_t samples = src_iview->image->info.samples;
 789         const uint32_t samples_log2 = ffs(samples) - 1;
 790         VkPipeline *pipeline;
 791
 792         pipeline = &device->meta_state.resolve_fragment.rc[samples_log2].pipeline[fs_key];
 793         if (!*pipeline ) {
 794                 VkResult ret;
 795
 796                 ret = create_resolve_pipeline(device, samples_log2,
 797                                               radv_fs_key_format_exemplars[fs_key]);
 798                 if (ret != VK_SUCCESS) {
 799                         cmd_buffer->record_result = ret;
 800                         return NULL;
 801                 }
 802         }
 803
 804         return pipeline;
 805 }
 806
 807 static void
 808 emit_resolve(struct radv_cmd_buffer *cmd_buffer,
 809              struct radv_image_view *src_iview,
 810              struct radv_image_view *dest_iview,
 811              const VkOffset2D *src_offset,
 812              const VkOffset2D *dest_offset,
 813              const VkExtent2D *resolve_extent)
 814 {
 815         struct radv_device *device = cmd_buffer->device;
 816         VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
 817         VkPipeline *pipeline;
 818
 819         radv_meta_push_descriptor_set(cmd_buffer,
 820                                       VK_PIPELINE_BIND_POINT_GRAPHICS,
 821                                       cmd_buffer->device->meta_state.resolve_fragment.p_layout,
 822                                       0, /* set */
 823                                       1, /* descriptorWriteCount */
 824                                       (VkWriteDescriptorSet[]) {
 825                                               {
 826                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 827                                                       .dstBinding = 0,
 828                                                       .dstArrayElement = 0,
 829                                                       .descriptorCount = 1,
 830                                                       .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 831                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 832                                                       {
 833                                                       .sampler = VK_NULL_HANDLE,
 834                                                       .imageView = radv_image_view_to_handle(src_iview),
 835                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 836                                                       },
 837                                                       }
 838                                               },
 839                                       });
 840
 841         cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
 842
 843         unsigned push_constants[2] = {
 844                 src_offset->x - dest_offset->x,
 845                 src_offset->y - dest_offset->y,
 846         };
 847         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 848                               device->meta_state.resolve_fragment.p_layout,
 849                               VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8,
 850                               push_constants);
 851
 852         pipeline = radv_get_resolve_pipeline(cmd_buffer, src_iview, dest_iview);
 853
 854         radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
 855                              *pipeline);
 856
 857         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 858                 .x = dest_offset->x,
 859                 .y = dest_offset->y,
 860                 .width = resolve_extent->width,
 861                 .height = resolve_extent->height,
 862                 .minDepth = 0.0f,
 863                 .maxDepth = 1.0f
 864         });
 865
 866         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 867                 .offset = *dest_offset,
 868                 .extent = *resolve_extent,
 869         });
 870
 871         radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
 872         cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
 873 }
 874
 875 static void
 876 emit_depth_stencil_resolve(struct radv_cmd_buffer *cmd_buffer,
 877                            struct radv_image_view *src_iview,
 878                            struct radv_image_view *dst_iview,
 879                            const VkOffset2D *src_offset,
 880                            const VkOffset2D *dst_offset,
 881                            const VkExtent2D *resolve_extent,
 882                            VkImageAspectFlags aspects,
 883                            VkResolveModeFlagBitsKHR resolve_mode)
 884 {
 885         struct radv_device *device = cmd_buffer->device;
 886         const uint32_t samples = src_iview->image->info.samples;
 887         const uint32_t samples_log2 = ffs(samples) - 1;
 888         VkPipeline *pipeline;
 889
 890         radv_meta_push_descriptor_set(cmd_buffer,
 891                                       VK_PIPELINE_BIND_POINT_GRAPHICS,
 892                                       cmd_buffer->device->meta_state.resolve_fragment.p_layout,
 893                                       0, /* set */
 894                                       1, /* descriptorWriteCount */
 895                                       (VkWriteDescriptorSet[]) {
 896                                               {
 897                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 898                                                       .dstBinding = 0,
 899                                                       .dstArrayElement = 0,
 900                                                       .descriptorCount = 1,
 901                                                       .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 902                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 903                                                       {
 904                                                       .sampler = VK_NULL_HANDLE,
 905                                                       .imageView = radv_image_view_to_handle(src_iview),
 906                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 907                                                       },
 908                                                       }
 909                                               },
 910                                       });
 911
 912         unsigned push_constants[2] = {
 913                 src_offset->x - dst_offset->x,
 914                 src_offset->y - dst_offset->y,
 915         };
 916         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 917                               device->meta_state.resolve_fragment.p_layout,
 918                               VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8,
 919                               push_constants);
 920
 921         switch (resolve_mode) {
 922         case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
 923                 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
 924                         pipeline = &device->meta_state.resolve_fragment.depth_zero_pipeline;
 925                 else
 926                         pipeline = &device->meta_state.resolve_fragment.stencil_zero_pipeline;
 927                 break;
 928         case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
 929                 assert(aspects == VK_IMAGE_ASPECT_DEPTH_BIT);
 930                 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].average_pipeline;
 931                 break;
 932         case VK_RESOLVE_MODE_MIN_BIT_KHR:
 933                 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
 934                         pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].min_pipeline;
 935                 else
 936                         pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].min_pipeline;
 937                 break;
 938         case VK_RESOLVE_MODE_MAX_BIT_KHR:
 939                 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
 940                         pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].max_pipeline;
 941                 else
 942                         pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].max_pipeline;
 943                 break;
 944         default:
 945                 unreachable("invalid resolve mode");
 946         }
 947
 948         if (!*pipeline) {
 949                 int index = aspects == VK_IMAGE_ASPECT_DEPTH_BIT ? DEPTH_RESOLVE : STENCIL_RESOLVE;
 950                 VkResult ret;
 951
 952                 ret = create_depth_stencil_resolve_pipeline(device, samples_log2,
 953                                                             index, resolve_mode);
 954                 if (ret != VK_SUCCESS) {
 955                         cmd_buffer->record_result = ret;
 956                         return;
 957                 }
 958         }
 959
 960         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 961                              VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline);
 962
 963         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 964                 .x = dst_offset->x,
 965                 .y = dst_offset->y,
 966                 .width = resolve_extent->width,
 967                 .height = resolve_extent->height,
 968                 .minDepth = 0.0f,
 969                 .maxDepth = 1.0f
 970         });
 971
 972         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 973                 .offset = *dst_offset,
 974                 .extent = *resolve_extent,
 975         });
 976
 977         radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
 978 }
 979
 980 void radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer,
 981                                       struct radv_image *src_image,
 982                                       VkImageLayout src_image_layout,
 983                                       struct radv_image *dest_image,
 984                                       VkImageLayout dest_image_layout,
 985                                       uint32_t region_count,
 986                                       const VkImageResolve *regions)
 987 {
 988         struct radv_device *device = cmd_buffer->device;
 989         struct radv_meta_saved_state saved_state;
 990         const uint32_t samples = src_image->info.samples;
 991         const uint32_t samples_log2 = ffs(samples) - 1;
 992         unsigned fs_key = radv_format_meta_fs_key(dest_image->vk_format);
 993         unsigned dst_layout = radv_meta_dst_layout_from_layout(dest_image_layout);
 994         VkRenderPass rp;
 995
 996         radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout,
 997                                     region_count, regions);
 998
 999         if (!device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][dst_layout]) {
1000                 VkResult ret = create_resolve_pipeline(device, samples_log2, radv_fs_key_format_exemplars[fs_key]);
1001                 if (ret != VK_SUCCESS) {
1002                         cmd_buffer->record_result = ret;
1003                         return;
1004                 }
1005         }
1006
1007         rp = device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][dst_layout];
1008
1009         radv_meta_save(&saved_state, cmd_buffer,
1010                        RADV_META_SAVE_GRAPHICS_PIPELINE |
1011                        RADV_META_SAVE_CONSTANTS |
1012                        RADV_META_SAVE_DESCRIPTORS);
1013
1014         for (uint32_t r = 0; r < region_count; ++r) {
1015                 const VkImageResolve *region = &regions[r];
1016
1017                 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1018                 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1019                 assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount);
1020
1021                 const uint32_t src_base_layer =
1022                         radv_meta_get_iview_layer(src_image, &region->srcSubresource,
1023                                                   &region->srcOffset);
1024
1025                 const uint32_t dest_base_layer =
1026                         radv_meta_get_iview_layer(dest_image, &region->dstSubresource,
1027                                                   &region->dstOffset);
1028
1029                 const struct VkExtent3D extent =
1030                         radv_sanitize_image_extent(src_image->type, region->extent);
1031                 const struct VkOffset3D srcOffset =
1032                         radv_sanitize_image_offset(src_image->type, region->srcOffset);
1033                 const struct VkOffset3D dstOffset =
1034                         radv_sanitize_image_offset(dest_image->type, region->dstOffset);
1035
1036                 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount;
1037                      ++layer) {
1038
1039                         struct radv_image_view src_iview;
1040                         radv_image_view_init(&src_iview, cmd_buffer->device,
1041                                              &(VkImageViewCreateInfo) {
1042                                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1043                                                              .image = radv_image_to_handle(src_image),
1044                                                              .viewType = radv_meta_get_view_type(src_image),
1045                                                              .format = src_image->vk_format,
1046                                                              .subresourceRange = {
1047                                                              .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1048                                                              .baseMipLevel = region->srcSubresource.mipLevel,
1049                                                              .levelCount = 1,
1050                                                              .baseArrayLayer = src_base_layer + layer,
1051                                                              .layerCount = 1,
1052                                                      },
1053                                              });
1054
1055                         struct radv_image_view dest_iview;
1056                         radv_image_view_init(&dest_iview, cmd_buffer->device,
1057                                              &(VkImageViewCreateInfo) {
1058                                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1059                                                              .image = radv_image_to_handle(dest_image),
1060                                                              .viewType = radv_meta_get_view_type(dest_image),
1061                                                              .format = dest_image->vk_format,
1062                                                              .subresourceRange = {
1063                                                              .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1064                                                              .baseMipLevel = region->dstSubresource.mipLevel,
1065                                                              .levelCount = 1,
1066                                                              .baseArrayLayer = dest_base_layer + layer,
1067                                                              .layerCount = 1,
1068                                                      },
1069                                              });
1070
1071
1072                         VkFramebuffer fb;
1073                         radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
1074                                &(VkFramebufferCreateInfo) {
1075                                        .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
1076                                                .attachmentCount = 1,
1077                                                .pAttachments = (VkImageView[]) {
1078                                                radv_image_view_to_handle(&dest_iview),
1079                                        },
1080                                        .width = extent.width + dstOffset.x,
1081                                        .height = extent.height + dstOffset.y,
1082                                        .layers = 1
1083                                 }, &cmd_buffer->pool->alloc, &fb);
1084
1085                         radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
1086                                                 &(VkRenderPassBeginInfo) {
1087                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
1088                                                                 .renderPass = rp,
1089                                                                 .framebuffer = fb,
1090                                                                 .renderArea = {
1091                                                                 .offset = { dstOffset.x, dstOffset.y, },
1092                                                                 .extent = { extent.width, extent.height },
1093                                                         },
1094                                                                 .clearValueCount = 0,
1095                                                                 .pClearValues = NULL,
1096                                                 }, VK_SUBPASS_CONTENTS_INLINE);
1097
1098
1099
1100                         emit_resolve(cmd_buffer,
1101                                      &src_iview,
1102                                      &dest_iview,
1103                                      &(VkOffset2D) { srcOffset.x, srcOffset.y },
1104                                      &(VkOffset2D) { dstOffset.x, dstOffset.y },
1105                                      &(VkExtent2D) { extent.width, extent.height });
1106
1107                         radv_CmdEndRenderPass(radv_cmd_buffer_to_handle(cmd_buffer));
1108
1109                         radv_DestroyFramebuffer(radv_device_to_handle(cmd_buffer->device), fb, &cmd_buffer->pool->alloc);
1110                 }
1111         }
1112
1113         radv_meta_restore(&saved_state, cmd_buffer);
1114 }
1115
1116
1117 /**
1118  * Emit any needed resolves for the current subpass.
1119  */
1120 void
1121 radv_cmd_buffer_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer)
1122 {
1123         struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
1124         const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1125         struct radv_meta_saved_state saved_state;
1126         struct radv_subpass_barrier barrier;
1127
1128         /* Resolves happen before the end-of-subpass barriers get executed,
1129          * so we have to make the attachment shader-readable */
1130         barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
1131         barrier.src_access_mask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
1132         barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
1133         radv_subpass_barrier(cmd_buffer, &barrier);
1134
1135         radv_decompress_resolve_subpass_src(cmd_buffer);
1136
1137         radv_meta_save(&saved_state, cmd_buffer,
1138                        RADV_META_SAVE_GRAPHICS_PIPELINE |
1139                        RADV_META_SAVE_CONSTANTS |
1140                        RADV_META_SAVE_DESCRIPTORS);
1141
1142         for (uint32_t i = 0; i < subpass->color_count; ++i) {
1143                 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
1144                 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
1145
1146                 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
1147                         continue;
1148
1149                 struct radv_image_view *dest_iview = cmd_buffer->state.framebuffer->attachments[dest_att.attachment];
1150                 struct radv_image_view *src_iview = cmd_buffer->state.framebuffer->attachments[src_att.attachment];
1151
1152                 struct radv_subpass resolve_subpass = {
1153                         .color_count = 1,
1154                         .color_attachments = (struct radv_subpass_attachment[]) { dest_att },
1155                         .depth_stencil_attachment = NULL,
1156                 };
1157
1158                 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
1159
1160                 emit_resolve(cmd_buffer,
1161                              src_iview,
1162                              dest_iview,
1163                              &(VkOffset2D) { 0, 0 },
1164                              &(VkOffset2D) { 0, 0 },
1165                              &(VkExtent2D) { fb->width, fb->height });
1166         }
1167
1168         radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
1169
1170         radv_meta_restore(&saved_state, cmd_buffer);
1171 }
1172
1173 /**
1174  * Depth/stencil resolves for the current subpass.
1175  */
1176 void
1177 radv_depth_stencil_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer,
1178                                       VkImageAspectFlags aspects,
1179                                       VkResolveModeFlagBitsKHR resolve_mode)
1180 {
1181         struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
1182         const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1183         struct radv_meta_saved_state saved_state;
1184         struct radv_subpass_barrier barrier;
1185
1186         /* Resolves happen before the end-of-subpass barriers get executed,
1187          * so we have to make the attachment shader-readable */
1188         barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
1189         barrier.src_access_mask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
1190         barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
1191         radv_subpass_barrier(cmd_buffer, &barrier);
1192
1193         radv_decompress_resolve_subpass_src(cmd_buffer);
1194
1195         radv_meta_save(&saved_state, cmd_buffer,
1196                        RADV_META_SAVE_GRAPHICS_PIPELINE |
1197                        RADV_META_SAVE_CONSTANTS |
1198                        RADV_META_SAVE_DESCRIPTORS);
1199
1200         struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
1201         struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment;
1202
1203         struct radv_image_view *src_iview =
1204                 cmd_buffer->state.framebuffer->attachments[src_att.attachment];
1205         struct radv_image *src_image = src_iview->image;
1206         struct radv_image_view *dst_iview =
1207                 cmd_buffer->state.framebuffer->attachments[dst_att.attachment];
1208
1209         struct radv_subpass resolve_subpass = {
1210                 .color_count = 0,
1211                 .color_attachments = NULL,
1212                 .depth_stencil_attachment = (struct radv_subpass_attachment *) { &dst_att },
1213         };
1214
1215         radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
1216
1217         struct radv_image_view tsrc_iview;
1218         radv_image_view_init(&tsrc_iview, cmd_buffer->device,
1219                              &(VkImageViewCreateInfo) {
1220                                 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1221                                 .image = radv_image_to_handle(src_image),
1222                                 .viewType = radv_meta_get_view_type(src_image),
1223                                 .format = src_iview->vk_format,
1224                                 .subresourceRange = {
1225                                         .aspectMask = aspects,
1226                                         .baseMipLevel = 0,
1227                                         .levelCount = 1,
1228                                         .baseArrayLayer = 0,
1229                                         .layerCount = 1,
1230                                 },
1231                               });
1232
1233         emit_depth_stencil_resolve(cmd_buffer, &tsrc_iview, dst_iview,
1234                                    &(VkOffset2D) { 0, 0 },
1235                                    &(VkOffset2D) { 0, 0 },
1236                                    &(VkExtent2D) { fb->width, fb->height },
1237                                    aspects,
1238                                    resolve_mode);
1239
1240         radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
1241
1242         radv_meta_restore(&saved_state, cmd_buffer);
1243 }