src/amd/vulkan/radv_meta_blit2d.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  *
   4  * based on anv driver:
   5  * Copyright © 2016 Intel Corporation
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24  * IN THE SOFTWARE.
  25  */
  26
  27 #include "radv_meta.h"
  28 #include "nir/nir_builder.h"
  29 #include "vk_format.h"
  30
  31 enum blit2d_src_type {
  32         BLIT2D_SRC_TYPE_IMAGE,
  33         BLIT2D_SRC_TYPE_IMAGE_3D,
  34         BLIT2D_SRC_TYPE_BUFFER,
  35         BLIT2D_NUM_SRC_TYPES,
  36 };
  37
  38 static VkResult
  39 blit2d_init_color_pipeline(struct radv_device *device,
  40                            enum blit2d_src_type src_type,
  41                            VkFormat format,
  42                            uint32_t log2_samples);
  43
  44 static VkResult
  45 blit2d_init_depth_only_pipeline(struct radv_device *device,
  46                                 enum blit2d_src_type src_type,
  47                                 uint32_t log2_samples);
  48
  49 static VkResult
  50 blit2d_init_stencil_only_pipeline(struct radv_device *device,
  51                                   enum blit2d_src_type src_type,
  52                                   uint32_t log2_samples);
  53
  54 static void
  55 create_iview(struct radv_cmd_buffer *cmd_buffer,
  56              struct radv_meta_blit2d_surf *surf,
  57              struct radv_image_view *iview, VkFormat depth_format,
  58               VkImageAspectFlagBits aspects)
  59 {
  60         VkFormat format;
  61         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
  62                 radv_meta_get_view_type(surf->image);
  63
  64         if (depth_format)
  65                 format = depth_format;
  66         else
  67                 format = surf->format;
  68
  69         radv_image_view_init(iview, cmd_buffer->device,
  70                              &(VkImageViewCreateInfo) {
  71                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
  72                                              .image = radv_image_to_handle(surf->image),
  73                                              .viewType = view_type,
  74                                              .format = format,
  75                                              .subresourceRange = {
  76                                              .aspectMask = aspects,
  77                                              .baseMipLevel = surf->level,
  78                                              .levelCount = 1,
  79                                              .baseArrayLayer = surf->layer,
  80                                              .layerCount = 1
  81                                      },
  82                              }, NULL);
  83 }
  84
  85 static void
  86 create_bview(struct radv_cmd_buffer *cmd_buffer,
  87              struct radv_meta_blit2d_buffer *src,
  88              struct radv_buffer_view *bview, VkFormat depth_format)
  89 {
  90         VkFormat format;
  91
  92         if (depth_format)
  93                 format = depth_format;
  94         else
  95                 format = src->format;
  96         radv_buffer_view_init(bview, cmd_buffer->device,
  97                               &(VkBufferViewCreateInfo) {
  98                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
  99                                       .flags = 0,
 100                                       .buffer = radv_buffer_to_handle(src->buffer),
 101                                       .format = format,
 102                                       .offset = src->offset,
 103                                       .range = VK_WHOLE_SIZE,
 104                               });
 105
 106 }
 107
 108 struct blit2d_src_temps {
 109         struct radv_image_view iview;
 110         struct radv_buffer_view bview;
 111 };
 112
 113 static void
 114 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer,
 115                 struct radv_meta_blit2d_surf *src_img,
 116                 struct radv_meta_blit2d_buffer *src_buf,
 117                 struct blit2d_src_temps *tmp,
 118                 enum blit2d_src_type src_type, VkFormat depth_format,
 119                 VkImageAspectFlagBits aspects,
 120                 uint32_t log2_samples)
 121 {
 122         struct radv_device *device = cmd_buffer->device;
 123
 124         if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
 125                 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
 126
 127                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 128                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 129                                               0, /* set */
 130                                               1, /* descriptorWriteCount */
 131                                               (VkWriteDescriptorSet[]) {
 132                                                       {
 133                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 134                                                               .dstBinding = 0,
 135                                                               .dstArrayElement = 0,
 136                                                               .descriptorCount = 1,
 137                                                               .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
 138                                                               .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(&tmp->bview) }
 139                                                       }
 140                                               });
 141
 142                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 143                                       device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 144                                       VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 145                                       &src_buf->pitch);
 146         } else {
 147                 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
 148
 149                 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
 150                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 151                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 152                                               VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 153                                               &src_img->layer);
 154
 155                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 156                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 157                                               0, /* set */
 158                                               1, /* descriptorWriteCount */
 159                                               (VkWriteDescriptorSet[]) {
 160                                                       {
 161                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 162                                                               .dstBinding = 0,
 163                                                               .dstArrayElement = 0,
 164                                                               .descriptorCount = 1,
 165                                                               .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 166                                                               .pImageInfo = (VkDescriptorImageInfo[]) {
 167                                                                       {
 168                                                                               .sampler = VK_NULL_HANDLE,
 169                                                                               .imageView = radv_image_view_to_handle(&tmp->iview),
 170                                                                               .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 171                                                                       },
 172                                                               }
 173                                                       }
 174                                               });
 175         }
 176 }
 177
 178 struct blit2d_dst_temps {
 179         VkImage image;
 180         struct radv_image_view iview;
 181         VkFramebuffer fb;
 182 };
 183
 184 static void
 185 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer,
 186                 struct radv_meta_blit2d_surf *dst,
 187                 uint32_t width,
 188                 uint32_t height,
 189                 VkFormat depth_format,
 190                 struct blit2d_dst_temps *tmp,
 191                 VkImageAspectFlagBits aspects)
 192 {
 193         create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
 194
 195         radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
 196                                &(VkFramebufferCreateInfo) {
 197                                        .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
 198                                                .attachmentCount = 1,
 199                                                .pAttachments = (VkImageView[]) {
 200                                                radv_image_view_to_handle(&tmp->iview),
 201                                        },
 202                                        .width = width,
 203                                        .height = height,
 204                                        .layers = 1
 205                                 }, &cmd_buffer->pool->alloc, &tmp->fb);
 206 }
 207
 208 static void
 209 bind_pipeline(struct radv_cmd_buffer *cmd_buffer,
 210               enum blit2d_src_type src_type, unsigned fs_key,
 211               uint32_t log2_samples)
 212 {
 213         VkPipeline pipeline =
 214                 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
 215
 216         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 217                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 218 }
 219
 220 static void
 221 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer,
 222                     enum blit2d_src_type src_type,
 223                     uint32_t log2_samples)
 224 {
 225         VkPipeline pipeline =
 226                 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
 227
 228         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 229                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 230 }
 231
 232 static void
 233 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer,
 234                       enum blit2d_src_type src_type,
 235                       uint32_t log2_samples)
 236 {
 237         VkPipeline pipeline =
 238                 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
 239
 240         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 241                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 242 }
 243
 244 static void
 245 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
 246                             struct radv_meta_blit2d_surf *src_img,
 247                             struct radv_meta_blit2d_buffer *src_buf,
 248                             struct radv_meta_blit2d_surf *dst,
 249                             unsigned num_rects,
 250                             struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
 251                             uint32_t log2_samples)
 252 {
 253         struct radv_device *device = cmd_buffer->device;
 254
 255         for (unsigned r = 0; r < num_rects; ++r) {
 256                 unsigned i;
 257                 for_each_bit(i, dst->aspect_mask) {
 258                         unsigned aspect_mask = 1u << i;
 259                         unsigned src_aspect_mask = aspect_mask;
 260                         VkFormat depth_format = 0;
 261                         if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
 262                                 depth_format = vk_format_stencil_only(dst->image->vk_format);
 263                         else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
 264                                 depth_format = vk_format_depth_only(dst->image->vk_format);
 265                         else if (src_img)
 266                                 src_aspect_mask = src_img->aspect_mask;
 267
 268                         struct blit2d_src_temps src_temps;
 269                         blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format, src_aspect_mask, log2_samples);
 270
 271                         struct blit2d_dst_temps dst_temps;
 272                         blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
 273                                         rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
 274
 275                         float vertex_push_constants[4] = {
 276                                 rects[r].src_x,
 277                                 rects[r].src_y,
 278                                 rects[r].src_x + rects[r].width,
 279                                 rects[r].src_y + rects[r].height,
 280                         };
 281
 282                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 283                                         device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 284                                         VK_SHADER_STAGE_VERTEX_BIT, 0, 16,
 285                                         vertex_push_constants);
 286
 287                         if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT ||
 288                             aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
 289                             aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT ||
 290                             aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
 291                                 unsigned fs_key = radv_format_meta_fs_key(dst_temps.iview.vk_format);
 292                                 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
 293
 294                                 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] == VK_NULL_HANDLE) {
 295                                         VkResult ret = blit2d_init_color_pipeline(device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
 296                                         if (ret != VK_SUCCESS) {
 297                                                 cmd_buffer->record_result = ret;
 298                                                 goto fail_pipeline;
 299                                         }
 300                                 }
 301
 302                                 radv_cmd_buffer_begin_render_pass(cmd_buffer,
 303                                                                   &(VkRenderPassBeginInfo) {
 304                                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 305                                                                         .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
 306                                                                         .framebuffer = dst_temps.fb,
 307                                                                         .renderArea = {
 308                                                                                 .offset = { rects[r].dst_x, rects[r].dst_y, },
 309                                                                                 .extent = { rects[r].width, rects[r].height },
 310                                                                         },
 311                                                                         .clearValueCount = 0,
 312                                                                         .pClearValues = NULL,
 313                                                                   });
 314
 315                                 radv_cmd_buffer_set_subpass(cmd_buffer,
 316                                                             &cmd_buffer->state.pass->subpasses[0]);
 317
 318                                 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
 319                         } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 320                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 321
 322                                 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] == VK_NULL_HANDLE) {
 323                                         VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
 324                                         if (ret != VK_SUCCESS) {
 325                                                 cmd_buffer->record_result = ret;
 326                                                 goto fail_pipeline;
 327                                         }
 328                                 }
 329
 330                                 radv_cmd_buffer_begin_render_pass(cmd_buffer,
 331                                                                   &(VkRenderPassBeginInfo) {
 332                                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 333                                                                         .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
 334                                                                         .framebuffer = dst_temps.fb,
 335                                                                         .renderArea = {
 336                                                                                 .offset = { rects[r].dst_x, rects[r].dst_y, },
 337                                                                                 .extent = { rects[r].width, rects[r].height },
 338                                                                         },
 339                                                                         .clearValueCount = 0,
 340                                                                         .pClearValues = NULL,
 341                                                                   });
 342
 343                                 radv_cmd_buffer_set_subpass(cmd_buffer,
 344                                                             &cmd_buffer->state.pass->subpasses[0]);
 345
 346                                 bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
 347
 348                         } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 349                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 350
 351                                 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] == VK_NULL_HANDLE) {
 352                                         VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
 353                                         if (ret != VK_SUCCESS) {
 354                                                 cmd_buffer->record_result = ret;
 355                                                 goto fail_pipeline;
 356                                         }
 357                                 }
 358
 359                                 radv_cmd_buffer_begin_render_pass(cmd_buffer,
 360                                                                   &(VkRenderPassBeginInfo) {
 361                                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 362                                                                         .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
 363                                                                         .framebuffer = dst_temps.fb,
 364                                                                         .renderArea = {
 365                                                                                 .offset = { rects[r].dst_x, rects[r].dst_y, },
 366                                                                                 .extent = { rects[r].width, rects[r].height },
 367                                                                         },
 368                                                                         .clearValueCount = 0,
 369                                                                         .pClearValues = NULL,
 370                                                                    });
 371
 372                                 radv_cmd_buffer_set_subpass(cmd_buffer,
 373                                                             &cmd_buffer->state.pass->subpasses[0]);
 374
 375                                 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
 376                         } else
 377                                 unreachable("Processing blit2d with multiple aspects.");
 378
 379                         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 380                                 .x = rects[r].dst_x,
 381                                 .y = rects[r].dst_y,
 382                                 .width = rects[r].width,
 383                                 .height = rects[r].height,
 384                                 .minDepth = 0.0f,
 385                                 .maxDepth = 1.0f
 386                         });
 387
 388                         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 389                                 .offset = (VkOffset2D) { rects[r].dst_x, rects[r].dst_y },
 390                                 .extent = (VkExtent2D) { rects[r].width, rects[r].height },
 391                         });
 392
 393
 394
 395                         radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
 396                         radv_cmd_buffer_end_render_pass(cmd_buffer);
 397
 398 fail_pipeline:
 399                         /* At the point where we emit the draw call, all data from the
 400                         * descriptor sets, etc. has been used.  We are free to delete it.
 401                         */
 402                         radv_DestroyFramebuffer(radv_device_to_handle(device),
 403                                                 dst_temps.fb,
 404                                                 &cmd_buffer->pool->alloc);
 405                 }
 406         }
 407 }
 408
 409 void
 410 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer,
 411                  struct radv_meta_blit2d_surf *src_img,
 412                  struct radv_meta_blit2d_buffer *src_buf,
 413                  struct radv_meta_blit2d_surf *dst,
 414                  unsigned num_rects,
 415                  struct radv_meta_blit2d_rect *rects)
 416 {
 417         bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
 418                 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
 419         enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER :
 420                 use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D : BLIT2D_SRC_TYPE_IMAGE;
 421         radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst,
 422                                     num_rects, rects, src_type,
 423                                     src_img ? util_logbase2(src_img->image->info.samples) : 0);
 424 }
 425
 426 static nir_shader *
 427 build_nir_vertex_shader(void)
 428 {
 429         const struct glsl_type *vec4 = glsl_vec4_type();
 430         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 431         nir_builder b;
 432
 433         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 434         b.shader->info.name = ralloc_strdup(b.shader, "meta_blit2d_vs");
 435
 436         nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 437                                                     vec4, "gl_Position");
 438         pos_out->data.location = VARYING_SLOT_POS;
 439
 440         nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 441                                                         vec2, "v_tex_pos");
 442         tex_pos_out->data.location = VARYING_SLOT_VAR0;
 443         tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
 444
 445         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 446         nir_store_var(&b, pos_out, outvec, 0xf);
 447
 448         nir_intrinsic_instr *src_box = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 449         src_box->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 450         nir_intrinsic_set_base(src_box, 0);
 451         nir_intrinsic_set_range(src_box, 16);
 452         src_box->num_components = 4;
 453         nir_ssa_dest_init(&src_box->instr, &src_box->dest, 4, 32, "src_box");
 454         nir_builder_instr_insert(&b, &src_box->instr);
 455
 456         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_vertex_id_zero_base);
 457         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 458         nir_builder_instr_insert(&b, &vertex_id->instr);
 459
 460         /* vertex 0 - src_x, src_y */
 461         /* vertex 1 - src_x, src_y+h */
 462         /* vertex 2 - src_x+w, src_y */
 463         /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
 464            channel 1 is vertex id != 1 ? src_y : src_y + w */
 465
 466         nir_ssa_def *c0cmp = nir_ine(&b, &vertex_id->dest.ssa,
 467                                      nir_imm_int(&b, 2));
 468         nir_ssa_def *c1cmp = nir_ine(&b, &vertex_id->dest.ssa,
 469                                      nir_imm_int(&b, 1));
 470
 471         nir_ssa_def *comp[2];
 472         comp[0] = nir_bcsel(&b, c0cmp,
 473                             nir_channel(&b, &src_box->dest.ssa, 0),
 474                             nir_channel(&b, &src_box->dest.ssa, 2));
 475
 476         comp[1] = nir_bcsel(&b, c1cmp,
 477                             nir_channel(&b, &src_box->dest.ssa, 1),
 478                             nir_channel(&b, &src_box->dest.ssa, 3));
 479         nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
 480         nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
 481         return b.shader;
 482 }
 483
 484 typedef nir_ssa_def* (*texel_fetch_build_func)(struct nir_builder *,
 485                                                struct radv_device *,
 486                                                nir_ssa_def *, bool, bool);
 487
 488 static nir_ssa_def *
 489 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device,
 490                       nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 491 {
 492         enum glsl_sampler_dim dim =
 493                 is_3d ? GLSL_SAMPLER_DIM_3D : is_multisampled ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D;
 494         const struct glsl_type *sampler_type =
 495                 glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
 496         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 497                                                     sampler_type, "s_tex");
 498         sampler->data.descriptor_set = 0;
 499         sampler->data.binding = 0;
 500
 501         nir_ssa_def *tex_pos_3d = NULL;
 502         nir_intrinsic_instr *sample_idx = NULL;
 503         if (is_3d) {
 504                 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 505                 nir_intrinsic_set_base(layer, 16);
 506                 nir_intrinsic_set_range(layer, 4);
 507                 layer->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 508                 layer->num_components = 1;
 509                 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
 510                 nir_builder_instr_insert(b, &layer->instr);
 511
 512                 nir_ssa_def *chans[3];
 513                 chans[0] = nir_channel(b, tex_pos, 0);
 514                 chans[1] = nir_channel(b, tex_pos, 1);
 515                 chans[2] = &layer->dest.ssa;
 516                 tex_pos_3d = nir_vec(b, chans, 3);
 517         }
 518         if (is_multisampled) {
 519                 sample_idx = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_sample_id);
 520                 sample_idx->num_components = 1;
 521                 nir_ssa_dest_init(&sample_idx->instr, &sample_idx->dest, 1, 32, "sample_idx");
 522                 nir_builder_instr_insert(b, &sample_idx->instr);
 523         }
 524
 525         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 526
 527         nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
 528         tex->sampler_dim = dim;
 529         tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
 530         tex->src[0].src_type = nir_tex_src_coord;
 531         tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
 532         tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
 533         tex->src[1].src = nir_src_for_ssa(is_multisampled ? &sample_idx->dest.ssa : nir_imm_int(b, 0));
 534         tex->src[2].src_type = nir_tex_src_texture_deref;
 535         tex->src[2].src = nir_src_for_ssa(tex_deref);
 536         if (is_multisampled) {
 537                 tex->src[3].src_type = nir_tex_src_lod;
 538                 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
 539         }
 540         tex->dest_type = nir_type_uint;
 541         tex->is_array = false;
 542         tex->coord_components = is_3d ? 3 : 2;
 543
 544         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 545         nir_builder_instr_insert(b, &tex->instr);
 546
 547         return &tex->dest.ssa;
 548 }
 549
 550
 551 static nir_ssa_def *
 552 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device,
 553                        nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 554 {
 555         const struct glsl_type *sampler_type =
 556                 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
 557         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 558                                                     sampler_type, "s_tex");
 559         sampler->data.descriptor_set = 0;
 560         sampler->data.binding = 0;
 561
 562         nir_intrinsic_instr *width = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 563         nir_intrinsic_set_base(width, 16);
 564         nir_intrinsic_set_range(width, 4);
 565         width->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 566         width->num_components = 1;
 567         nir_ssa_dest_init(&width->instr, &width->dest, 1, 32, "width");
 568         nir_builder_instr_insert(b, &width->instr);
 569
 570         nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
 571         nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
 572         pos_y = nir_imul(b, pos_y, &width->dest.ssa);
 573         pos_x = nir_iadd(b, pos_x, pos_y);
 574
 575         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 576
 577         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
 578         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 579         tex->op = nir_texop_txf;
 580         tex->src[0].src_type = nir_tex_src_coord;
 581         tex->src[0].src = nir_src_for_ssa(pos_x);
 582         tex->src[1].src_type = nir_tex_src_texture_deref;
 583         tex->src[1].src = nir_src_for_ssa(tex_deref);
 584         tex->dest_type = nir_type_uint;
 585         tex->is_array = false;
 586         tex->coord_components = 1;
 587
 588         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 589         nir_builder_instr_insert(b, &tex->instr);
 590
 591         return &tex->dest.ssa;
 592 }
 593
 594 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
 595         .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 596         .vertexBindingDescriptionCount = 0,
 597         .vertexAttributeDescriptionCount = 0,
 598 };
 599
 600 static nir_shader *
 601 build_nir_copy_fragment_shader(struct radv_device *device,
 602                                texel_fetch_build_func txf_func, const char* name, bool is_3d,
 603                                bool is_multisampled)
 604 {
 605         const struct glsl_type *vec4 = glsl_vec4_type();
 606         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 607         nir_builder b;
 608
 609         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 610         b.shader->info.name = ralloc_strdup(b.shader, name);
 611
 612         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 613                                                        vec2, "v_tex_pos");
 614         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 615
 616         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 617                                                       vec4, "f_color");
 618         color_out->data.location = FRAG_RESULT_DATA0;
 619
 620         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 621         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 622
 623         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 624         nir_store_var(&b, color_out, color, 0xf);
 625
 626         return b.shader;
 627 }
 628
 629 static nir_shader *
 630 build_nir_copy_fragment_shader_depth(struct radv_device *device,
 631                                      texel_fetch_build_func txf_func, const char* name, bool is_3d,
 632                                      bool is_multisampled)
 633 {
 634         const struct glsl_type *vec4 = glsl_vec4_type();
 635         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 636         nir_builder b;
 637
 638         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 639         b.shader->info.name = ralloc_strdup(b.shader, name);
 640
 641         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 642                                                        vec2, "v_tex_pos");
 643         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 644
 645         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 646                                                       vec4, "f_color");
 647         color_out->data.location = FRAG_RESULT_DEPTH;
 648
 649         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 650         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 651
 652         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 653         nir_store_var(&b, color_out, color, 0x1);
 654
 655         return b.shader;
 656 }
 657
 658 static nir_shader *
 659 build_nir_copy_fragment_shader_stencil(struct radv_device *device,
 660                                        texel_fetch_build_func txf_func, const char* name, bool is_3d,
 661                                        bool is_multisampled)
 662 {
 663         const struct glsl_type *vec4 = glsl_vec4_type();
 664         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 665         nir_builder b;
 666
 667         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 668         b.shader->info.name = ralloc_strdup(b.shader, name);
 669
 670         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 671                                                        vec2, "v_tex_pos");
 672         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 673
 674         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 675                                                       vec4, "f_color");
 676         color_out->data.location = FRAG_RESULT_STENCIL;
 677
 678         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 679         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 680
 681         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 682         nir_store_var(&b, color_out, color, 0x1);
 683
 684         return b.shader;
 685 }
 686
 687 void
 688 radv_device_finish_meta_blit2d_state(struct radv_device *device)
 689 {
 690         struct radv_meta_state *state = &device->meta_state;
 691
 692         for(unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 693                 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
 694                         radv_DestroyRenderPass(radv_device_to_handle(device),
 695                                                state->blit2d_render_passes[j][k],
 696                                                &state->alloc);
 697                 }
 698         }
 699
 700         for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT; j++) {
 701                 radv_DestroyRenderPass(radv_device_to_handle(device),
 702                                        state->blit2d_depth_only_rp[j], &state->alloc);
 703                 radv_DestroyRenderPass(radv_device_to_handle(device),
 704                                        state->blit2d_stencil_only_rp[j], &state->alloc);
 705         }
 706
 707         for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; ++log2_samples) {
 708                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
 709                         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 710                                                    state->blit2d[log2_samples].p_layouts[src],
 711                                                    &state->alloc);
 712                         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 713                                                         state->blit2d[log2_samples].ds_layouts[src],
 714                                                         &state->alloc);
 715
 716                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 717                                 radv_DestroyPipeline(radv_device_to_handle(device),
 718                                                      state->blit2d[log2_samples].pipelines[src][j],
 719                                                      &state->alloc);
 720                         }
 721
 722                         radv_DestroyPipeline(radv_device_to_handle(device),
 723                                              state->blit2d[log2_samples].depth_only_pipeline[src],
 724                                              &state->alloc);
 725                         radv_DestroyPipeline(radv_device_to_handle(device),
 726                                              state->blit2d[log2_samples].stencil_only_pipeline[src],
 727                                              &state->alloc);
 728                 }
 729         }
 730 }
 731
 732 static VkResult
 733 blit2d_init_color_pipeline(struct radv_device *device,
 734                            enum blit2d_src_type src_type,
 735                            VkFormat format,
 736                            uint32_t log2_samples)
 737 {
 738         VkResult result;
 739         unsigned fs_key = radv_format_meta_fs_key(format);
 740         const char *name;
 741
 742         mtx_lock(&device->meta_state.mtx);
 743         if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
 744                 mtx_unlock(&device->meta_state.mtx);
 745                 return VK_SUCCESS;
 746         }
 747
 748         texel_fetch_build_func src_func;
 749         switch(src_type) {
 750         case BLIT2D_SRC_TYPE_IMAGE:
 751                 src_func = build_nir_texel_fetch;
 752                 name = "meta_blit2d_image_fs";
 753                 break;
 754         case BLIT2D_SRC_TYPE_IMAGE_3D:
 755                 src_func = build_nir_texel_fetch;
 756                 name = "meta_blit3d_image_fs";
 757                 break;
 758         case BLIT2D_SRC_TYPE_BUFFER:
 759                 src_func = build_nir_buffer_fetch;
 760                 name = "meta_blit2d_buffer_fs";
 761                 break;
 762         default:
 763                 unreachable("unknown blit src type\n");
 764                 break;
 765         }
 766
 767         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 768         struct radv_shader_module fs = { .nir = NULL };
 769
 770
 771         fs.nir = build_nir_copy_fragment_shader(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 772         vi_create_info = &normal_vi_create_info;
 773
 774         struct radv_shader_module vs = {
 775                 .nir = build_nir_vertex_shader(),
 776         };
 777
 778         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 779                 {
 780                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 781                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 782                         .module = radv_shader_module_to_handle(&vs),
 783                         .pName = "main",
 784                         .pSpecializationInfo = NULL
 785                 }, {
 786                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 787                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 788                         .module = radv_shader_module_to_handle(&fs),
 789                         .pName = "main",
 790                         .pSpecializationInfo = NULL
 791                 },
 792         };
 793
 794         for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
 795                 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
 796                         VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
 797
 798                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 799                                                 &(VkRenderPassCreateInfo) {
 800                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 801                                                         .attachmentCount = 1,
 802                                                         .pAttachments = &(VkAttachmentDescription) {
 803                                                         .format = format,
 804                                                         .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 805                                                         .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 806                                                         .initialLayout = layout,
 807                                                         .finalLayout = layout,
 808                                                         },
 809                                                 .subpassCount = 1,
 810                                                 .pSubpasses = &(VkSubpassDescription) {
 811                                                         .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 812                                                         .inputAttachmentCount = 0,
 813                                                         .colorAttachmentCount = 1,
 814                                                         .pColorAttachments = &(VkAttachmentReference) {
 815                                                                 .attachment = 0,
 816                                                                 .layout = layout,
 817                                                                 },
 818                                                 .pResolveAttachments = NULL,
 819                                                 .pDepthStencilAttachment = &(VkAttachmentReference) {
 820                                                         .attachment = VK_ATTACHMENT_UNUSED,
 821                                                         .layout = layout,
 822                                                 },
 823                                                 .preserveAttachmentCount = 0,
 824                                                 .pPreserveAttachments = NULL,
 825                                                 },
 826                                                 .dependencyCount = 2,
 827                                                 .pDependencies = (VkSubpassDependency[]) {
 828                                                         {
 829                                                                 .srcSubpass = VK_SUBPASS_EXTERNAL,
 830                                                                 .dstSubpass = 0,
 831                                                                 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 832                                                                 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 833                                                                 .srcAccessMask = 0,
 834                                                                 .dstAccessMask = 0,
 835                                                                 .dependencyFlags = 0
 836                                                         },
 837                                                         {
 838                                                                 .srcSubpass = 0,
 839                                                                 .dstSubpass = VK_SUBPASS_EXTERNAL,
 840                                                                 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 841                                                                 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 842                                                                 .srcAccessMask = 0,
 843                                                                 .dstAccessMask = 0,
 844                                                                 .dependencyFlags = 0
 845                                                         }
 846                                                 },
 847                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
 848                 }
 849         }
 850
 851         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 852                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 853                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 854                 .pStages = pipeline_shader_stages,
 855                 .pVertexInputState = vi_create_info,
 856                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 857                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 858                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 859                         .primitiveRestartEnable = false,
 860                 },
 861                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 862                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 863                         .viewportCount = 1,
 864                         .scissorCount = 1,
 865                 },
 866                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 867                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 868                         .rasterizerDiscardEnable = false,
 869                         .polygonMode = VK_POLYGON_MODE_FILL,
 870                         .cullMode = VK_CULL_MODE_NONE,
 871                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
 872                 },
 873                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
 874                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 875                         .rasterizationSamples = 1 << log2_samples,
 876                         .sampleShadingEnable = log2_samples > 1,
 877                         .minSampleShading = 1.0,
 878                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
 879                 },
 880                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 881                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 882                         .attachmentCount = 1,
 883                         .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 884                                 { .colorWriteMask =
 885                                   VK_COLOR_COMPONENT_A_BIT |
 886                                   VK_COLOR_COMPONENT_R_BIT |
 887                                   VK_COLOR_COMPONENT_G_BIT |
 888                                   VK_COLOR_COMPONENT_B_BIT },
 889                         }
 890                 },
 891                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 892                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 893                         .dynamicStateCount = 9,
 894                         .pDynamicStates = (VkDynamicState[]) {
 895                                 VK_DYNAMIC_STATE_VIEWPORT,
 896                                 VK_DYNAMIC_STATE_SCISSOR,
 897                                 VK_DYNAMIC_STATE_LINE_WIDTH,
 898                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
 899                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
 900                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
 901                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
 902                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
 903                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
 904                         },
 905                 },
 906                 .flags = 0,
 907                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 908                 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
 909                 .subpass = 0,
 910         };
 911
 912         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
 913                 .use_rectlist = true
 914         };
 915
 916         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
 917                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 918                                                &vk_pipeline_info, &radv_pipeline_info,
 919                                                &device->meta_state.alloc,
 920                                                &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
 921
 922
 923         ralloc_free(vs.nir);
 924         ralloc_free(fs.nir);
 925
 926         mtx_unlock(&device->meta_state.mtx);
 927         return result;
 928 }
 929
 930 static VkResult
 931 blit2d_init_depth_only_pipeline(struct radv_device *device,
 932                                 enum blit2d_src_type src_type,
 933                                 uint32_t log2_samples)
 934 {
 935         VkResult result;
 936         const char *name;
 937
 938         mtx_lock(&device->meta_state.mtx);
 939         if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
 940                 mtx_unlock(&device->meta_state.mtx);
 941                 return VK_SUCCESS;
 942         }
 943
 944         texel_fetch_build_func src_func;
 945         switch(src_type) {
 946         case BLIT2D_SRC_TYPE_IMAGE:
 947                 src_func = build_nir_texel_fetch;
 948                 name = "meta_blit2d_depth_image_fs";
 949                 break;
 950         case BLIT2D_SRC_TYPE_IMAGE_3D:
 951                 src_func = build_nir_texel_fetch;
 952                 name = "meta_blit3d_depth_image_fs";
 953                 break;
 954         case BLIT2D_SRC_TYPE_BUFFER:
 955                 src_func = build_nir_buffer_fetch;
 956                 name = "meta_blit2d_depth_buffer_fs";
 957                 break;
 958         default:
 959                 unreachable("unknown blit src type\n");
 960                 break;
 961         }
 962
 963         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 964         struct radv_shader_module fs = { .nir = NULL };
 965
 966         fs.nir = build_nir_copy_fragment_shader_depth(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 967         vi_create_info = &normal_vi_create_info;
 968
 969         struct radv_shader_module vs = {
 970                 .nir = build_nir_vertex_shader(),
 971         };
 972
 973         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 974                 {
 975                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 976                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 977                         .module = radv_shader_module_to_handle(&vs),
 978                         .pName = "main",
 979                         .pSpecializationInfo = NULL
 980                 }, {
 981                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 982                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 983                         .module = radv_shader_module_to_handle(&fs),
 984                         .pName = "main",
 985                         .pSpecializationInfo = NULL
 986                 },
 987         };
 988
 989         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
 990                 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
 991                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
 992                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 993                                                        &(VkRenderPassCreateInfo) {
 994                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 995                                                                .attachmentCount = 1,
 996                                                                .pAttachments = &(VkAttachmentDescription) {
 997                                                                        .format = VK_FORMAT_D32_SFLOAT,
 998                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 999                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1000                                                                        .initialLayout = layout,
1001                                                                        .finalLayout = layout,
1002                                                                },
1003                                                                .subpassCount = 1,
1004                                                                .pSubpasses = &(VkSubpassDescription) {
1005                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1006                                                                        .inputAttachmentCount = 0,
1007                                                                        .colorAttachmentCount = 0,
1008                                                                        .pColorAttachments = NULL,
1009                                                                        .pResolveAttachments = NULL,
1010                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
1011                                                                                .attachment = 0,
1012                                                                                .layout = layout,
1013                                                                        },
1014                                                                        .preserveAttachmentCount = 0,
1015                                                                        .pPreserveAttachments = NULL,
1016                                                                },
1017                                                                .dependencyCount = 2,
1018                                                                .pDependencies = (VkSubpassDependency[]) {
1019                                                                 {
1020                                                                         .srcSubpass = VK_SUBPASS_EXTERNAL,
1021                                                                         .dstSubpass = 0,
1022                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1023                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1024                                                                         .srcAccessMask = 0,
1025                                                                         .dstAccessMask = 0,
1026                                                                         .dependencyFlags = 0
1027                                                                 },
1028                                                                 {
1029                                                                         .srcSubpass = 0,
1030                                                                         .dstSubpass = VK_SUBPASS_EXTERNAL,
1031                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1032                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1033                                                                         .srcAccessMask = 0,
1034                                                                         .dstAccessMask = 0,
1035                                                                         .dependencyFlags = 0
1036                                                                 }
1037                                                         },
1038                                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
1039                 }
1040         }
1041
1042         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1043                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1044                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1045                 .pStages = pipeline_shader_stages,
1046                 .pVertexInputState = vi_create_info,
1047                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1048                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1049                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1050                         .primitiveRestartEnable = false,
1051                 },
1052                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1053                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1054                         .viewportCount = 1,
1055                         .scissorCount = 1,
1056                 },
1057                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1058                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1059                         .rasterizerDiscardEnable = false,
1060                         .polygonMode = VK_POLYGON_MODE_FILL,
1061                         .cullMode = VK_CULL_MODE_NONE,
1062                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1063                 },
1064                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1065                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1066                         .rasterizationSamples = 1 << log2_samples,
1067                         .sampleShadingEnable = false,
1068                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1069                 },
1070                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1071                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1072                         .attachmentCount = 0,
1073                         .pAttachments = NULL,
1074                 },
1075                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1076                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1077                         .depthTestEnable = true,
1078                         .depthWriteEnable = true,
1079                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1080                 },
1081                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1082                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1083                         .dynamicStateCount = 9,
1084                         .pDynamicStates = (VkDynamicState[]) {
1085                                 VK_DYNAMIC_STATE_VIEWPORT,
1086                                 VK_DYNAMIC_STATE_SCISSOR,
1087                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1088                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1089                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1090                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1091                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1092                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1093                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1094                         },
1095                 },
1096                 .flags = 0,
1097                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1098                 .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1099                 .subpass = 0,
1100         };
1101
1102         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1103                 .use_rectlist = true
1104         };
1105
1106         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1107                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1108                                                &vk_pipeline_info, &radv_pipeline_info,
1109                                                &device->meta_state.alloc,
1110                                                &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1111
1112
1113         ralloc_free(vs.nir);
1114         ralloc_free(fs.nir);
1115
1116         mtx_unlock(&device->meta_state.mtx);
1117         return result;
1118 }
1119
1120 static VkResult
1121 blit2d_init_stencil_only_pipeline(struct radv_device *device,
1122                                   enum blit2d_src_type src_type,
1123                                   uint32_t log2_samples)
1124 {
1125         VkResult result;
1126         const char *name;
1127
1128         mtx_lock(&device->meta_state.mtx);
1129         if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1130                 mtx_unlock(&device->meta_state.mtx);
1131                 return VK_SUCCESS;
1132         }
1133
1134         texel_fetch_build_func src_func;
1135         switch(src_type) {
1136         case BLIT2D_SRC_TYPE_IMAGE:
1137                 src_func = build_nir_texel_fetch;
1138                 name = "meta_blit2d_stencil_image_fs";
1139                 break;
1140         case BLIT2D_SRC_TYPE_IMAGE_3D:
1141                 src_func = build_nir_texel_fetch;
1142                 name = "meta_blit3d_stencil_image_fs";
1143                 break;
1144         case BLIT2D_SRC_TYPE_BUFFER:
1145                 src_func = build_nir_buffer_fetch;
1146                 name = "meta_blit2d_stencil_buffer_fs";
1147                 break;
1148         default:
1149                 unreachable("unknown blit src type\n");
1150                 break;
1151         }
1152
1153         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1154         struct radv_shader_module fs = { .nir = NULL };
1155
1156         fs.nir = build_nir_copy_fragment_shader_stencil(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1157         vi_create_info = &normal_vi_create_info;
1158
1159         struct radv_shader_module vs = {
1160                 .nir = build_nir_vertex_shader(),
1161         };
1162
1163         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1164                 {
1165                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1166                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
1167                         .module = radv_shader_module_to_handle(&vs),
1168                         .pName = "main",
1169                         .pSpecializationInfo = NULL
1170                 }, {
1171                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1172                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1173                         .module = radv_shader_module_to_handle(&fs),
1174                         .pName = "main",
1175                         .pSpecializationInfo = NULL
1176                 },
1177         };
1178
1179         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1180                 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1181                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1182                         result = radv_CreateRenderPass(radv_device_to_handle(device),
1183                                                        &(VkRenderPassCreateInfo) {
1184                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
1185                                                                .attachmentCount = 1,
1186                                                                .pAttachments = &(VkAttachmentDescription) {
1187                                                                        .format = VK_FORMAT_S8_UINT,
1188                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1189                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1190                                                                        .initialLayout = layout,
1191                                                                        .finalLayout = layout,
1192                                                                },
1193                                                                .subpassCount = 1,
1194                                                                .pSubpasses = &(VkSubpassDescription) {
1195                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1196                                                                        .inputAttachmentCount = 0,
1197                                                                        .colorAttachmentCount = 0,
1198                                                                        .pColorAttachments = NULL,
1199                                                                        .pResolveAttachments = NULL,
1200                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
1201                                                                                .attachment = 0,
1202                                                                                .layout = layout,
1203                                                                        },
1204                                                                        .preserveAttachmentCount = 0,
1205                                                                        .pPreserveAttachments = NULL,
1206                                                                },
1207                                                                .dependencyCount = 2,
1208                                                                .pDependencies = (VkSubpassDependency[]) {
1209                                                                 {
1210                                                                         .srcSubpass = VK_SUBPASS_EXTERNAL,
1211                                                                         .dstSubpass = 0,
1212                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1213                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1214                                                                         .srcAccessMask = 0,
1215                                                                         .dstAccessMask = 0,
1216                                                                         .dependencyFlags = 0
1217                                                                 },
1218                                                                 {
1219                                                                         .srcSubpass = 0,
1220                                                                         .dstSubpass = VK_SUBPASS_EXTERNAL,
1221                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1222                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1223                                                                         .srcAccessMask = 0,
1224                                                                         .dstAccessMask = 0,
1225                                                                         .dependencyFlags = 0
1226                                                                 }
1227                                                         },
1228                                                        }, &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1229                 }
1230         }
1231
1232         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1233                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1234                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1235                 .pStages = pipeline_shader_stages,
1236                 .pVertexInputState = vi_create_info,
1237                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1238                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1239                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1240                         .primitiveRestartEnable = false,
1241                 },
1242                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1243                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1244                         .viewportCount = 1,
1245                         .scissorCount = 1,
1246                 },
1247                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1248                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1249                         .rasterizerDiscardEnable = false,
1250                         .polygonMode = VK_POLYGON_MODE_FILL,
1251                         .cullMode = VK_CULL_MODE_NONE,
1252                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1253                 },
1254                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1255                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1256                         .rasterizationSamples = 1 << log2_samples,
1257                         .sampleShadingEnable = false,
1258                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1259                 },
1260                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1261                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1262                         .attachmentCount = 0,
1263                         .pAttachments = NULL,
1264                 },
1265                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1266                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1267                         .depthTestEnable = false,
1268                         .depthWriteEnable = false,
1269                         .stencilTestEnable = true,
1270                         .front = {
1271                                 .failOp = VK_STENCIL_OP_REPLACE,
1272                                 .passOp = VK_STENCIL_OP_REPLACE,
1273                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1274                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1275                                 .compareMask = 0xff,
1276                                 .writeMask = 0xff,
1277                                 .reference = 0
1278                         },
1279                         .back = {
1280                                 .failOp = VK_STENCIL_OP_REPLACE,
1281                                 .passOp = VK_STENCIL_OP_REPLACE,
1282                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1283                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1284                                 .compareMask = 0xff,
1285                                 .writeMask = 0xff,
1286                                 .reference = 0
1287                         },
1288                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1289                 },
1290                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1291                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1292                         .dynamicStateCount = 6,
1293                         .pDynamicStates = (VkDynamicState[]) {
1294                                 VK_DYNAMIC_STATE_VIEWPORT,
1295                                 VK_DYNAMIC_STATE_SCISSOR,
1296                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1297                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1298                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1299                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1300                         },
1301                 },
1302                 .flags = 0,
1303                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1304                 .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1305                 .subpass = 0,
1306         };
1307
1308         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1309                 .use_rectlist = true
1310         };
1311
1312         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1313                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1314                                                &vk_pipeline_info, &radv_pipeline_info,
1315                                                &device->meta_state.alloc,
1316                                                &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1317
1318
1319         ralloc_free(vs.nir);
1320         ralloc_free(fs.nir);
1321
1322         mtx_unlock(&device->meta_state.mtx);
1323         return result;
1324 }
1325
1326 static VkResult
1327 meta_blit2d_create_pipe_layout(struct radv_device *device,
1328                                int idx,
1329                                uint32_t log2_samples)
1330 {
1331         VkResult result;
1332         VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER) ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1333         const VkPushConstantRange push_constant_ranges[] = {
1334                 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1335                 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1336         };
1337         int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1338
1339         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
1340                                                 &(VkDescriptorSetLayoutCreateInfo) {
1341                                                         .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1342                                                         .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1343                                                         .bindingCount = 1,
1344                                                         .pBindings = (VkDescriptorSetLayoutBinding[]) {
1345                                                         {
1346                                                                 .binding = 0,
1347                                                                 .descriptorType = desc_type,
1348                                                                 .descriptorCount = 1,
1349                                                                 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1350                                                                 .pImmutableSamplers = NULL
1351                                                         },
1352                                                         }
1353                                                 }, &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1354         if (result != VK_SUCCESS)
1355                 goto fail;
1356
1357         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
1358                                            &(VkPipelineLayoutCreateInfo) {
1359                                                    .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1360                                                            .setLayoutCount = 1,
1361                                                            .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1362                                                            .pushConstantRangeCount = num_push_constant_range,
1363                                                            .pPushConstantRanges = push_constant_ranges,
1364                                                            },
1365                                            &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1366         if (result != VK_SUCCESS)
1367                 goto fail;
1368         return VK_SUCCESS;
1369 fail:
1370         return result;
1371 }
1372
1373 VkResult
1374 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1375 {
1376         VkResult result;
1377         bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1378
1379         for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; log2_samples++) {
1380                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1381                         if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1382                                 continue;
1383
1384                         /* Don't need to handle copies between buffers and multisample images. */
1385                         if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1386                                 continue;
1387
1388                         result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1389                         if (result != VK_SUCCESS)
1390                                 goto fail;
1391
1392                         if (on_demand)
1393                                 continue;
1394
1395                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1396                                 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j], log2_samples);
1397                                 if (result != VK_SUCCESS)
1398                                         goto fail;
1399                         }
1400
1401                         result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1402                         if (result != VK_SUCCESS)
1403                                 goto fail;
1404
1405                         result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1406                         if (result != VK_SUCCESS)
1407                                 goto fail;
1408                 }
1409         }
1410
1411         return VK_SUCCESS;
1412
1413 fail:
1414         radv_device_finish_meta_blit2d_state(device);
1415         return result;
1416 }