src/amd/vulkan/radv_meta_blit2d.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  *
   4  * based on anv driver:
   5  * Copyright © 2016 Intel Corporation
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24  * IN THE SOFTWARE.
  25  */
  26
  27 #include "radv_meta.h"
  28 #include "nir/nir_builder.h"
  29 #include "vk_format.h"
  30
  31 enum blit2d_src_type {
  32         BLIT2D_SRC_TYPE_IMAGE,
  33         BLIT2D_SRC_TYPE_IMAGE_3D,
  34         BLIT2D_SRC_TYPE_BUFFER,
  35         BLIT2D_NUM_SRC_TYPES,
  36 };
  37
  38 static VkResult
  39 blit2d_init_color_pipeline(struct radv_device *device,
  40                            enum blit2d_src_type src_type,
  41                            VkFormat format,
  42                            uint32_t log2_samples);
  43
  44 static VkResult
  45 blit2d_init_depth_only_pipeline(struct radv_device *device,
  46                                 enum blit2d_src_type src_type,
  47                                 uint32_t log2_samples);
  48
  49 static VkResult
  50 blit2d_init_stencil_only_pipeline(struct radv_device *device,
  51                                   enum blit2d_src_type src_type,
  52                                   uint32_t log2_samples);
  53
  54 static void
  55 create_iview(struct radv_cmd_buffer *cmd_buffer,
  56              struct radv_meta_blit2d_surf *surf,
  57              struct radv_image_view *iview, VkFormat depth_format,
  58               VkImageAspectFlagBits aspects)
  59 {
  60         VkFormat format;
  61         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
  62                 radv_meta_get_view_type(surf->image);
  63
  64         if (depth_format)
  65                 format = depth_format;
  66         else
  67                 format = surf->format;
  68
  69         radv_image_view_init(iview, cmd_buffer->device,
  70                              &(VkImageViewCreateInfo) {
  71                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
  72                                              .image = radv_image_to_handle(surf->image),
  73                                              .viewType = view_type,
  74                                              .format = format,
  75                                              .subresourceRange = {
  76                                              .aspectMask = aspects,
  77                                              .baseMipLevel = surf->level,
  78                                              .levelCount = 1,
  79                                              .baseArrayLayer = surf->layer,
  80                                              .layerCount = 1
  81                                      },
  82                              }, NULL);
  83 }
  84
  85 static void
  86 create_bview(struct radv_cmd_buffer *cmd_buffer,
  87              struct radv_meta_blit2d_buffer *src,
  88              struct radv_buffer_view *bview, VkFormat depth_format)
  89 {
  90         VkFormat format;
  91
  92         if (depth_format)
  93                 format = depth_format;
  94         else
  95                 format = src->format;
  96         radv_buffer_view_init(bview, cmd_buffer->device,
  97                               &(VkBufferViewCreateInfo) {
  98                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
  99                                       .flags = 0,
 100                                       .buffer = radv_buffer_to_handle(src->buffer),
 101                                       .format = format,
 102                                       .offset = src->offset,
 103                                       .range = VK_WHOLE_SIZE,
 104                               });
 105
 106 }
 107
 108 struct blit2d_src_temps {
 109         struct radv_image_view iview;
 110         struct radv_buffer_view bview;
 111 };
 112
 113 static void
 114 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer,
 115                 struct radv_meta_blit2d_surf *src_img,
 116                 struct radv_meta_blit2d_buffer *src_buf,
 117                 struct blit2d_src_temps *tmp,
 118                 enum blit2d_src_type src_type, VkFormat depth_format,
 119                 VkImageAspectFlagBits aspects,
 120                 uint32_t log2_samples)
 121 {
 122         struct radv_device *device = cmd_buffer->device;
 123
 124         if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
 125                 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
 126
 127                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 128                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 129                                               0, /* set */
 130                                               1, /* descriptorWriteCount */
 131                                               (VkWriteDescriptorSet[]) {
 132                                                       {
 133                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 134                                                               .dstBinding = 0,
 135                                                               .dstArrayElement = 0,
 136                                                               .descriptorCount = 1,
 137                                                               .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
 138                                                               .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(&tmp->bview) }
 139                                                       }
 140                                               });
 141
 142                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 143                                       device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 144                                       VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 145                                       &src_buf->pitch);
 146         } else {
 147                 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
 148
 149                 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
 150                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 151                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 152                                               VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 153                                               &src_img->layer);
 154
 155                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 156                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 157                                               0, /* set */
 158                                               1, /* descriptorWriteCount */
 159                                               (VkWriteDescriptorSet[]) {
 160                                                       {
 161                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 162                                                               .dstBinding = 0,
 163                                                               .dstArrayElement = 0,
 164                                                               .descriptorCount = 1,
 165                                                               .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 166                                                               .pImageInfo = (VkDescriptorImageInfo[]) {
 167                                                                       {
 168                                                                               .sampler = VK_NULL_HANDLE,
 169                                                                               .imageView = radv_image_view_to_handle(&tmp->iview),
 170                                                                               .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 171                                                                       },
 172                                                               }
 173                                                       }
 174                                               });
 175         }
 176 }
 177
 178 struct blit2d_dst_temps {
 179         VkImage image;
 180         struct radv_image_view iview;
 181         VkFramebuffer fb;
 182 };
 183
 184 static void
 185 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer,
 186                 struct radv_meta_blit2d_surf *dst,
 187                 uint32_t width,
 188                 uint32_t height,
 189                 VkFormat depth_format,
 190                 struct blit2d_dst_temps *tmp,
 191                 VkImageAspectFlagBits aspects)
 192 {
 193         create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
 194
 195         radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
 196                                &(VkFramebufferCreateInfo) {
 197                                        .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
 198                                                .attachmentCount = 1,
 199                                                .pAttachments = (VkImageView[]) {
 200                                                radv_image_view_to_handle(&tmp->iview),
 201                                        },
 202                                        .width = width,
 203                                        .height = height,
 204                                        .layers = 1
 205                                 }, &cmd_buffer->pool->alloc, &tmp->fb);
 206 }
 207
 208 static void
 209 bind_pipeline(struct radv_cmd_buffer *cmd_buffer,
 210               enum blit2d_src_type src_type, unsigned fs_key,
 211               uint32_t log2_samples)
 212 {
 213         VkPipeline pipeline =
 214                 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
 215
 216         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 217                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 218 }
 219
 220 static void
 221 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer,
 222                     enum blit2d_src_type src_type,
 223                     uint32_t log2_samples)
 224 {
 225         VkPipeline pipeline =
 226                 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
 227
 228         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 229                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 230 }
 231
 232 static void
 233 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer,
 234                       enum blit2d_src_type src_type,
 235                       uint32_t log2_samples)
 236 {
 237         VkPipeline pipeline =
 238                 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
 239
 240         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 241                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 242 }
 243
 244 static void
 245 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
 246                             struct radv_meta_blit2d_surf *src_img,
 247                             struct radv_meta_blit2d_buffer *src_buf,
 248                             struct radv_meta_blit2d_surf *dst,
 249                             unsigned num_rects,
 250                             struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
 251                             uint32_t log2_samples)
 252 {
 253         struct radv_device *device = cmd_buffer->device;
 254
 255         for (unsigned r = 0; r < num_rects; ++r) {
 256                 unsigned i;
 257                 for_each_bit(i, dst->aspect_mask) {
 258                         unsigned aspect_mask = 1u << i;
 259                         unsigned src_aspect_mask = aspect_mask;
 260                         VkFormat depth_format = 0;
 261                         if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
 262                                 depth_format = vk_format_stencil_only(dst->image->vk_format);
 263                         else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
 264                                 depth_format = vk_format_depth_only(dst->image->vk_format);
 265                         else if (src_img)
 266                                 src_aspect_mask = src_img->aspect_mask;
 267
 268                         struct blit2d_src_temps src_temps;
 269                         blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format, src_aspect_mask, log2_samples);
 270
 271                         struct blit2d_dst_temps dst_temps;
 272                         blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
 273                                         rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
 274
 275                         float vertex_push_constants[4] = {
 276                                 rects[r].src_x,
 277                                 rects[r].src_y,
 278                                 rects[r].src_x + rects[r].width,
 279                                 rects[r].src_y + rects[r].height,
 280                         };
 281
 282                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 283                                         device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 284                                         VK_SHADER_STAGE_VERTEX_BIT, 0, 16,
 285                                         vertex_push_constants);
 286
 287                         if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT ||
 288                             aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
 289                             aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT ||
 290                             aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
 291                                 unsigned fs_key = radv_format_meta_fs_key(dst_temps.iview.vk_format);
 292                                 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
 293
 294                                 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] == VK_NULL_HANDLE) {
 295                                         VkResult ret = blit2d_init_color_pipeline(device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
 296                                         if (ret != VK_SUCCESS) {
 297                                                 cmd_buffer->record_result = ret;
 298                                                 goto fail_pipeline;
 299                                         }
 300                                 }
 301
 302                                 radv_cmd_buffer_begin_render_pass(cmd_buffer,
 303                                                                   &(VkRenderPassBeginInfo) {
 304                                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 305                                                                         .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
 306                                                                         .framebuffer = dst_temps.fb,
 307                                                                         .renderArea = {
 308                                                                                 .offset = { rects[r].dst_x, rects[r].dst_y, },
 309                                                                                 .extent = { rects[r].width, rects[r].height },
 310                                                                         },
 311                                                                         .clearValueCount = 0,
 312                                                                         .pClearValues = NULL,
 313                                                                   });
 314
 315                                 radv_cmd_buffer_set_subpass(cmd_buffer,
 316                                                             &cmd_buffer->state.pass->subpasses[0]);
 317
 318                                 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
 319                         } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 320                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 321
 322                                 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] == VK_NULL_HANDLE) {
 323                                         VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
 324                                         if (ret != VK_SUCCESS) {
 325                                                 cmd_buffer->record_result = ret;
 326                                                 goto fail_pipeline;
 327                                         }
 328                                 }
 329
 330                                 radv_cmd_buffer_begin_render_pass(cmd_buffer,
 331                                                                   &(VkRenderPassBeginInfo) {
 332                                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 333                                                                         .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
 334                                                                         .framebuffer = dst_temps.fb,
 335                                                                         .renderArea = {
 336                                                                                 .offset = { rects[r].dst_x, rects[r].dst_y, },
 337                                                                                 .extent = { rects[r].width, rects[r].height },
 338                                                                         },
 339                                                                         .clearValueCount = 0,
 340                                                                         .pClearValues = NULL,
 341                                                                   });
 342
 343                                 radv_cmd_buffer_set_subpass(cmd_buffer,
 344                                                             &cmd_buffer->state.pass->subpasses[0]);
 345
 346                                 bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
 347
 348                         } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 349                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 350
 351                                 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] == VK_NULL_HANDLE) {
 352                                         VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
 353                                         if (ret != VK_SUCCESS) {
 354                                                 cmd_buffer->record_result = ret;
 355                                                 goto fail_pipeline;
 356                                         }
 357                                 }
 358
 359                                 radv_cmd_buffer_begin_render_pass(cmd_buffer,
 360                                                                   &(VkRenderPassBeginInfo) {
 361                                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 362                                                                         .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
 363                                                                         .framebuffer = dst_temps.fb,
 364                                                                         .renderArea = {
 365                                                                                 .offset = { rects[r].dst_x, rects[r].dst_y, },
 366                                                                                 .extent = { rects[r].width, rects[r].height },
 367                                                                         },
 368                                                                         .clearValueCount = 0,
 369                                                                         .pClearValues = NULL,
 370                                                                    });
 371
 372                                 radv_cmd_buffer_set_subpass(cmd_buffer,
 373                                                             &cmd_buffer->state.pass->subpasses[0]);
 374
 375                                 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
 376                         } else
 377                                 unreachable("Processing blit2d with multiple aspects.");
 378
 379                         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 380                                 .x = rects[r].dst_x,
 381                                 .y = rects[r].dst_y,
 382                                 .width = rects[r].width,
 383                                 .height = rects[r].height,
 384                                 .minDepth = 0.0f,
 385                                 .maxDepth = 1.0f
 386                         });
 387
 388                         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 389                                 .offset = (VkOffset2D) { rects[r].dst_x, rects[r].dst_y },
 390                                 .extent = (VkExtent2D) { rects[r].width, rects[r].height },
 391                         });
 392
 393
 394
 395                         radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
 396                         radv_cmd_buffer_end_render_pass(cmd_buffer);
 397
 398 fail_pipeline:
 399                         /* At the point where we emit the draw call, all data from the
 400                         * descriptor sets, etc. has been used.  We are free to delete it.
 401                         */
 402                         radv_DestroyFramebuffer(radv_device_to_handle(device),
 403                                                 dst_temps.fb,
 404                                                 &cmd_buffer->pool->alloc);
 405                 }
 406         }
 407 }
 408
 409 void
 410 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer,
 411                  struct radv_meta_blit2d_surf *src_img,
 412                  struct radv_meta_blit2d_buffer *src_buf,
 413                  struct radv_meta_blit2d_surf *dst,
 414                  unsigned num_rects,
 415                  struct radv_meta_blit2d_rect *rects)
 416 {
 417         bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
 418                 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
 419         enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER :
 420                 use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D : BLIT2D_SRC_TYPE_IMAGE;
 421         radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst,
 422                                     num_rects, rects, src_type,
 423                                     src_img ? util_logbase2(src_img->image->info.samples) : 0);
 424 }
 425
 426 static nir_shader *
 427 build_nir_vertex_shader(void)
 428 {
 429         const struct glsl_type *vec4 = glsl_vec4_type();
 430         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 431         nir_builder b;
 432
 433         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 434         b.shader->info.name = ralloc_strdup(b.shader, "meta_blit2d_vs");
 435
 436         nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 437                                                     vec4, "gl_Position");
 438         pos_out->data.location = VARYING_SLOT_POS;
 439
 440         nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 441                                                         vec2, "v_tex_pos");
 442         tex_pos_out->data.location = VARYING_SLOT_VAR0;
 443         tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
 444
 445         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 446         nir_store_var(&b, pos_out, outvec, 0xf);
 447
 448         nir_intrinsic_instr *src_box = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 449         src_box->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 450         nir_intrinsic_set_base(src_box, 0);
 451         nir_intrinsic_set_range(src_box, 16);
 452         src_box->num_components = 4;
 453         nir_ssa_dest_init(&src_box->instr, &src_box->dest, 4, 32, "src_box");
 454         nir_builder_instr_insert(&b, &src_box->instr);
 455
 456         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_vertex_id_zero_base);
 457         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 458         nir_builder_instr_insert(&b, &vertex_id->instr);
 459
 460         /* vertex 0 - src_x, src_y */
 461         /* vertex 1 - src_x, src_y+h */
 462         /* vertex 2 - src_x+w, src_y */
 463         /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
 464            channel 1 is vertex id != 1 ? src_y : src_y + w */
 465
 466         nir_ssa_def *c0cmp = nir_ine(&b, &vertex_id->dest.ssa,
 467                                      nir_imm_int(&b, 2));
 468         nir_ssa_def *c1cmp = nir_ine(&b, &vertex_id->dest.ssa,
 469                                      nir_imm_int(&b, 1));
 470
 471         nir_ssa_def *comp[2];
 472         comp[0] = nir_bcsel(&b, c0cmp,
 473                             nir_channel(&b, &src_box->dest.ssa, 0),
 474                             nir_channel(&b, &src_box->dest.ssa, 2));
 475
 476         comp[1] = nir_bcsel(&b, c1cmp,
 477                             nir_channel(&b, &src_box->dest.ssa, 1),
 478                             nir_channel(&b, &src_box->dest.ssa, 3));
 479         nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
 480         nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
 481         return b.shader;
 482 }
 483
 484 typedef nir_ssa_def* (*texel_fetch_build_func)(struct nir_builder *,
 485                                                struct radv_device *,
 486                                                nir_ssa_def *, bool, bool);
 487
 488 static nir_ssa_def *
 489 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device,
 490                       nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 491 {
 492         enum glsl_sampler_dim dim =
 493                 is_3d ? GLSL_SAMPLER_DIM_3D : is_multisampled ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D;
 494         const struct glsl_type *sampler_type =
 495                 glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
 496         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 497                                                     sampler_type, "s_tex");
 498         sampler->data.descriptor_set = 0;
 499         sampler->data.binding = 0;
 500
 501         nir_ssa_def *tex_pos_3d = NULL;
 502         nir_intrinsic_instr *sample_idx = NULL;
 503         if (is_3d) {
 504                 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 505                 nir_intrinsic_set_base(layer, 16);
 506                 nir_intrinsic_set_range(layer, 4);
 507                 layer->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 508                 layer->num_components = 1;
 509                 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
 510                 nir_builder_instr_insert(b, &layer->instr);
 511
 512                 nir_ssa_def *chans[3];
 513                 chans[0] = nir_channel(b, tex_pos, 0);
 514                 chans[1] = nir_channel(b, tex_pos, 1);
 515                 chans[2] = &layer->dest.ssa;
 516                 tex_pos_3d = nir_vec(b, chans, 3);
 517         }
 518         if (is_multisampled) {
 519                 sample_idx = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_sample_id);
 520                 nir_ssa_dest_init(&sample_idx->instr, &sample_idx->dest, 1, 32, "sample_idx");
 521                 nir_builder_instr_insert(b, &sample_idx->instr);
 522         }
 523
 524         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 525
 526         nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
 527         tex->sampler_dim = dim;
 528         tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
 529         tex->src[0].src_type = nir_tex_src_coord;
 530         tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
 531         tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
 532         tex->src[1].src = nir_src_for_ssa(is_multisampled ? &sample_idx->dest.ssa : nir_imm_int(b, 0));
 533         tex->src[2].src_type = nir_tex_src_texture_deref;
 534         tex->src[2].src = nir_src_for_ssa(tex_deref);
 535         if (is_multisampled) {
 536                 tex->src[3].src_type = nir_tex_src_lod;
 537                 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
 538         }
 539         tex->dest_type = nir_type_uint;
 540         tex->is_array = false;
 541         tex->coord_components = is_3d ? 3 : 2;
 542
 543         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 544         nir_builder_instr_insert(b, &tex->instr);
 545
 546         return &tex->dest.ssa;
 547 }
 548
 549
 550 static nir_ssa_def *
 551 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device,
 552                        nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 553 {
 554         const struct glsl_type *sampler_type =
 555                 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
 556         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 557                                                     sampler_type, "s_tex");
 558         sampler->data.descriptor_set = 0;
 559         sampler->data.binding = 0;
 560
 561         nir_intrinsic_instr *width = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 562         nir_intrinsic_set_base(width, 16);
 563         nir_intrinsic_set_range(width, 4);
 564         width->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 565         width->num_components = 1;
 566         nir_ssa_dest_init(&width->instr, &width->dest, 1, 32, "width");
 567         nir_builder_instr_insert(b, &width->instr);
 568
 569         nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
 570         nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
 571         pos_y = nir_imul(b, pos_y, &width->dest.ssa);
 572         pos_x = nir_iadd(b, pos_x, pos_y);
 573
 574         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 575
 576         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
 577         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 578         tex->op = nir_texop_txf;
 579         tex->src[0].src_type = nir_tex_src_coord;
 580         tex->src[0].src = nir_src_for_ssa(pos_x);
 581         tex->src[1].src_type = nir_tex_src_texture_deref;
 582         tex->src[1].src = nir_src_for_ssa(tex_deref);
 583         tex->dest_type = nir_type_uint;
 584         tex->is_array = false;
 585         tex->coord_components = 1;
 586
 587         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 588         nir_builder_instr_insert(b, &tex->instr);
 589
 590         return &tex->dest.ssa;
 591 }
 592
 593 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
 594         .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 595         .vertexBindingDescriptionCount = 0,
 596         .vertexAttributeDescriptionCount = 0,
 597 };
 598
 599 static nir_shader *
 600 build_nir_copy_fragment_shader(struct radv_device *device,
 601                                texel_fetch_build_func txf_func, const char* name, bool is_3d,
 602                                bool is_multisampled)
 603 {
 604         const struct glsl_type *vec4 = glsl_vec4_type();
 605         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 606         nir_builder b;
 607
 608         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 609         b.shader->info.name = ralloc_strdup(b.shader, name);
 610
 611         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 612                                                        vec2, "v_tex_pos");
 613         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 614
 615         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 616                                                       vec4, "f_color");
 617         color_out->data.location = FRAG_RESULT_DATA0;
 618
 619         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 620         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 621
 622         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 623         nir_store_var(&b, color_out, color, 0xf);
 624
 625         return b.shader;
 626 }
 627
 628 static nir_shader *
 629 build_nir_copy_fragment_shader_depth(struct radv_device *device,
 630                                      texel_fetch_build_func txf_func, const char* name, bool is_3d,
 631                                      bool is_multisampled)
 632 {
 633         const struct glsl_type *vec4 = glsl_vec4_type();
 634         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 635         nir_builder b;
 636
 637         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 638         b.shader->info.name = ralloc_strdup(b.shader, name);
 639
 640         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 641                                                        vec2, "v_tex_pos");
 642         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 643
 644         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 645                                                       vec4, "f_color");
 646         color_out->data.location = FRAG_RESULT_DEPTH;
 647
 648         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 649         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 650
 651         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 652         nir_store_var(&b, color_out, color, 0x1);
 653
 654         return b.shader;
 655 }
 656
 657 static nir_shader *
 658 build_nir_copy_fragment_shader_stencil(struct radv_device *device,
 659                                        texel_fetch_build_func txf_func, const char* name, bool is_3d,
 660                                        bool is_multisampled)
 661 {
 662         const struct glsl_type *vec4 = glsl_vec4_type();
 663         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 664         nir_builder b;
 665
 666         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 667         b.shader->info.name = ralloc_strdup(b.shader, name);
 668
 669         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 670                                                        vec2, "v_tex_pos");
 671         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 672
 673         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 674                                                       vec4, "f_color");
 675         color_out->data.location = FRAG_RESULT_STENCIL;
 676
 677         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 678         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 679
 680         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 681         nir_store_var(&b, color_out, color, 0x1);
 682
 683         return b.shader;
 684 }
 685
 686 void
 687 radv_device_finish_meta_blit2d_state(struct radv_device *device)
 688 {
 689         struct radv_meta_state *state = &device->meta_state;
 690
 691         for(unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 692                 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
 693                         radv_DestroyRenderPass(radv_device_to_handle(device),
 694                                                state->blit2d_render_passes[j][k],
 695                                                &state->alloc);
 696                 }
 697         }
 698
 699         for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT; j++) {
 700                 radv_DestroyRenderPass(radv_device_to_handle(device),
 701                                        state->blit2d_depth_only_rp[j], &state->alloc);
 702                 radv_DestroyRenderPass(radv_device_to_handle(device),
 703                                        state->blit2d_stencil_only_rp[j], &state->alloc);
 704         }
 705
 706         for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; ++log2_samples) {
 707                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
 708                         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 709                                                    state->blit2d[log2_samples].p_layouts[src],
 710                                                    &state->alloc);
 711                         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 712                                                         state->blit2d[log2_samples].ds_layouts[src],
 713                                                         &state->alloc);
 714
 715                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 716                                 radv_DestroyPipeline(radv_device_to_handle(device),
 717                                                      state->blit2d[log2_samples].pipelines[src][j],
 718                                                      &state->alloc);
 719                         }
 720
 721                         radv_DestroyPipeline(radv_device_to_handle(device),
 722                                              state->blit2d[log2_samples].depth_only_pipeline[src],
 723                                              &state->alloc);
 724                         radv_DestroyPipeline(radv_device_to_handle(device),
 725                                              state->blit2d[log2_samples].stencil_only_pipeline[src],
 726                                              &state->alloc);
 727                 }
 728         }
 729 }
 730
 731 static VkResult
 732 blit2d_init_color_pipeline(struct radv_device *device,
 733                            enum blit2d_src_type src_type,
 734                            VkFormat format,
 735                            uint32_t log2_samples)
 736 {
 737         VkResult result;
 738         unsigned fs_key = radv_format_meta_fs_key(format);
 739         const char *name;
 740
 741         mtx_lock(&device->meta_state.mtx);
 742         if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
 743                 mtx_unlock(&device->meta_state.mtx);
 744                 return VK_SUCCESS;
 745         }
 746
 747         texel_fetch_build_func src_func;
 748         switch(src_type) {
 749         case BLIT2D_SRC_TYPE_IMAGE:
 750                 src_func = build_nir_texel_fetch;
 751                 name = "meta_blit2d_image_fs";
 752                 break;
 753         case BLIT2D_SRC_TYPE_IMAGE_3D:
 754                 src_func = build_nir_texel_fetch;
 755                 name = "meta_blit3d_image_fs";
 756                 break;
 757         case BLIT2D_SRC_TYPE_BUFFER:
 758                 src_func = build_nir_buffer_fetch;
 759                 name = "meta_blit2d_buffer_fs";
 760                 break;
 761         default:
 762                 unreachable("unknown blit src type\n");
 763                 break;
 764         }
 765
 766         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 767         struct radv_shader_module fs = { .nir = NULL };
 768
 769
 770         fs.nir = build_nir_copy_fragment_shader(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 771         vi_create_info = &normal_vi_create_info;
 772
 773         struct radv_shader_module vs = {
 774                 .nir = build_nir_vertex_shader(),
 775         };
 776
 777         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 778                 {
 779                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 780                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 781                         .module = radv_shader_module_to_handle(&vs),
 782                         .pName = "main",
 783                         .pSpecializationInfo = NULL
 784                 }, {
 785                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 786                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 787                         .module = radv_shader_module_to_handle(&fs),
 788                         .pName = "main",
 789                         .pSpecializationInfo = NULL
 790                 },
 791         };
 792
 793         for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
 794                 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
 795                         VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
 796
 797                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 798                                                 &(VkRenderPassCreateInfo) {
 799                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 800                                                         .attachmentCount = 1,
 801                                                         .pAttachments = &(VkAttachmentDescription) {
 802                                                         .format = format,
 803                                                         .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 804                                                         .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 805                                                         .initialLayout = layout,
 806                                                         .finalLayout = layout,
 807                                                         },
 808                                                 .subpassCount = 1,
 809                                                 .pSubpasses = &(VkSubpassDescription) {
 810                                                         .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 811                                                         .inputAttachmentCount = 0,
 812                                                         .colorAttachmentCount = 1,
 813                                                         .pColorAttachments = &(VkAttachmentReference) {
 814                                                                 .attachment = 0,
 815                                                                 .layout = layout,
 816                                                                 },
 817                                                 .pResolveAttachments = NULL,
 818                                                 .pDepthStencilAttachment = &(VkAttachmentReference) {
 819                                                         .attachment = VK_ATTACHMENT_UNUSED,
 820                                                         .layout = layout,
 821                                                 },
 822                                                 .preserveAttachmentCount = 0,
 823                                                 .pPreserveAttachments = NULL,
 824                                                 },
 825                                                 .dependencyCount = 2,
 826                                                 .pDependencies = (VkSubpassDependency[]) {
 827                                                         {
 828                                                                 .srcSubpass = VK_SUBPASS_EXTERNAL,
 829                                                                 .dstSubpass = 0,
 830                                                                 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 831                                                                 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 832                                                                 .srcAccessMask = 0,
 833                                                                 .dstAccessMask = 0,
 834                                                                 .dependencyFlags = 0
 835                                                         },
 836                                                         {
 837                                                                 .srcSubpass = 0,
 838                                                                 .dstSubpass = VK_SUBPASS_EXTERNAL,
 839                                                                 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 840                                                                 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 841                                                                 .srcAccessMask = 0,
 842                                                                 .dstAccessMask = 0,
 843                                                                 .dependencyFlags = 0
 844                                                         }
 845                                                 },
 846                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
 847                 }
 848         }
 849
 850         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 851                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 852                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 853                 .pStages = pipeline_shader_stages,
 854                 .pVertexInputState = vi_create_info,
 855                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 856                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 857                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 858                         .primitiveRestartEnable = false,
 859                 },
 860                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 861                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 862                         .viewportCount = 1,
 863                         .scissorCount = 1,
 864                 },
 865                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 866                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 867                         .rasterizerDiscardEnable = false,
 868                         .polygonMode = VK_POLYGON_MODE_FILL,
 869                         .cullMode = VK_CULL_MODE_NONE,
 870                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
 871                 },
 872                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
 873                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 874                         .rasterizationSamples = 1 << log2_samples,
 875                         .sampleShadingEnable = log2_samples > 1,
 876                         .minSampleShading = 1.0,
 877                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
 878                 },
 879                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 880                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 881                         .attachmentCount = 1,
 882                         .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 883                                 { .colorWriteMask =
 884                                   VK_COLOR_COMPONENT_A_BIT |
 885                                   VK_COLOR_COMPONENT_R_BIT |
 886                                   VK_COLOR_COMPONENT_G_BIT |
 887                                   VK_COLOR_COMPONENT_B_BIT },
 888                         }
 889                 },
 890                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 891                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 892                         .dynamicStateCount = 9,
 893                         .pDynamicStates = (VkDynamicState[]) {
 894                                 VK_DYNAMIC_STATE_VIEWPORT,
 895                                 VK_DYNAMIC_STATE_SCISSOR,
 896                                 VK_DYNAMIC_STATE_LINE_WIDTH,
 897                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
 898                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
 899                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
 900                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
 901                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
 902                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
 903                         },
 904                 },
 905                 .flags = 0,
 906                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 907                 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
 908                 .subpass = 0,
 909         };
 910
 911         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
 912                 .use_rectlist = true
 913         };
 914
 915         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
 916                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 917                                                &vk_pipeline_info, &radv_pipeline_info,
 918                                                &device->meta_state.alloc,
 919                                                &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
 920
 921
 922         ralloc_free(vs.nir);
 923         ralloc_free(fs.nir);
 924
 925         mtx_unlock(&device->meta_state.mtx);
 926         return result;
 927 }
 928
 929 static VkResult
 930 blit2d_init_depth_only_pipeline(struct radv_device *device,
 931                                 enum blit2d_src_type src_type,
 932                                 uint32_t log2_samples)
 933 {
 934         VkResult result;
 935         const char *name;
 936
 937         mtx_lock(&device->meta_state.mtx);
 938         if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
 939                 mtx_unlock(&device->meta_state.mtx);
 940                 return VK_SUCCESS;
 941         }
 942
 943         texel_fetch_build_func src_func;
 944         switch(src_type) {
 945         case BLIT2D_SRC_TYPE_IMAGE:
 946                 src_func = build_nir_texel_fetch;
 947                 name = "meta_blit2d_depth_image_fs";
 948                 break;
 949         case BLIT2D_SRC_TYPE_IMAGE_3D:
 950                 src_func = build_nir_texel_fetch;
 951                 name = "meta_blit3d_depth_image_fs";
 952                 break;
 953         case BLIT2D_SRC_TYPE_BUFFER:
 954                 src_func = build_nir_buffer_fetch;
 955                 name = "meta_blit2d_depth_buffer_fs";
 956                 break;
 957         default:
 958                 unreachable("unknown blit src type\n");
 959                 break;
 960         }
 961
 962         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 963         struct radv_shader_module fs = { .nir = NULL };
 964
 965         fs.nir = build_nir_copy_fragment_shader_depth(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 966         vi_create_info = &normal_vi_create_info;
 967
 968         struct radv_shader_module vs = {
 969                 .nir = build_nir_vertex_shader(),
 970         };
 971
 972         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 973                 {
 974                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 975                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 976                         .module = radv_shader_module_to_handle(&vs),
 977                         .pName = "main",
 978                         .pSpecializationInfo = NULL
 979                 }, {
 980                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 981                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 982                         .module = radv_shader_module_to_handle(&fs),
 983                         .pName = "main",
 984                         .pSpecializationInfo = NULL
 985                 },
 986         };
 987
 988         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
 989                 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
 990                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
 991                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 992                                                        &(VkRenderPassCreateInfo) {
 993                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 994                                                                .attachmentCount = 1,
 995                                                                .pAttachments = &(VkAttachmentDescription) {
 996                                                                        .format = VK_FORMAT_D32_SFLOAT,
 997                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 998                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 999                                                                        .initialLayout = layout,
1000                                                                        .finalLayout = layout,
1001                                                                },
1002                                                                .subpassCount = 1,
1003                                                                .pSubpasses = &(VkSubpassDescription) {
1004                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1005                                                                        .inputAttachmentCount = 0,
1006                                                                        .colorAttachmentCount = 0,
1007                                                                        .pColorAttachments = NULL,
1008                                                                        .pResolveAttachments = NULL,
1009                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
1010                                                                                .attachment = 0,
1011                                                                                .layout = layout,
1012                                                                        },
1013                                                                        .preserveAttachmentCount = 0,
1014                                                                        .pPreserveAttachments = NULL,
1015                                                                },
1016                                                                .dependencyCount = 2,
1017                                                                .pDependencies = (VkSubpassDependency[]) {
1018                                                                 {
1019                                                                         .srcSubpass = VK_SUBPASS_EXTERNAL,
1020                                                                         .dstSubpass = 0,
1021                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1022                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1023                                                                         .srcAccessMask = 0,
1024                                                                         .dstAccessMask = 0,
1025                                                                         .dependencyFlags = 0
1026                                                                 },
1027                                                                 {
1028                                                                         .srcSubpass = 0,
1029                                                                         .dstSubpass = VK_SUBPASS_EXTERNAL,
1030                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1031                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1032                                                                         .srcAccessMask = 0,
1033                                                                         .dstAccessMask = 0,
1034                                                                         .dependencyFlags = 0
1035                                                                 }
1036                                                         },
1037                                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
1038                 }
1039         }
1040
1041         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1042                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1043                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1044                 .pStages = pipeline_shader_stages,
1045                 .pVertexInputState = vi_create_info,
1046                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1047                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1048                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1049                         .primitiveRestartEnable = false,
1050                 },
1051                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1052                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1053                         .viewportCount = 1,
1054                         .scissorCount = 1,
1055                 },
1056                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1057                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1058                         .rasterizerDiscardEnable = false,
1059                         .polygonMode = VK_POLYGON_MODE_FILL,
1060                         .cullMode = VK_CULL_MODE_NONE,
1061                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1062                 },
1063                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1064                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1065                         .rasterizationSamples = 1 << log2_samples,
1066                         .sampleShadingEnable = false,
1067                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1068                 },
1069                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1070                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1071                         .attachmentCount = 0,
1072                         .pAttachments = NULL,
1073                 },
1074                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1075                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1076                         .depthTestEnable = true,
1077                         .depthWriteEnable = true,
1078                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1079                 },
1080                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1081                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1082                         .dynamicStateCount = 9,
1083                         .pDynamicStates = (VkDynamicState[]) {
1084                                 VK_DYNAMIC_STATE_VIEWPORT,
1085                                 VK_DYNAMIC_STATE_SCISSOR,
1086                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1087                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1088                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1089                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1090                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1091                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1092                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1093                         },
1094                 },
1095                 .flags = 0,
1096                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1097                 .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1098                 .subpass = 0,
1099         };
1100
1101         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1102                 .use_rectlist = true
1103         };
1104
1105         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1106                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1107                                                &vk_pipeline_info, &radv_pipeline_info,
1108                                                &device->meta_state.alloc,
1109                                                &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1110
1111
1112         ralloc_free(vs.nir);
1113         ralloc_free(fs.nir);
1114
1115         mtx_unlock(&device->meta_state.mtx);
1116         return result;
1117 }
1118
1119 static VkResult
1120 blit2d_init_stencil_only_pipeline(struct radv_device *device,
1121                                   enum blit2d_src_type src_type,
1122                                   uint32_t log2_samples)
1123 {
1124         VkResult result;
1125         const char *name;
1126
1127         mtx_lock(&device->meta_state.mtx);
1128         if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1129                 mtx_unlock(&device->meta_state.mtx);
1130                 return VK_SUCCESS;
1131         }
1132
1133         texel_fetch_build_func src_func;
1134         switch(src_type) {
1135         case BLIT2D_SRC_TYPE_IMAGE:
1136                 src_func = build_nir_texel_fetch;
1137                 name = "meta_blit2d_stencil_image_fs";
1138                 break;
1139         case BLIT2D_SRC_TYPE_IMAGE_3D:
1140                 src_func = build_nir_texel_fetch;
1141                 name = "meta_blit3d_stencil_image_fs";
1142                 break;
1143         case BLIT2D_SRC_TYPE_BUFFER:
1144                 src_func = build_nir_buffer_fetch;
1145                 name = "meta_blit2d_stencil_buffer_fs";
1146                 break;
1147         default:
1148                 unreachable("unknown blit src type\n");
1149                 break;
1150         }
1151
1152         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1153         struct radv_shader_module fs = { .nir = NULL };
1154
1155         fs.nir = build_nir_copy_fragment_shader_stencil(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1156         vi_create_info = &normal_vi_create_info;
1157
1158         struct radv_shader_module vs = {
1159                 .nir = build_nir_vertex_shader(),
1160         };
1161
1162         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1163                 {
1164                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1165                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
1166                         .module = radv_shader_module_to_handle(&vs),
1167                         .pName = "main",
1168                         .pSpecializationInfo = NULL
1169                 }, {
1170                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1171                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1172                         .module = radv_shader_module_to_handle(&fs),
1173                         .pName = "main",
1174                         .pSpecializationInfo = NULL
1175                 },
1176         };
1177
1178         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1179                 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1180                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1181                         result = radv_CreateRenderPass(radv_device_to_handle(device),
1182                                                        &(VkRenderPassCreateInfo) {
1183                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
1184                                                                .attachmentCount = 1,
1185                                                                .pAttachments = &(VkAttachmentDescription) {
1186                                                                        .format = VK_FORMAT_S8_UINT,
1187                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1188                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1189                                                                        .initialLayout = layout,
1190                                                                        .finalLayout = layout,
1191                                                                },
1192                                                                .subpassCount = 1,
1193                                                                .pSubpasses = &(VkSubpassDescription) {
1194                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1195                                                                        .inputAttachmentCount = 0,
1196                                                                        .colorAttachmentCount = 0,
1197                                                                        .pColorAttachments = NULL,
1198                                                                        .pResolveAttachments = NULL,
1199                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
1200                                                                                .attachment = 0,
1201                                                                                .layout = layout,
1202                                                                        },
1203                                                                        .preserveAttachmentCount = 0,
1204                                                                        .pPreserveAttachments = NULL,
1205                                                                },
1206                                                                .dependencyCount = 2,
1207                                                                .pDependencies = (VkSubpassDependency[]) {
1208                                                                 {
1209                                                                         .srcSubpass = VK_SUBPASS_EXTERNAL,
1210                                                                         .dstSubpass = 0,
1211                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1212                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1213                                                                         .srcAccessMask = 0,
1214                                                                         .dstAccessMask = 0,
1215                                                                         .dependencyFlags = 0
1216                                                                 },
1217                                                                 {
1218                                                                         .srcSubpass = 0,
1219                                                                         .dstSubpass = VK_SUBPASS_EXTERNAL,
1220                                                                         .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1221                                                                         .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1222                                                                         .srcAccessMask = 0,
1223                                                                         .dstAccessMask = 0,
1224                                                                         .dependencyFlags = 0
1225                                                                 }
1226                                                         },
1227                                                        }, &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1228                 }
1229         }
1230
1231         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1232                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1233                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1234                 .pStages = pipeline_shader_stages,
1235                 .pVertexInputState = vi_create_info,
1236                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1237                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1238                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1239                         .primitiveRestartEnable = false,
1240                 },
1241                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1242                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1243                         .viewportCount = 1,
1244                         .scissorCount = 1,
1245                 },
1246                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1247                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1248                         .rasterizerDiscardEnable = false,
1249                         .polygonMode = VK_POLYGON_MODE_FILL,
1250                         .cullMode = VK_CULL_MODE_NONE,
1251                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1252                 },
1253                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1254                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1255                         .rasterizationSamples = 1 << log2_samples,
1256                         .sampleShadingEnable = false,
1257                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1258                 },
1259                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1260                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1261                         .attachmentCount = 0,
1262                         .pAttachments = NULL,
1263                 },
1264                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1265                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1266                         .depthTestEnable = false,
1267                         .depthWriteEnable = false,
1268                         .stencilTestEnable = true,
1269                         .front = {
1270                                 .failOp = VK_STENCIL_OP_REPLACE,
1271                                 .passOp = VK_STENCIL_OP_REPLACE,
1272                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1273                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1274                                 .compareMask = 0xff,
1275                                 .writeMask = 0xff,
1276                                 .reference = 0
1277                         },
1278                         .back = {
1279                                 .failOp = VK_STENCIL_OP_REPLACE,
1280                                 .passOp = VK_STENCIL_OP_REPLACE,
1281                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1282                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1283                                 .compareMask = 0xff,
1284                                 .writeMask = 0xff,
1285                                 .reference = 0
1286                         },
1287                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1288                 },
1289                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1290                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1291                         .dynamicStateCount = 6,
1292                         .pDynamicStates = (VkDynamicState[]) {
1293                                 VK_DYNAMIC_STATE_VIEWPORT,
1294                                 VK_DYNAMIC_STATE_SCISSOR,
1295                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1296                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1297                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1298                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1299                         },
1300                 },
1301                 .flags = 0,
1302                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1303                 .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1304                 .subpass = 0,
1305         };
1306
1307         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1308                 .use_rectlist = true
1309         };
1310
1311         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1312                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1313                                                &vk_pipeline_info, &radv_pipeline_info,
1314                                                &device->meta_state.alloc,
1315                                                &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1316
1317
1318         ralloc_free(vs.nir);
1319         ralloc_free(fs.nir);
1320
1321         mtx_unlock(&device->meta_state.mtx);
1322         return result;
1323 }
1324
1325 static VkResult
1326 meta_blit2d_create_pipe_layout(struct radv_device *device,
1327                                int idx,
1328                                uint32_t log2_samples)
1329 {
1330         VkResult result;
1331         VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER) ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1332         const VkPushConstantRange push_constant_ranges[] = {
1333                 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1334                 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1335         };
1336         int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1337
1338         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
1339                                                 &(VkDescriptorSetLayoutCreateInfo) {
1340                                                         .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1341                                                         .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1342                                                         .bindingCount = 1,
1343                                                         .pBindings = (VkDescriptorSetLayoutBinding[]) {
1344                                                         {
1345                                                                 .binding = 0,
1346                                                                 .descriptorType = desc_type,
1347                                                                 .descriptorCount = 1,
1348                                                                 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1349                                                                 .pImmutableSamplers = NULL
1350                                                         },
1351                                                         }
1352                                                 }, &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1353         if (result != VK_SUCCESS)
1354                 goto fail;
1355
1356         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
1357                                            &(VkPipelineLayoutCreateInfo) {
1358                                                    .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1359                                                            .setLayoutCount = 1,
1360                                                            .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1361                                                            .pushConstantRangeCount = num_push_constant_range,
1362                                                            .pPushConstantRanges = push_constant_ranges,
1363                                                            },
1364                                            &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1365         if (result != VK_SUCCESS)
1366                 goto fail;
1367         return VK_SUCCESS;
1368 fail:
1369         return result;
1370 }
1371
1372 VkResult
1373 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1374 {
1375         VkResult result;
1376         bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1377
1378         for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; log2_samples++) {
1379                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1380                         if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1381                                 continue;
1382
1383                         /* Don't need to handle copies between buffers and multisample images. */
1384                         if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1385                                 continue;
1386
1387                         result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1388                         if (result != VK_SUCCESS)
1389                                 goto fail;
1390
1391                         if (on_demand)
1392                                 continue;
1393
1394                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1395                                 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j], log2_samples);
1396                                 if (result != VK_SUCCESS)
1397                                         goto fail;
1398                         }
1399
1400                         result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1401                         if (result != VK_SUCCESS)
1402                                 goto fail;
1403
1404                         result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1405                         if (result != VK_SUCCESS)
1406                                 goto fail;
1407                 }
1408         }
1409
1410         return VK_SUCCESS;
1411
1412 fail:
1413         radv_device_finish_meta_blit2d_state(device);
1414         return result;
1415 }