src/amd/vulkan/radv_meta_blit2d.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  *
   4  * based on anv driver:
   5  * Copyright © 2016 Intel Corporation
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24  * IN THE SOFTWARE.
  25  */
  26
  27 #include "radv_meta.h"
  28 #include "nir/nir_builder.h"
  29 #include "vk_format.h"
  30
  31 enum blit2d_src_type {
  32         BLIT2D_SRC_TYPE_IMAGE,
  33         BLIT2D_SRC_TYPE_IMAGE_3D,
  34         BLIT2D_SRC_TYPE_BUFFER,
  35         BLIT2D_NUM_SRC_TYPES,
  36 };
  37
  38 static VkResult
  39 blit2d_init_color_pipeline(struct radv_device *device,
  40                            enum blit2d_src_type src_type,
  41                            VkFormat format,
  42                            uint32_t log2_samples);
  43
  44 static VkResult
  45 blit2d_init_depth_only_pipeline(struct radv_device *device,
  46                                 enum blit2d_src_type src_type,
  47                                 uint32_t log2_samples);
  48
  49 static VkResult
  50 blit2d_init_stencil_only_pipeline(struct radv_device *device,
  51                                   enum blit2d_src_type src_type,
  52                                   uint32_t log2_samples);
  53
  54 static void
  55 create_iview(struct radv_cmd_buffer *cmd_buffer,
  56              struct radv_meta_blit2d_surf *surf,
  57              struct radv_image_view *iview, VkFormat depth_format,
  58               VkImageAspectFlagBits aspects)
  59 {
  60         VkFormat format;
  61         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
  62                 radv_meta_get_view_type(surf->image);
  63
  64         if (depth_format)
  65                 format = depth_format;
  66         else
  67                 format = surf->format;
  68
  69         radv_image_view_init(iview, cmd_buffer->device,
  70                              &(VkImageViewCreateInfo) {
  71                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
  72                                              .image = radv_image_to_handle(surf->image),
  73                                              .viewType = view_type,
  74                                              .format = format,
  75                                              .subresourceRange = {
  76                                              .aspectMask = aspects,
  77                                              .baseMipLevel = surf->level,
  78                                              .levelCount = 1,
  79                                              .baseArrayLayer = surf->layer,
  80                                              .layerCount = 1
  81                                      },
  82                              });
  83 }
  84
  85 static void
  86 create_bview(struct radv_cmd_buffer *cmd_buffer,
  87              struct radv_meta_blit2d_buffer *src,
  88              struct radv_buffer_view *bview, VkFormat depth_format)
  89 {
  90         VkFormat format;
  91
  92         if (depth_format)
  93                 format = depth_format;
  94         else
  95                 format = src->format;
  96         radv_buffer_view_init(bview, cmd_buffer->device,
  97                               &(VkBufferViewCreateInfo) {
  98                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
  99                                       .flags = 0,
 100                                       .buffer = radv_buffer_to_handle(src->buffer),
 101                                       .format = format,
 102                                       .offset = src->offset,
 103                                       .range = VK_WHOLE_SIZE,
 104                               });
 105
 106 }
 107
 108 struct blit2d_src_temps {
 109         struct radv_image_view iview;
 110         struct radv_buffer_view bview;
 111 };
 112
 113 static void
 114 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer,
 115                 struct radv_meta_blit2d_surf *src_img,
 116                 struct radv_meta_blit2d_buffer *src_buf,
 117                 struct blit2d_src_temps *tmp,
 118                 enum blit2d_src_type src_type, VkFormat depth_format,
 119                 VkImageAspectFlagBits aspects,
 120                 uint32_t log2_samples)
 121 {
 122         struct radv_device *device = cmd_buffer->device;
 123
 124         if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
 125                 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
 126
 127                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 128                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 129                                               0, /* set */
 130                                               1, /* descriptorWriteCount */
 131                                               (VkWriteDescriptorSet[]) {
 132                                                       {
 133                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 134                                                               .dstBinding = 0,
 135                                                               .dstArrayElement = 0,
 136                                                               .descriptorCount = 1,
 137                                                               .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
 138                                                               .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(&tmp->bview) }
 139                                                       }
 140                                               });
 141
 142                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 143                                       device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 144                                       VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 145                                       &src_buf->pitch);
 146         } else {
 147                 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
 148
 149                 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
 150                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 151                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 152                                               VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 153                                               &src_img->layer);
 154
 155                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 156                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 157                                               0, /* set */
 158                                               1, /* descriptorWriteCount */
 159                                               (VkWriteDescriptorSet[]) {
 160                                                       {
 161                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 162                                                               .dstBinding = 0,
 163                                                               .dstArrayElement = 0,
 164                                                               .descriptorCount = 1,
 165                                                               .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 166                                                               .pImageInfo = (VkDescriptorImageInfo[]) {
 167                                                                       {
 168                                                                               .sampler = VK_NULL_HANDLE,
 169                                                                               .imageView = radv_image_view_to_handle(&tmp->iview),
 170                                                                               .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 171                                                                       },
 172                                                               }
 173                                                       }
 174                                               });
 175         }
 176 }
 177
 178 struct blit2d_dst_temps {
 179         VkImage image;
 180         struct radv_image_view iview;
 181         VkFramebuffer fb;
 182 };
 183
 184 static void
 185 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer,
 186                 struct radv_meta_blit2d_surf *dst,
 187                 uint32_t width,
 188                 uint32_t height,
 189                 VkFormat depth_format,
 190                 struct blit2d_dst_temps *tmp,
 191                 VkImageAspectFlagBits aspects)
 192 {
 193         create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
 194
 195         radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
 196                                &(VkFramebufferCreateInfo) {
 197                                        .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
 198                                                .attachmentCount = 1,
 199                                                .pAttachments = (VkImageView[]) {
 200                                                radv_image_view_to_handle(&tmp->iview),
 201                                        },
 202                                        .width = width,
 203                                        .height = height,
 204                                        .layers = 1
 205                                 }, &cmd_buffer->pool->alloc, &tmp->fb);
 206 }
 207
 208 static void
 209 bind_pipeline(struct radv_cmd_buffer *cmd_buffer,
 210               enum blit2d_src_type src_type, unsigned fs_key,
 211               uint32_t log2_samples)
 212 {
 213         VkPipeline pipeline =
 214                 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
 215
 216         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 217                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 218 }
 219
 220 static void
 221 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer,
 222                     enum blit2d_src_type src_type,
 223                     uint32_t log2_samples)
 224 {
 225         VkPipeline pipeline =
 226                 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
 227
 228         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 229                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 230 }
 231
 232 static void
 233 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer,
 234                       enum blit2d_src_type src_type,
 235                       uint32_t log2_samples)
 236 {
 237         VkPipeline pipeline =
 238                 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
 239
 240         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 241                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 242 }
 243
 244 static void
 245 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
 246                             struct radv_meta_blit2d_surf *src_img,
 247                             struct radv_meta_blit2d_buffer *src_buf,
 248                             struct radv_meta_blit2d_surf *dst,
 249                             unsigned num_rects,
 250                             struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
 251                             uint32_t log2_samples)
 252 {
 253         struct radv_device *device = cmd_buffer->device;
 254
 255         for (unsigned r = 0; r < num_rects; ++r) {
 256                 unsigned i;
 257                 for_each_bit(i, dst->aspect_mask) {
 258                         unsigned aspect_mask = 1u << i;
 259                         unsigned src_aspect_mask = aspect_mask;
 260                         VkFormat depth_format = 0;
 261                         if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
 262                                 depth_format = vk_format_stencil_only(dst->image->vk_format);
 263                         else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
 264                                 depth_format = vk_format_depth_only(dst->image->vk_format);
 265                         else if (src_img)
 266                                 src_aspect_mask = src_img->aspect_mask;
 267
 268                         struct blit2d_src_temps src_temps;
 269                         blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format, src_aspect_mask, log2_samples);
 270
 271                         struct blit2d_dst_temps dst_temps;
 272                         blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
 273                                         rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
 274
 275                         float vertex_push_constants[4] = {
 276                                 rects[r].src_x,
 277                                 rects[r].src_y,
 278                                 rects[r].src_x + rects[r].width,
 279                                 rects[r].src_y + rects[r].height,
 280                         };
 281
 282                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 283                                         device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 284                                         VK_SHADER_STAGE_VERTEX_BIT, 0, 16,
 285                                         vertex_push_constants);
 286
 287                         if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT ||
 288                             aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
 289                             aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT ||
 290                             aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
 291                                 unsigned fs_key = radv_format_meta_fs_key(dst_temps.iview.vk_format);
 292                                 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
 293
 294                                 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] == VK_NULL_HANDLE) {
 295                                         VkResult ret = blit2d_init_color_pipeline(device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
 296                                         if (ret != VK_SUCCESS) {
 297                                                 cmd_buffer->record_result = ret;
 298                                                 goto fail_pipeline;
 299                                         }
 300                                 }
 301
 302                                 radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
 303                                                         &(VkRenderPassBeginInfo) {
 304                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 305                                                                         .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
 306                                                                         .framebuffer = dst_temps.fb,
 307                                                                         .renderArea = {
 308                                                                         .offset = { rects[r].dst_x, rects[r].dst_y, },
 309                                                                         .extent = { rects[r].width, rects[r].height },
 310                                                                 },
 311                                                                         .clearValueCount = 0,
 312                                                                                 .pClearValues = NULL,
 313                                                                                 }, VK_SUBPASS_CONTENTS_INLINE);
 314
 315
 316                                 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
 317                         } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 318                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 319
 320                                 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] == VK_NULL_HANDLE) {
 321                                         VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
 322                                         if (ret != VK_SUCCESS) {
 323                                                 cmd_buffer->record_result = ret;
 324                                                 goto fail_pipeline;
 325                                         }
 326                                 }
 327
 328                                 radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
 329                                                         &(VkRenderPassBeginInfo) {
 330                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 331                                                                         .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
 332                                                                         .framebuffer = dst_temps.fb,
 333                                                                         .renderArea = {
 334                                                                         .offset = { rects[r].dst_x, rects[r].dst_y, },
 335                                                                         .extent = { rects[r].width, rects[r].height },
 336                                                                 },
 337                                                                         .clearValueCount = 0,
 338                                                                                 .pClearValues = NULL,
 339                                                                                 }, VK_SUBPASS_CONTENTS_INLINE);
 340
 341
 342                                 bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
 343
 344                         } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 345                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 346
 347                                 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] == VK_NULL_HANDLE) {
 348                                         VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
 349                                         if (ret != VK_SUCCESS) {
 350                                                 cmd_buffer->record_result = ret;
 351                                                 goto fail_pipeline;
 352                                         }
 353                                 }
 354
 355                                 radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
 356                                                         &(VkRenderPassBeginInfo) {
 357                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 358                                                                         .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
 359                                                                         .framebuffer = dst_temps.fb,
 360                                                                         .renderArea = {
 361                                                                         .offset = { rects[r].dst_x, rects[r].dst_y, },
 362                                                                         .extent = { rects[r].width, rects[r].height },
 363                                                                 },
 364                                                                         .clearValueCount = 0,
 365                                                                                 .pClearValues = NULL,
 366                                                                                 }, VK_SUBPASS_CONTENTS_INLINE);
 367
 368
 369                                 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
 370                         } else
 371                                 unreachable("Processing blit2d with multiple aspects.");
 372
 373                         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 374                                 .x = rects[r].dst_x,
 375                                 .y = rects[r].dst_y,
 376                                 .width = rects[r].width,
 377                                 .height = rects[r].height,
 378                                 .minDepth = 0.0f,
 379                                 .maxDepth = 1.0f
 380                         });
 381
 382                         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 383                                 .offset = (VkOffset2D) { rects[r].dst_x, rects[r].dst_y },
 384                                 .extent = (VkExtent2D) { rects[r].width, rects[r].height },
 385                         });
 386
 387
 388
 389                         radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
 390                         radv_CmdEndRenderPass(radv_cmd_buffer_to_handle(cmd_buffer));
 391
 392 fail_pipeline:
 393                         /* At the point where we emit the draw call, all data from the
 394                         * descriptor sets, etc. has been used.  We are free to delete it.
 395                         */
 396                         radv_DestroyFramebuffer(radv_device_to_handle(device),
 397                                                 dst_temps.fb,
 398                                                 &cmd_buffer->pool->alloc);
 399                 }
 400         }
 401 }
 402
 403 void
 404 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer,
 405                  struct radv_meta_blit2d_surf *src_img,
 406                  struct radv_meta_blit2d_buffer *src_buf,
 407                  struct radv_meta_blit2d_surf *dst,
 408                  unsigned num_rects,
 409                  struct radv_meta_blit2d_rect *rects)
 410 {
 411         bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
 412                 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
 413         enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER :
 414                 use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D : BLIT2D_SRC_TYPE_IMAGE;
 415         radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst,
 416                                     num_rects, rects, src_type,
 417                                     src_img ? util_logbase2(src_img->image->info.samples) : 0);
 418 }
 419
 420 static nir_shader *
 421 build_nir_vertex_shader(void)
 422 {
 423         const struct glsl_type *vec4 = glsl_vec4_type();
 424         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 425         nir_builder b;
 426
 427         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 428         b.shader->info.name = ralloc_strdup(b.shader, "meta_blit2d_vs");
 429
 430         nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 431                                                     vec4, "gl_Position");
 432         pos_out->data.location = VARYING_SLOT_POS;
 433
 434         nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 435                                                         vec2, "v_tex_pos");
 436         tex_pos_out->data.location = VARYING_SLOT_VAR0;
 437         tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
 438
 439         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 440         nir_store_var(&b, pos_out, outvec, 0xf);
 441
 442         nir_intrinsic_instr *src_box = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 443         src_box->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 444         nir_intrinsic_set_base(src_box, 0);
 445         nir_intrinsic_set_range(src_box, 16);
 446         src_box->num_components = 4;
 447         nir_ssa_dest_init(&src_box->instr, &src_box->dest, 4, 32, "src_box");
 448         nir_builder_instr_insert(&b, &src_box->instr);
 449
 450         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_vertex_id_zero_base);
 451         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 452         nir_builder_instr_insert(&b, &vertex_id->instr);
 453
 454         /* vertex 0 - src_x, src_y */
 455         /* vertex 1 - src_x, src_y+h */
 456         /* vertex 2 - src_x+w, src_y */
 457         /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
 458            channel 1 is vertex id != 1 ? src_y : src_y + w */
 459
 460         nir_ssa_def *c0cmp = nir_ine(&b, &vertex_id->dest.ssa,
 461                                      nir_imm_int(&b, 2));
 462         nir_ssa_def *c1cmp = nir_ine(&b, &vertex_id->dest.ssa,
 463                                      nir_imm_int(&b, 1));
 464
 465         nir_ssa_def *comp[2];
 466         comp[0] = nir_bcsel(&b, c0cmp,
 467                             nir_channel(&b, &src_box->dest.ssa, 0),
 468                             nir_channel(&b, &src_box->dest.ssa, 2));
 469
 470         comp[1] = nir_bcsel(&b, c1cmp,
 471                             nir_channel(&b, &src_box->dest.ssa, 1),
 472                             nir_channel(&b, &src_box->dest.ssa, 3));
 473         nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
 474         nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
 475         return b.shader;
 476 }
 477
 478 typedef nir_ssa_def* (*texel_fetch_build_func)(struct nir_builder *,
 479                                                struct radv_device *,
 480                                                nir_ssa_def *, bool, bool);
 481
 482 static nir_ssa_def *
 483 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device,
 484                       nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 485 {
 486         enum glsl_sampler_dim dim =
 487                 is_3d ? GLSL_SAMPLER_DIM_3D : is_multisampled ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D;
 488         const struct glsl_type *sampler_type =
 489                 glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
 490         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 491                                                     sampler_type, "s_tex");
 492         sampler->data.descriptor_set = 0;
 493         sampler->data.binding = 0;
 494
 495         nir_ssa_def *tex_pos_3d = NULL;
 496         nir_intrinsic_instr *sample_idx = NULL;
 497         if (is_3d) {
 498                 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 499                 nir_intrinsic_set_base(layer, 16);
 500                 nir_intrinsic_set_range(layer, 4);
 501                 layer->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 502                 layer->num_components = 1;
 503                 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
 504                 nir_builder_instr_insert(b, &layer->instr);
 505
 506                 nir_ssa_def *chans[3];
 507                 chans[0] = nir_channel(b, tex_pos, 0);
 508                 chans[1] = nir_channel(b, tex_pos, 1);
 509                 chans[2] = &layer->dest.ssa;
 510                 tex_pos_3d = nir_vec(b, chans, 3);
 511         }
 512         if (is_multisampled) {
 513                 sample_idx = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_sample_id);
 514                 sample_idx->num_components = 1;
 515                 nir_ssa_dest_init(&sample_idx->instr, &sample_idx->dest, 1, 32, "sample_idx");
 516                 nir_builder_instr_insert(b, &sample_idx->instr);
 517         }
 518
 519         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 520
 521         nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
 522         tex->sampler_dim = dim;
 523         tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
 524         tex->src[0].src_type = nir_tex_src_coord;
 525         tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
 526         tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
 527         tex->src[1].src = nir_src_for_ssa(is_multisampled ? &sample_idx->dest.ssa : nir_imm_int(b, 0));
 528         tex->src[2].src_type = nir_tex_src_texture_deref;
 529         tex->src[2].src = nir_src_for_ssa(tex_deref);
 530         if (is_multisampled) {
 531                 tex->src[3].src_type = nir_tex_src_lod;
 532                 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
 533         }
 534         tex->dest_type = nir_type_uint;
 535         tex->is_array = false;
 536         tex->coord_components = is_3d ? 3 : 2;
 537
 538         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 539         nir_builder_instr_insert(b, &tex->instr);
 540
 541         return &tex->dest.ssa;
 542 }
 543
 544
 545 static nir_ssa_def *
 546 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device,
 547                        nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 548 {
 549         const struct glsl_type *sampler_type =
 550                 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
 551         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 552                                                     sampler_type, "s_tex");
 553         sampler->data.descriptor_set = 0;
 554         sampler->data.binding = 0;
 555
 556         nir_intrinsic_instr *width = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 557         nir_intrinsic_set_base(width, 16);
 558         nir_intrinsic_set_range(width, 4);
 559         width->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 560         width->num_components = 1;
 561         nir_ssa_dest_init(&width->instr, &width->dest, 1, 32, "width");
 562         nir_builder_instr_insert(b, &width->instr);
 563
 564         nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
 565         nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
 566         pos_y = nir_imul(b, pos_y, &width->dest.ssa);
 567         pos_x = nir_iadd(b, pos_x, pos_y);
 568
 569         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 570
 571         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
 572         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 573         tex->op = nir_texop_txf;
 574         tex->src[0].src_type = nir_tex_src_coord;
 575         tex->src[0].src = nir_src_for_ssa(pos_x);
 576         tex->src[1].src_type = nir_tex_src_texture_deref;
 577         tex->src[1].src = nir_src_for_ssa(tex_deref);
 578         tex->dest_type = nir_type_uint;
 579         tex->is_array = false;
 580         tex->coord_components = 1;
 581
 582         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 583         nir_builder_instr_insert(b, &tex->instr);
 584
 585         return &tex->dest.ssa;
 586 }
 587
 588 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
 589         .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 590         .vertexBindingDescriptionCount = 0,
 591         .vertexAttributeDescriptionCount = 0,
 592 };
 593
 594 static nir_shader *
 595 build_nir_copy_fragment_shader(struct radv_device *device,
 596                                texel_fetch_build_func txf_func, const char* name, bool is_3d,
 597                                bool is_multisampled)
 598 {
 599         const struct glsl_type *vec4 = glsl_vec4_type();
 600         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 601         nir_builder b;
 602
 603         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 604         b.shader->info.name = ralloc_strdup(b.shader, name);
 605
 606         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 607                                                        vec2, "v_tex_pos");
 608         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 609
 610         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 611                                                       vec4, "f_color");
 612         color_out->data.location = FRAG_RESULT_DATA0;
 613
 614         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 615         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 616
 617         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 618         nir_store_var(&b, color_out, color, 0xf);
 619
 620         return b.shader;
 621 }
 622
 623 static nir_shader *
 624 build_nir_copy_fragment_shader_depth(struct radv_device *device,
 625                                      texel_fetch_build_func txf_func, const char* name, bool is_3d,
 626                                      bool is_multisampled)
 627 {
 628         const struct glsl_type *vec4 = glsl_vec4_type();
 629         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 630         nir_builder b;
 631
 632         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 633         b.shader->info.name = ralloc_strdup(b.shader, name);
 634
 635         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 636                                                        vec2, "v_tex_pos");
 637         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 638
 639         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 640                                                       vec4, "f_color");
 641         color_out->data.location = FRAG_RESULT_DEPTH;
 642
 643         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 644         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 645
 646         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 647         nir_store_var(&b, color_out, color, 0x1);
 648
 649         return b.shader;
 650 }
 651
 652 static nir_shader *
 653 build_nir_copy_fragment_shader_stencil(struct radv_device *device,
 654                                        texel_fetch_build_func txf_func, const char* name, bool is_3d,
 655                                        bool is_multisampled)
 656 {
 657         const struct glsl_type *vec4 = glsl_vec4_type();
 658         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 659         nir_builder b;
 660
 661         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 662         b.shader->info.name = ralloc_strdup(b.shader, name);
 663
 664         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 665                                                        vec2, "v_tex_pos");
 666         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 667
 668         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 669                                                       vec4, "f_color");
 670         color_out->data.location = FRAG_RESULT_STENCIL;
 671
 672         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 673         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 674
 675         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 676         nir_store_var(&b, color_out, color, 0x1);
 677
 678         return b.shader;
 679 }
 680
 681 void
 682 radv_device_finish_meta_blit2d_state(struct radv_device *device)
 683 {
 684         struct radv_meta_state *state = &device->meta_state;
 685
 686         for(unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 687                 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
 688                         radv_DestroyRenderPass(radv_device_to_handle(device),
 689                                                state->blit2d_render_passes[j][k],
 690                                                &state->alloc);
 691                 }
 692         }
 693
 694         for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT; j++) {
 695                 radv_DestroyRenderPass(radv_device_to_handle(device),
 696                                        state->blit2d_depth_only_rp[j], &state->alloc);
 697                 radv_DestroyRenderPass(radv_device_to_handle(device),
 698                                        state->blit2d_stencil_only_rp[j], &state->alloc);
 699         }
 700
 701         for (unsigned log2_samples = 0; log2_samples < 1 + MAX_SAMPLES_LOG2; ++log2_samples) {
 702                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
 703                         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 704                                                    state->blit2d[log2_samples].p_layouts[src],
 705                                                    &state->alloc);
 706                         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 707                                                         state->blit2d[log2_samples].ds_layouts[src],
 708                                                         &state->alloc);
 709
 710                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 711                                 radv_DestroyPipeline(radv_device_to_handle(device),
 712                                                      state->blit2d[log2_samples].pipelines[src][j],
 713                                                      &state->alloc);
 714                         }
 715
 716                         radv_DestroyPipeline(radv_device_to_handle(device),
 717                                              state->blit2d[log2_samples].depth_only_pipeline[src],
 718                                              &state->alloc);
 719                         radv_DestroyPipeline(radv_device_to_handle(device),
 720                                              state->blit2d[log2_samples].stencil_only_pipeline[src],
 721                                              &state->alloc);
 722                 }
 723         }
 724 }
 725
 726 static VkResult
 727 blit2d_init_color_pipeline(struct radv_device *device,
 728                            enum blit2d_src_type src_type,
 729                            VkFormat format,
 730                            uint32_t log2_samples)
 731 {
 732         VkResult result;
 733         unsigned fs_key = radv_format_meta_fs_key(format);
 734         const char *name;
 735
 736         mtx_lock(&device->meta_state.mtx);
 737         if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
 738                 mtx_unlock(&device->meta_state.mtx);
 739                 return VK_SUCCESS;
 740         }
 741
 742         texel_fetch_build_func src_func;
 743         switch(src_type) {
 744         case BLIT2D_SRC_TYPE_IMAGE:
 745                 src_func = build_nir_texel_fetch;
 746                 name = "meta_blit2d_image_fs";
 747                 break;
 748         case BLIT2D_SRC_TYPE_IMAGE_3D:
 749                 src_func = build_nir_texel_fetch;
 750                 name = "meta_blit3d_image_fs";
 751                 break;
 752         case BLIT2D_SRC_TYPE_BUFFER:
 753                 src_func = build_nir_buffer_fetch;
 754                 name = "meta_blit2d_buffer_fs";
 755                 break;
 756         default:
 757                 unreachable("unknown blit src type\n");
 758                 break;
 759         }
 760
 761         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 762         struct radv_shader_module fs = { .nir = NULL };
 763
 764
 765         fs.nir = build_nir_copy_fragment_shader(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 766         vi_create_info = &normal_vi_create_info;
 767
 768         struct radv_shader_module vs = {
 769                 .nir = build_nir_vertex_shader(),
 770         };
 771
 772         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 773                 {
 774                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 775                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 776                         .module = radv_shader_module_to_handle(&vs),
 777                         .pName = "main",
 778                         .pSpecializationInfo = NULL
 779                 }, {
 780                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 781                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 782                         .module = radv_shader_module_to_handle(&fs),
 783                         .pName = "main",
 784                         .pSpecializationInfo = NULL
 785                 },
 786         };
 787
 788         for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
 789                 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
 790                         VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
 791
 792                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 793                                                 &(VkRenderPassCreateInfo) {
 794                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 795                                                         .attachmentCount = 1,
 796                                                         .pAttachments = &(VkAttachmentDescription) {
 797                                                         .format = format,
 798                                                         .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 799                                                         .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 800                                                         .initialLayout = layout,
 801                                                         .finalLayout = layout,
 802                                                         },
 803                                                 .subpassCount = 1,
 804                                                 .pSubpasses = &(VkSubpassDescription) {
 805                                                         .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 806                                                         .inputAttachmentCount = 0,
 807                                                         .colorAttachmentCount = 1,
 808                                                         .pColorAttachments = &(VkAttachmentReference) {
 809                                                                 .attachment = 0,
 810                                                                 .layout = layout,
 811                                                                 },
 812                                                 .pResolveAttachments = NULL,
 813                                                 .pDepthStencilAttachment = &(VkAttachmentReference) {
 814                                                         .attachment = VK_ATTACHMENT_UNUSED,
 815                                                         .layout = layout,
 816                                                 },
 817                                                 .preserveAttachmentCount = 0,
 818                                                 .pPreserveAttachments = NULL,
 819                                                 },
 820                                                 .dependencyCount = 0,
 821                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
 822                 }
 823         }
 824
 825         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 826                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 827                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 828                 .pStages = pipeline_shader_stages,
 829                 .pVertexInputState = vi_create_info,
 830                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 831                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 832                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 833                         .primitiveRestartEnable = false,
 834                 },
 835                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 836                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 837                         .viewportCount = 1,
 838                         .scissorCount = 1,
 839                 },
 840                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 841                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 842                         .rasterizerDiscardEnable = false,
 843                         .polygonMode = VK_POLYGON_MODE_FILL,
 844                         .cullMode = VK_CULL_MODE_NONE,
 845                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
 846                 },
 847                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
 848                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 849                         .rasterizationSamples = 1 << log2_samples,
 850                         .sampleShadingEnable = log2_samples > 1,
 851                         .minSampleShading = 1.0,
 852                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
 853                 },
 854                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 855                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 856                         .attachmentCount = 1,
 857                         .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 858                                 { .colorWriteMask =
 859                                   VK_COLOR_COMPONENT_A_BIT |
 860                                   VK_COLOR_COMPONENT_R_BIT |
 861                                   VK_COLOR_COMPONENT_G_BIT |
 862                                   VK_COLOR_COMPONENT_B_BIT },
 863                         }
 864                 },
 865                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 866                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 867                         .dynamicStateCount = 9,
 868                         .pDynamicStates = (VkDynamicState[]) {
 869                                 VK_DYNAMIC_STATE_VIEWPORT,
 870                                 VK_DYNAMIC_STATE_SCISSOR,
 871                                 VK_DYNAMIC_STATE_LINE_WIDTH,
 872                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
 873                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
 874                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
 875                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
 876                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
 877                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
 878                         },
 879                 },
 880                 .flags = 0,
 881                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 882                 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
 883                 .subpass = 0,
 884         };
 885
 886         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
 887                 .use_rectlist = true
 888         };
 889
 890         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
 891                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 892                                                &vk_pipeline_info, &radv_pipeline_info,
 893                                                &device->meta_state.alloc,
 894                                                &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
 895
 896
 897         ralloc_free(vs.nir);
 898         ralloc_free(fs.nir);
 899
 900         mtx_unlock(&device->meta_state.mtx);
 901         return result;
 902 }
 903
 904 static VkResult
 905 blit2d_init_depth_only_pipeline(struct radv_device *device,
 906                                 enum blit2d_src_type src_type,
 907                                 uint32_t log2_samples)
 908 {
 909         VkResult result;
 910         const char *name;
 911
 912         mtx_lock(&device->meta_state.mtx);
 913         if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
 914                 mtx_unlock(&device->meta_state.mtx);
 915                 return VK_SUCCESS;
 916         }
 917
 918         texel_fetch_build_func src_func;
 919         switch(src_type) {
 920         case BLIT2D_SRC_TYPE_IMAGE:
 921                 src_func = build_nir_texel_fetch;
 922                 name = "meta_blit2d_depth_image_fs";
 923                 break;
 924         case BLIT2D_SRC_TYPE_IMAGE_3D:
 925                 src_func = build_nir_texel_fetch;
 926                 name = "meta_blit3d_depth_image_fs";
 927                 break;
 928         case BLIT2D_SRC_TYPE_BUFFER:
 929                 src_func = build_nir_buffer_fetch;
 930                 name = "meta_blit2d_depth_buffer_fs";
 931                 break;
 932         default:
 933                 unreachable("unknown blit src type\n");
 934                 break;
 935         }
 936
 937         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 938         struct radv_shader_module fs = { .nir = NULL };
 939
 940         fs.nir = build_nir_copy_fragment_shader_depth(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 941         vi_create_info = &normal_vi_create_info;
 942
 943         struct radv_shader_module vs = {
 944                 .nir = build_nir_vertex_shader(),
 945         };
 946
 947         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 948                 {
 949                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 950                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 951                         .module = radv_shader_module_to_handle(&vs),
 952                         .pName = "main",
 953                         .pSpecializationInfo = NULL
 954                 }, {
 955                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 956                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 957                         .module = radv_shader_module_to_handle(&fs),
 958                         .pName = "main",
 959                         .pSpecializationInfo = NULL
 960                 },
 961         };
 962
 963         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
 964                 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
 965                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
 966                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 967                                                        &(VkRenderPassCreateInfo) {
 968                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 969                                                                .attachmentCount = 1,
 970                                                                .pAttachments = &(VkAttachmentDescription) {
 971                                                                        .format = VK_FORMAT_D32_SFLOAT,
 972                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 973                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 974                                                                        .initialLayout = layout,
 975                                                                        .finalLayout = layout,
 976                                                                },
 977                                                                .subpassCount = 1,
 978                                                                .pSubpasses = &(VkSubpassDescription) {
 979                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 980                                                                        .inputAttachmentCount = 0,
 981                                                                        .colorAttachmentCount = 0,
 982                                                                        .pColorAttachments = NULL,
 983                                                                        .pResolveAttachments = NULL,
 984                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
 985                                                                                .attachment = 0,
 986                                                                                .layout = layout,
 987                                                                        },
 988                                                                        .preserveAttachmentCount = 0,
 989                                                                        .pPreserveAttachments = NULL,
 990                                                                },
 991                                                                .dependencyCount = 0,
 992                                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
 993                 }
 994         }
 995
 996         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 997                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 998                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 999                 .pStages = pipeline_shader_stages,
1000                 .pVertexInputState = vi_create_info,
1001                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1002                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1003                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1004                         .primitiveRestartEnable = false,
1005                 },
1006                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1007                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1008                         .viewportCount = 1,
1009                         .scissorCount = 1,
1010                 },
1011                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1012                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1013                         .rasterizerDiscardEnable = false,
1014                         .polygonMode = VK_POLYGON_MODE_FILL,
1015                         .cullMode = VK_CULL_MODE_NONE,
1016                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1017                 },
1018                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1019                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1020                         .rasterizationSamples = 1 << log2_samples,
1021                         .sampleShadingEnable = false,
1022                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1023                 },
1024                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1025                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1026                         .attachmentCount = 0,
1027                         .pAttachments = NULL,
1028                 },
1029                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1030                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1031                         .depthTestEnable = true,
1032                         .depthWriteEnable = true,
1033                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1034                 },
1035                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1036                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1037                         .dynamicStateCount = 9,
1038                         .pDynamicStates = (VkDynamicState[]) {
1039                                 VK_DYNAMIC_STATE_VIEWPORT,
1040                                 VK_DYNAMIC_STATE_SCISSOR,
1041                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1042                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1043                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1044                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1045                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1046                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1047                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1048                         },
1049                 },
1050                 .flags = 0,
1051                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1052                 .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1053                 .subpass = 0,
1054         };
1055
1056         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1057                 .use_rectlist = true
1058         };
1059
1060         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1061                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1062                                                &vk_pipeline_info, &radv_pipeline_info,
1063                                                &device->meta_state.alloc,
1064                                                &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1065
1066
1067         ralloc_free(vs.nir);
1068         ralloc_free(fs.nir);
1069
1070         mtx_unlock(&device->meta_state.mtx);
1071         return result;
1072 }
1073
1074 static VkResult
1075 blit2d_init_stencil_only_pipeline(struct radv_device *device,
1076                                   enum blit2d_src_type src_type,
1077                                   uint32_t log2_samples)
1078 {
1079         VkResult result;
1080         const char *name;
1081
1082         mtx_lock(&device->meta_state.mtx);
1083         if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1084                 mtx_unlock(&device->meta_state.mtx);
1085                 return VK_SUCCESS;
1086         }
1087
1088         texel_fetch_build_func src_func;
1089         switch(src_type) {
1090         case BLIT2D_SRC_TYPE_IMAGE:
1091                 src_func = build_nir_texel_fetch;
1092                 name = "meta_blit2d_stencil_image_fs";
1093                 break;
1094         case BLIT2D_SRC_TYPE_IMAGE_3D:
1095                 src_func = build_nir_texel_fetch;
1096                 name = "meta_blit3d_stencil_image_fs";
1097                 break;
1098         case BLIT2D_SRC_TYPE_BUFFER:
1099                 src_func = build_nir_buffer_fetch;
1100                 name = "meta_blit2d_stencil_buffer_fs";
1101                 break;
1102         default:
1103                 unreachable("unknown blit src type\n");
1104                 break;
1105         }
1106
1107         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1108         struct radv_shader_module fs = { .nir = NULL };
1109
1110         fs.nir = build_nir_copy_fragment_shader_stencil(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1111         vi_create_info = &normal_vi_create_info;
1112
1113         struct radv_shader_module vs = {
1114                 .nir = build_nir_vertex_shader(),
1115         };
1116
1117         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1118                 {
1119                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1120                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
1121                         .module = radv_shader_module_to_handle(&vs),
1122                         .pName = "main",
1123                         .pSpecializationInfo = NULL
1124                 }, {
1125                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1126                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1127                         .module = radv_shader_module_to_handle(&fs),
1128                         .pName = "main",
1129                         .pSpecializationInfo = NULL
1130                 },
1131         };
1132
1133         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1134                 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1135                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1136                         result = radv_CreateRenderPass(radv_device_to_handle(device),
1137                                                        &(VkRenderPassCreateInfo) {
1138                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
1139                                                                .attachmentCount = 1,
1140                                                                .pAttachments = &(VkAttachmentDescription) {
1141                                                                        .format = VK_FORMAT_S8_UINT,
1142                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1143                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1144                                                                        .initialLayout = layout,
1145                                                                        .finalLayout = layout,
1146                                                                },
1147                                                                .subpassCount = 1,
1148                                                                .pSubpasses = &(VkSubpassDescription) {
1149                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1150                                                                        .inputAttachmentCount = 0,
1151                                                                        .colorAttachmentCount = 0,
1152                                                                        .pColorAttachments = NULL,
1153                                                                        .pResolveAttachments = NULL,
1154                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
1155                                                                                .attachment = 0,
1156                                                                                .layout = layout,
1157                                                                        },
1158                                                                        .preserveAttachmentCount = 0,
1159                                                                        .pPreserveAttachments = NULL,
1160                                                                },
1161                                                                .dependencyCount = 0,
1162                                                        }, &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1163                 }
1164         }
1165
1166         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1167                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1168                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1169                 .pStages = pipeline_shader_stages,
1170                 .pVertexInputState = vi_create_info,
1171                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1172                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1173                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1174                         .primitiveRestartEnable = false,
1175                 },
1176                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1177                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1178                         .viewportCount = 1,
1179                         .scissorCount = 1,
1180                 },
1181                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1182                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1183                         .rasterizerDiscardEnable = false,
1184                         .polygonMode = VK_POLYGON_MODE_FILL,
1185                         .cullMode = VK_CULL_MODE_NONE,
1186                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1187                 },
1188                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1189                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1190                         .rasterizationSamples = 1 << log2_samples,
1191                         .sampleShadingEnable = false,
1192                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1193                 },
1194                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1195                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1196                         .attachmentCount = 0,
1197                         .pAttachments = NULL,
1198                 },
1199                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1200                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1201                         .depthTestEnable = false,
1202                         .depthWriteEnable = false,
1203                         .stencilTestEnable = true,
1204                         .front = {
1205                                 .failOp = VK_STENCIL_OP_REPLACE,
1206                                 .passOp = VK_STENCIL_OP_REPLACE,
1207                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1208                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1209                                 .compareMask = 0xff,
1210                                 .writeMask = 0xff,
1211                                 .reference = 0
1212                         },
1213                         .back = {
1214                                 .failOp = VK_STENCIL_OP_REPLACE,
1215                                 .passOp = VK_STENCIL_OP_REPLACE,
1216                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1217                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1218                                 .compareMask = 0xff,
1219                                 .writeMask = 0xff,
1220                                 .reference = 0
1221                         },
1222                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1223                 },
1224                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1225                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1226                         .dynamicStateCount = 6,
1227                         .pDynamicStates = (VkDynamicState[]) {
1228                                 VK_DYNAMIC_STATE_VIEWPORT,
1229                                 VK_DYNAMIC_STATE_SCISSOR,
1230                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1231                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1232                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1233                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1234                         },
1235                 },
1236                 .flags = 0,
1237                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1238                 .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1239                 .subpass = 0,
1240         };
1241
1242         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1243                 .use_rectlist = true
1244         };
1245
1246         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1247                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1248                                                &vk_pipeline_info, &radv_pipeline_info,
1249                                                &device->meta_state.alloc,
1250                                                &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1251
1252
1253         ralloc_free(vs.nir);
1254         ralloc_free(fs.nir);
1255
1256         mtx_unlock(&device->meta_state.mtx);
1257         return result;
1258 }
1259
1260 static VkResult
1261 meta_blit2d_create_pipe_layout(struct radv_device *device,
1262                                int idx,
1263                                uint32_t log2_samples)
1264 {
1265         VkResult result;
1266         VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER) ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1267         const VkPushConstantRange push_constant_ranges[] = {
1268                 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1269                 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1270         };
1271         int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1272
1273         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
1274                                                 &(VkDescriptorSetLayoutCreateInfo) {
1275                                                         .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1276                                                         .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1277                                                         .bindingCount = 1,
1278                                                         .pBindings = (VkDescriptorSetLayoutBinding[]) {
1279                                                         {
1280                                                                 .binding = 0,
1281                                                                 .descriptorType = desc_type,
1282                                                                 .descriptorCount = 1,
1283                                                                 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1284                                                                 .pImmutableSamplers = NULL
1285                                                         },
1286                                                         }
1287                                                 }, &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1288         if (result != VK_SUCCESS)
1289                 goto fail;
1290
1291         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
1292                                            &(VkPipelineLayoutCreateInfo) {
1293                                                    .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1294                                                            .setLayoutCount = 1,
1295                                                            .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1296                                                            .pushConstantRangeCount = num_push_constant_range,
1297                                                            .pPushConstantRanges = push_constant_ranges,
1298                                                            },
1299                                            &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1300         if (result != VK_SUCCESS)
1301                 goto fail;
1302         return VK_SUCCESS;
1303 fail:
1304         return result;
1305 }
1306
1307 VkResult
1308 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1309 {
1310         VkResult result;
1311         bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1312
1313         for (unsigned log2_samples = 0; log2_samples < 1 + MAX_SAMPLES_LOG2; log2_samples++) {
1314                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1315                         if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1316                                 continue;
1317
1318                         /* Don't need to handle copies between buffers and multisample images. */
1319                         if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1320                                 continue;
1321
1322                         result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1323                         if (result != VK_SUCCESS)
1324                                 goto fail;
1325
1326                         if (on_demand)
1327                                 continue;
1328
1329                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1330                                 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j], log2_samples);
1331                                 if (result != VK_SUCCESS)
1332                                         goto fail;
1333                         }
1334
1335                         result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1336                         if (result != VK_SUCCESS)
1337                                 goto fail;
1338
1339                         result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1340                         if (result != VK_SUCCESS)
1341                                 goto fail;
1342                 }
1343         }
1344
1345         return VK_SUCCESS;
1346
1347 fail:
1348         radv_device_finish_meta_blit2d_state(device);
1349         return result;
1350 }