src/amd/vulkan/radv_meta_blit2d.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  *
   4  * based on anv driver:
   5  * Copyright © 2016 Intel Corporation
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24  * IN THE SOFTWARE.
  25  */
  26
  27 #include "radv_meta.h"
  28 #include "nir/nir_builder.h"
  29 #include "vk_format.h"
  30
  31 enum blit2d_src_type {
  32         BLIT2D_SRC_TYPE_IMAGE,
  33         BLIT2D_SRC_TYPE_IMAGE_3D,
  34         BLIT2D_SRC_TYPE_BUFFER,
  35         BLIT2D_NUM_SRC_TYPES,
  36 };
  37
  38 static VkResult
  39 blit2d_init_color_pipeline(struct radv_device *device,
  40                            enum blit2d_src_type src_type,
  41                            VkFormat format,
  42                            uint32_t log2_samples);
  43
  44 static VkResult
  45 blit2d_init_depth_only_pipeline(struct radv_device *device,
  46                                 enum blit2d_src_type src_type,
  47                                 uint32_t log2_samples);
  48
  49 static VkResult
  50 blit2d_init_stencil_only_pipeline(struct radv_device *device,
  51                                   enum blit2d_src_type src_type,
  52                                   uint32_t log2_samples);
  53
  54 static void
  55 create_iview(struct radv_cmd_buffer *cmd_buffer,
  56              struct radv_meta_blit2d_surf *surf,
  57              struct radv_image_view *iview, VkFormat depth_format,
  58               VkImageAspectFlagBits aspects)
  59 {
  60         VkFormat format;
  61         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
  62                 radv_meta_get_view_type(surf->image);
  63
  64         if (depth_format)
  65                 format = depth_format;
  66         else
  67                 format = surf->format;
  68
  69         radv_image_view_init(iview, cmd_buffer->device,
  70                              &(VkImageViewCreateInfo) {
  71                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
  72                                              .image = radv_image_to_handle(surf->image),
  73                                              .viewType = view_type,
  74                                              .format = format,
  75                                              .subresourceRange = {
  76                                              .aspectMask = aspects,
  77                                              .baseMipLevel = surf->level,
  78                                              .levelCount = 1,
  79                                              .baseArrayLayer = surf->layer,
  80                                              .layerCount = 1
  81                                      },
  82                              });
  83 }
  84
  85 static void
  86 create_bview(struct radv_cmd_buffer *cmd_buffer,
  87              struct radv_meta_blit2d_buffer *src,
  88              struct radv_buffer_view *bview, VkFormat depth_format)
  89 {
  90         VkFormat format;
  91
  92         if (depth_format)
  93                 format = depth_format;
  94         else
  95                 format = src->format;
  96         radv_buffer_view_init(bview, cmd_buffer->device,
  97                               &(VkBufferViewCreateInfo) {
  98                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
  99                                       .flags = 0,
 100                                       .buffer = radv_buffer_to_handle(src->buffer),
 101                                       .format = format,
 102                                       .offset = src->offset,
 103                                       .range = VK_WHOLE_SIZE,
 104                               });
 105
 106 }
 107
 108 struct blit2d_src_temps {
 109         struct radv_image_view iview;
 110         struct radv_buffer_view bview;
 111 };
 112
 113 static void
 114 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer,
 115                 struct radv_meta_blit2d_surf *src_img,
 116                 struct radv_meta_blit2d_buffer *src_buf,
 117                 struct blit2d_src_temps *tmp,
 118                 enum blit2d_src_type src_type, VkFormat depth_format,
 119                 VkImageAspectFlagBits aspects,
 120                 uint32_t log2_samples)
 121 {
 122         struct radv_device *device = cmd_buffer->device;
 123
 124         if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
 125                 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
 126
 127                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 128                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 129                                               0, /* set */
 130                                               1, /* descriptorWriteCount */
 131                                               (VkWriteDescriptorSet[]) {
 132                                                       {
 133                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 134                                                               .dstBinding = 0,
 135                                                               .dstArrayElement = 0,
 136                                                               .descriptorCount = 1,
 137                                                               .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
 138                                                               .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(&tmp->bview) }
 139                                                       }
 140                                               });
 141
 142                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 143                                       device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 144                                       VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 145                                       &src_buf->pitch);
 146         } else {
 147                 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
 148
 149                 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
 150                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 151                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 152                                               VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
 153                                               &src_img->layer);
 154
 155                 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
 156                                               device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 157                                               0, /* set */
 158                                               1, /* descriptorWriteCount */
 159                                               (VkWriteDescriptorSet[]) {
 160                                                       {
 161                                                               .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 162                                                               .dstBinding = 0,
 163                                                               .dstArrayElement = 0,
 164                                                               .descriptorCount = 1,
 165                                                               .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 166                                                               .pImageInfo = (VkDescriptorImageInfo[]) {
 167                                                                       {
 168                                                                               .sampler = VK_NULL_HANDLE,
 169                                                                               .imageView = radv_image_view_to_handle(&tmp->iview),
 170                                                                               .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 171                                                                       },
 172                                                               }
 173                                                       }
 174                                               });
 175         }
 176 }
 177
 178 struct blit2d_dst_temps {
 179         VkImage image;
 180         struct radv_image_view iview;
 181         VkFramebuffer fb;
 182 };
 183
 184 static void
 185 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer,
 186                 struct radv_meta_blit2d_surf *dst,
 187                 uint32_t width,
 188                 uint32_t height,
 189                 VkFormat depth_format,
 190                 struct blit2d_dst_temps *tmp,
 191                 VkImageAspectFlagBits aspects)
 192 {
 193         create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
 194
 195         radv_CreateFramebuffer(radv_device_to_handle(cmd_buffer->device),
 196                                &(VkFramebufferCreateInfo) {
 197                                        .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
 198                                                .attachmentCount = 1,
 199                                                .pAttachments = (VkImageView[]) {
 200                                                radv_image_view_to_handle(&tmp->iview),
 201                                        },
 202                                        .width = width,
 203                                        .height = height,
 204                                        .layers = 1
 205                                 }, &cmd_buffer->pool->alloc, &tmp->fb);
 206 }
 207
 208 static void
 209 bind_pipeline(struct radv_cmd_buffer *cmd_buffer,
 210               enum blit2d_src_type src_type, unsigned fs_key,
 211               uint32_t log2_samples)
 212 {
 213         VkPipeline pipeline =
 214                 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
 215
 216         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 217                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 218 }
 219
 220 static void
 221 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer,
 222                     enum blit2d_src_type src_type,
 223                     uint32_t log2_samples)
 224 {
 225         VkPipeline pipeline =
 226                 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
 227
 228         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 229                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 230 }
 231
 232 static void
 233 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer,
 234                       enum blit2d_src_type src_type,
 235                       uint32_t log2_samples)
 236 {
 237         VkPipeline pipeline =
 238                 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
 239
 240         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 241                              VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 242 }
 243
 244 static void
 245 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
 246                             struct radv_meta_blit2d_surf *src_img,
 247                             struct radv_meta_blit2d_buffer *src_buf,
 248                             struct radv_meta_blit2d_surf *dst,
 249                             unsigned num_rects,
 250                             struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
 251                             uint32_t log2_samples)
 252 {
 253         struct radv_device *device = cmd_buffer->device;
 254
 255         for (unsigned r = 0; r < num_rects; ++r) {
 256                 unsigned i;
 257                 for_each_bit(i, dst->aspect_mask) {
 258                         unsigned aspect_mask = 1u << i;
 259                         VkFormat depth_format = 0;
 260                         if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
 261                                 depth_format = vk_format_stencil_only(dst->image->vk_format);
 262                         else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
 263                                 depth_format = vk_format_depth_only(dst->image->vk_format);
 264                         struct blit2d_src_temps src_temps;
 265                         blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format, aspect_mask, log2_samples);
 266
 267                         struct blit2d_dst_temps dst_temps;
 268                         blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
 269                                         rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
 270
 271                         float vertex_push_constants[4] = {
 272                                 rects[r].src_x,
 273                                 rects[r].src_y,
 274                                 rects[r].src_x + rects[r].width,
 275                                 rects[r].src_y + rects[r].height,
 276                         };
 277
 278                         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 279                                         device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 280                                         VK_SHADER_STAGE_VERTEX_BIT, 0, 16,
 281                                         vertex_push_constants);
 282
 283                         if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT) {
 284                                 unsigned fs_key = radv_format_meta_fs_key(dst_temps.iview.vk_format);
 285                                 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
 286
 287                                 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] == VK_NULL_HANDLE) {
 288                                         VkResult ret = blit2d_init_color_pipeline(device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
 289                                         if (ret != VK_SUCCESS) {
 290                                                 cmd_buffer->record_result = ret;
 291                                                 goto fail_pipeline;
 292                                         }
 293                                 }
 294
 295                                 radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
 296                                                         &(VkRenderPassBeginInfo) {
 297                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 298                                                                         .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
 299                                                                         .framebuffer = dst_temps.fb,
 300                                                                         .renderArea = {
 301                                                                         .offset = { rects[r].dst_x, rects[r].dst_y, },
 302                                                                         .extent = { rects[r].width, rects[r].height },
 303                                                                 },
 304                                                                         .clearValueCount = 0,
 305                                                                                 .pClearValues = NULL,
 306                                                                                 }, VK_SUBPASS_CONTENTS_INLINE);
 307
 308
 309                                 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
 310                         } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 311                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 312
 313                                 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] == VK_NULL_HANDLE) {
 314                                         VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
 315                                         if (ret != VK_SUCCESS) {
 316                                                 cmd_buffer->record_result = ret;
 317                                                 goto fail_pipeline;
 318                                         }
 319                                 }
 320
 321                                 radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
 322                                                         &(VkRenderPassBeginInfo) {
 323                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 324                                                                         .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
 325                                                                         .framebuffer = dst_temps.fb,
 326                                                                         .renderArea = {
 327                                                                         .offset = { rects[r].dst_x, rects[r].dst_y, },
 328                                                                         .extent = { rects[r].width, rects[r].height },
 329                                                                 },
 330                                                                         .clearValueCount = 0,
 331                                                                                 .pClearValues = NULL,
 332                                                                                 }, VK_SUBPASS_CONTENTS_INLINE);
 333
 334
 335                                 bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
 336
 337                         } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 338                                 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
 339
 340                                 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] == VK_NULL_HANDLE) {
 341                                         VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
 342                                         if (ret != VK_SUCCESS) {
 343                                                 cmd_buffer->record_result = ret;
 344                                                 goto fail_pipeline;
 345                                         }
 346                                 }
 347
 348                                 radv_CmdBeginRenderPass(radv_cmd_buffer_to_handle(cmd_buffer),
 349                                                         &(VkRenderPassBeginInfo) {
 350                                                                 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 351                                                                         .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
 352                                                                         .framebuffer = dst_temps.fb,
 353                                                                         .renderArea = {
 354                                                                         .offset = { rects[r].dst_x, rects[r].dst_y, },
 355                                                                         .extent = { rects[r].width, rects[r].height },
 356                                                                 },
 357                                                                         .clearValueCount = 0,
 358                                                                                 .pClearValues = NULL,
 359                                                                                 }, VK_SUBPASS_CONTENTS_INLINE);
 360
 361
 362                                 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
 363                         } else
 364                                 unreachable("Processing blit2d with multiple aspects.");
 365
 366                         radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
 367                                 .x = rects[r].dst_x,
 368                                 .y = rects[r].dst_y,
 369                                 .width = rects[r].width,
 370                                 .height = rects[r].height,
 371                                 .minDepth = 0.0f,
 372                                 .maxDepth = 1.0f
 373                         });
 374
 375                         radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
 376                                 .offset = (VkOffset2D) { rects[r].dst_x, rects[r].dst_y },
 377                                 .extent = (VkExtent2D) { rects[r].width, rects[r].height },
 378                         });
 379
 380
 381
 382                         radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
 383                         radv_CmdEndRenderPass(radv_cmd_buffer_to_handle(cmd_buffer));
 384
 385 fail_pipeline:
 386                         /* At the point where we emit the draw call, all data from the
 387                         * descriptor sets, etc. has been used.  We are free to delete it.
 388                         */
 389                         radv_DestroyFramebuffer(radv_device_to_handle(device),
 390                                                 dst_temps.fb,
 391                                                 &cmd_buffer->pool->alloc);
 392                 }
 393         }
 394 }
 395
 396 void
 397 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer,
 398                  struct radv_meta_blit2d_surf *src_img,
 399                  struct radv_meta_blit2d_buffer *src_buf,
 400                  struct radv_meta_blit2d_surf *dst,
 401                  unsigned num_rects,
 402                  struct radv_meta_blit2d_rect *rects)
 403 {
 404         bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
 405                 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
 406         enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER :
 407                 use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D : BLIT2D_SRC_TYPE_IMAGE;
 408         radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst,
 409                                     num_rects, rects, src_type,
 410                                     src_img ? util_logbase2(src_img->image->info.samples) : 0);
 411 }
 412
 413 static nir_shader *
 414 build_nir_vertex_shader(void)
 415 {
 416         const struct glsl_type *vec4 = glsl_vec4_type();
 417         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 418         nir_builder b;
 419
 420         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 421         b.shader->info.name = ralloc_strdup(b.shader, "meta_blit2d_vs");
 422
 423         nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 424                                                     vec4, "gl_Position");
 425         pos_out->data.location = VARYING_SLOT_POS;
 426
 427         nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out,
 428                                                         vec2, "v_tex_pos");
 429         tex_pos_out->data.location = VARYING_SLOT_VAR0;
 430         tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
 431
 432         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 433         nir_store_var(&b, pos_out, outvec, 0xf);
 434
 435         nir_intrinsic_instr *src_box = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 436         src_box->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 437         nir_intrinsic_set_base(src_box, 0);
 438         nir_intrinsic_set_range(src_box, 16);
 439         src_box->num_components = 4;
 440         nir_ssa_dest_init(&src_box->instr, &src_box->dest, 4, 32, "src_box");
 441         nir_builder_instr_insert(&b, &src_box->instr);
 442
 443         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_vertex_id_zero_base);
 444         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 445         nir_builder_instr_insert(&b, &vertex_id->instr);
 446
 447         /* vertex 0 - src_x, src_y */
 448         /* vertex 1 - src_x, src_y+h */
 449         /* vertex 2 - src_x+w, src_y */
 450         /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
 451            channel 1 is vertex id != 1 ? src_y : src_y + w */
 452
 453         nir_ssa_def *c0cmp = nir_ine(&b, &vertex_id->dest.ssa,
 454                                      nir_imm_int(&b, 2));
 455         nir_ssa_def *c1cmp = nir_ine(&b, &vertex_id->dest.ssa,
 456                                      nir_imm_int(&b, 1));
 457
 458         nir_ssa_def *comp[2];
 459         comp[0] = nir_bcsel(&b, c0cmp,
 460                             nir_channel(&b, &src_box->dest.ssa, 0),
 461                             nir_channel(&b, &src_box->dest.ssa, 2));
 462
 463         comp[1] = nir_bcsel(&b, c1cmp,
 464                             nir_channel(&b, &src_box->dest.ssa, 1),
 465                             nir_channel(&b, &src_box->dest.ssa, 3));
 466         nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
 467         nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
 468         return b.shader;
 469 }
 470
 471 typedef nir_ssa_def* (*texel_fetch_build_func)(struct nir_builder *,
 472                                                struct radv_device *,
 473                                                nir_ssa_def *, bool, bool);
 474
 475 static nir_ssa_def *
 476 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device,
 477                       nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 478 {
 479         enum glsl_sampler_dim dim =
 480                 is_3d ? GLSL_SAMPLER_DIM_3D : is_multisampled ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D;
 481         const struct glsl_type *sampler_type =
 482                 glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
 483         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 484                                                     sampler_type, "s_tex");
 485         sampler->data.descriptor_set = 0;
 486         sampler->data.binding = 0;
 487
 488         nir_ssa_def *tex_pos_3d = NULL;
 489         nir_intrinsic_instr *sample_idx = NULL;
 490         if (is_3d) {
 491                 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 492                 nir_intrinsic_set_base(layer, 16);
 493                 nir_intrinsic_set_range(layer, 4);
 494                 layer->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 495                 layer->num_components = 1;
 496                 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
 497                 nir_builder_instr_insert(b, &layer->instr);
 498
 499                 nir_ssa_def *chans[3];
 500                 chans[0] = nir_channel(b, tex_pos, 0);
 501                 chans[1] = nir_channel(b, tex_pos, 1);
 502                 chans[2] = &layer->dest.ssa;
 503                 tex_pos_3d = nir_vec(b, chans, 3);
 504         }
 505         if (is_multisampled) {
 506                 sample_idx = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_sample_id);
 507                 sample_idx->num_components = 1;
 508                 nir_ssa_dest_init(&sample_idx->instr, &sample_idx->dest, 1, 32, "sample_idx");
 509                 nir_builder_instr_insert(b, &sample_idx->instr);
 510         }
 511
 512         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 513
 514         nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
 515         tex->sampler_dim = dim;
 516         tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
 517         tex->src[0].src_type = nir_tex_src_coord;
 518         tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
 519         tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
 520         tex->src[1].src = nir_src_for_ssa(is_multisampled ? &sample_idx->dest.ssa : nir_imm_int(b, 0));
 521         tex->src[2].src_type = nir_tex_src_texture_deref;
 522         tex->src[2].src = nir_src_for_ssa(tex_deref);
 523         if (is_multisampled) {
 524                 tex->src[3].src_type = nir_tex_src_lod;
 525                 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
 526         }
 527         tex->dest_type = nir_type_uint;
 528         tex->is_array = false;
 529         tex->coord_components = is_3d ? 3 : 2;
 530
 531         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 532         nir_builder_instr_insert(b, &tex->instr);
 533
 534         return &tex->dest.ssa;
 535 }
 536
 537
 538 static nir_ssa_def *
 539 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device,
 540                        nir_ssa_def *tex_pos, bool is_3d, bool is_multisampled)
 541 {
 542         const struct glsl_type *sampler_type =
 543                 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
 544         nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
 545                                                     sampler_type, "s_tex");
 546         sampler->data.descriptor_set = 0;
 547         sampler->data.binding = 0;
 548
 549         nir_intrinsic_instr *width = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_push_constant);
 550         nir_intrinsic_set_base(width, 16);
 551         nir_intrinsic_set_range(width, 4);
 552         width->src[0] = nir_src_for_ssa(nir_imm_int(b, 0));
 553         width->num_components = 1;
 554         nir_ssa_dest_init(&width->instr, &width->dest, 1, 32, "width");
 555         nir_builder_instr_insert(b, &width->instr);
 556
 557         nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
 558         nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
 559         pos_y = nir_imul(b, pos_y, &width->dest.ssa);
 560         pos_x = nir_iadd(b, pos_x, pos_y);
 561         //pos_x = nir_iadd(b, pos_x, nir_imm_int(b, 100000));
 562
 563         nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
 564
 565         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
 566         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 567         tex->op = nir_texop_txf;
 568         tex->src[0].src_type = nir_tex_src_coord;
 569         tex->src[0].src = nir_src_for_ssa(pos_x);
 570         tex->src[1].src_type = nir_tex_src_texture_deref;
 571         tex->src[1].src = nir_src_for_ssa(tex_deref);
 572         tex->dest_type = nir_type_uint;
 573         tex->is_array = false;
 574         tex->coord_components = 1;
 575
 576         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 577         nir_builder_instr_insert(b, &tex->instr);
 578
 579         return &tex->dest.ssa;
 580 }
 581
 582 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
 583         .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 584         .vertexBindingDescriptionCount = 0,
 585         .vertexAttributeDescriptionCount = 0,
 586 };
 587
 588 static nir_shader *
 589 build_nir_copy_fragment_shader(struct radv_device *device,
 590                                texel_fetch_build_func txf_func, const char* name, bool is_3d,
 591                                bool is_multisampled)
 592 {
 593         const struct glsl_type *vec4 = glsl_vec4_type();
 594         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 595         nir_builder b;
 596
 597         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 598         b.shader->info.name = ralloc_strdup(b.shader, name);
 599
 600         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 601                                                        vec2, "v_tex_pos");
 602         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 603
 604         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 605                                                       vec4, "f_color");
 606         color_out->data.location = FRAG_RESULT_DATA0;
 607
 608         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 609         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 610
 611         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 612         nir_store_var(&b, color_out, color, 0xf);
 613
 614         return b.shader;
 615 }
 616
 617 static nir_shader *
 618 build_nir_copy_fragment_shader_depth(struct radv_device *device,
 619                                      texel_fetch_build_func txf_func, const char* name, bool is_3d,
 620                                      bool is_multisampled)
 621 {
 622         const struct glsl_type *vec4 = glsl_vec4_type();
 623         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 624         nir_builder b;
 625
 626         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 627         b.shader->info.name = ralloc_strdup(b.shader, name);
 628
 629         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 630                                                        vec2, "v_tex_pos");
 631         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 632
 633         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 634                                                       vec4, "f_color");
 635         color_out->data.location = FRAG_RESULT_DEPTH;
 636
 637         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 638         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 639
 640         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 641         nir_store_var(&b, color_out, color, 0x1);
 642
 643         return b.shader;
 644 }
 645
 646 static nir_shader *
 647 build_nir_copy_fragment_shader_stencil(struct radv_device *device,
 648                                        texel_fetch_build_func txf_func, const char* name, bool is_3d,
 649                                        bool is_multisampled)
 650 {
 651         const struct glsl_type *vec4 = glsl_vec4_type();
 652         const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
 653         nir_builder b;
 654
 655         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 656         b.shader->info.name = ralloc_strdup(b.shader, name);
 657
 658         nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
 659                                                        vec2, "v_tex_pos");
 660         tex_pos_in->data.location = VARYING_SLOT_VAR0;
 661
 662         nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 663                                                       vec4, "f_color");
 664         color_out->data.location = FRAG_RESULT_STENCIL;
 665
 666         nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
 667         nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
 668
 669         nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
 670         nir_store_var(&b, color_out, color, 0x1);
 671
 672         return b.shader;
 673 }
 674
 675 void
 676 radv_device_finish_meta_blit2d_state(struct radv_device *device)
 677 {
 678         struct radv_meta_state *state = &device->meta_state;
 679
 680         for(unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 681                 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
 682                         radv_DestroyRenderPass(radv_device_to_handle(device),
 683                                                state->blit2d_render_passes[j][k],
 684                                                &state->alloc);
 685                 }
 686         }
 687
 688         for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT; j++) {
 689                 radv_DestroyRenderPass(radv_device_to_handle(device),
 690                                        state->blit2d_depth_only_rp[j], &state->alloc);
 691                 radv_DestroyRenderPass(radv_device_to_handle(device),
 692                                        state->blit2d_stencil_only_rp[j], &state->alloc);
 693         }
 694
 695         for (unsigned log2_samples = 0; log2_samples < 1 + MAX_SAMPLES_LOG2; ++log2_samples) {
 696                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
 697                         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 698                                                    state->blit2d[log2_samples].p_layouts[src],
 699                                                    &state->alloc);
 700                         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 701                                                         state->blit2d[log2_samples].ds_layouts[src],
 702                                                         &state->alloc);
 703
 704                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
 705                                 radv_DestroyPipeline(radv_device_to_handle(device),
 706                                                      state->blit2d[log2_samples].pipelines[src][j],
 707                                                      &state->alloc);
 708                         }
 709
 710                         radv_DestroyPipeline(radv_device_to_handle(device),
 711                                              state->blit2d[log2_samples].depth_only_pipeline[src],
 712                                              &state->alloc);
 713                         radv_DestroyPipeline(radv_device_to_handle(device),
 714                                              state->blit2d[log2_samples].stencil_only_pipeline[src],
 715                                              &state->alloc);
 716                 }
 717         }
 718 }
 719
 720 static VkResult
 721 blit2d_init_color_pipeline(struct radv_device *device,
 722                            enum blit2d_src_type src_type,
 723                            VkFormat format,
 724                            uint32_t log2_samples)
 725 {
 726         VkResult result;
 727         unsigned fs_key = radv_format_meta_fs_key(format);
 728         const char *name;
 729
 730         mtx_lock(&device->meta_state.mtx);
 731         if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
 732                 mtx_unlock(&device->meta_state.mtx);
 733                 return VK_SUCCESS;
 734         }
 735
 736         texel_fetch_build_func src_func;
 737         switch(src_type) {
 738         case BLIT2D_SRC_TYPE_IMAGE:
 739                 src_func = build_nir_texel_fetch;
 740                 name = "meta_blit2d_image_fs";
 741                 break;
 742         case BLIT2D_SRC_TYPE_IMAGE_3D:
 743                 src_func = build_nir_texel_fetch;
 744                 name = "meta_blit3d_image_fs";
 745                 break;
 746         case BLIT2D_SRC_TYPE_BUFFER:
 747                 src_func = build_nir_buffer_fetch;
 748                 name = "meta_blit2d_buffer_fs";
 749                 break;
 750         default:
 751                 unreachable("unknown blit src type\n");
 752                 break;
 753         }
 754
 755         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 756         struct radv_shader_module fs = { .nir = NULL };
 757
 758
 759         fs.nir = build_nir_copy_fragment_shader(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 760         vi_create_info = &normal_vi_create_info;
 761
 762         struct radv_shader_module vs = {
 763                 .nir = build_nir_vertex_shader(),
 764         };
 765
 766         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 767                 {
 768                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 769                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 770                         .module = radv_shader_module_to_handle(&vs),
 771                         .pName = "main",
 772                         .pSpecializationInfo = NULL
 773                 }, {
 774                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 775                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 776                         .module = radv_shader_module_to_handle(&fs),
 777                         .pName = "main",
 778                         .pSpecializationInfo = NULL
 779                 },
 780         };
 781
 782         for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
 783                 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
 784                         VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
 785
 786                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 787                                                 &(VkRenderPassCreateInfo) {
 788                                                         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 789                                                         .attachmentCount = 1,
 790                                                         .pAttachments = &(VkAttachmentDescription) {
 791                                                         .format = format,
 792                                                         .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 793                                                         .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 794                                                         .initialLayout = layout,
 795                                                         .finalLayout = layout,
 796                                                         },
 797                                                 .subpassCount = 1,
 798                                                 .pSubpasses = &(VkSubpassDescription) {
 799                                                         .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 800                                                         .inputAttachmentCount = 0,
 801                                                         .colorAttachmentCount = 1,
 802                                                         .pColorAttachments = &(VkAttachmentReference) {
 803                                                                 .attachment = 0,
 804                                                                 .layout = layout,
 805                                                                 },
 806                                                 .pResolveAttachments = NULL,
 807                                                 .pDepthStencilAttachment = &(VkAttachmentReference) {
 808                                                         .attachment = VK_ATTACHMENT_UNUSED,
 809                                                         .layout = layout,
 810                                                 },
 811                                                 .preserveAttachmentCount = 1,
 812                                                 .pPreserveAttachments = (uint32_t[]) { 0 },
 813                                                 },
 814                                                 .dependencyCount = 0,
 815                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
 816                 }
 817         }
 818
 819         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 820                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 821                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 822                 .pStages = pipeline_shader_stages,
 823                 .pVertexInputState = vi_create_info,
 824                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 825                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 826                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 827                         .primitiveRestartEnable = false,
 828                 },
 829                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
 830                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 831                         .viewportCount = 1,
 832                         .scissorCount = 1,
 833                 },
 834                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
 835                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 836                         .rasterizerDiscardEnable = false,
 837                         .polygonMode = VK_POLYGON_MODE_FILL,
 838                         .cullMode = VK_CULL_MODE_NONE,
 839                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
 840                 },
 841                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
 842                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 843                         .rasterizationSamples = 1 << log2_samples,
 844                         .sampleShadingEnable = log2_samples > 1,
 845                         .minSampleShading = 1.0,
 846                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
 847                 },
 848                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
 849                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 850                         .attachmentCount = 1,
 851                         .pAttachments = (VkPipelineColorBlendAttachmentState []) {
 852                                 { .colorWriteMask =
 853                                   VK_COLOR_COMPONENT_A_BIT |
 854                                   VK_COLOR_COMPONENT_R_BIT |
 855                                   VK_COLOR_COMPONENT_G_BIT |
 856                                   VK_COLOR_COMPONENT_B_BIT },
 857                         }
 858                 },
 859                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
 860                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
 861                         .dynamicStateCount = 9,
 862                         .pDynamicStates = (VkDynamicState[]) {
 863                                 VK_DYNAMIC_STATE_VIEWPORT,
 864                                 VK_DYNAMIC_STATE_SCISSOR,
 865                                 VK_DYNAMIC_STATE_LINE_WIDTH,
 866                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
 867                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
 868                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
 869                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
 870                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
 871                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
 872                         },
 873                 },
 874                 .flags = 0,
 875                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
 876                 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
 877                 .subpass = 0,
 878         };
 879
 880         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
 881                 .use_rectlist = true
 882         };
 883
 884         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
 885                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
 886                                                &vk_pipeline_info, &radv_pipeline_info,
 887                                                &device->meta_state.alloc,
 888                                                &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
 889
 890
 891         ralloc_free(vs.nir);
 892         ralloc_free(fs.nir);
 893
 894         mtx_unlock(&device->meta_state.mtx);
 895         return result;
 896 }
 897
 898 static VkResult
 899 blit2d_init_depth_only_pipeline(struct radv_device *device,
 900                                 enum blit2d_src_type src_type,
 901                                 uint32_t log2_samples)
 902 {
 903         VkResult result;
 904         const char *name;
 905
 906         mtx_lock(&device->meta_state.mtx);
 907         if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
 908                 mtx_unlock(&device->meta_state.mtx);
 909                 return VK_SUCCESS;
 910         }
 911
 912         texel_fetch_build_func src_func;
 913         switch(src_type) {
 914         case BLIT2D_SRC_TYPE_IMAGE:
 915                 src_func = build_nir_texel_fetch;
 916                 name = "meta_blit2d_depth_image_fs";
 917                 break;
 918         case BLIT2D_SRC_TYPE_IMAGE_3D:
 919                 src_func = build_nir_texel_fetch;
 920                 name = "meta_blit3d_depth_image_fs";
 921                 break;
 922         case BLIT2D_SRC_TYPE_BUFFER:
 923                 src_func = build_nir_buffer_fetch;
 924                 name = "meta_blit2d_depth_buffer_fs";
 925                 break;
 926         default:
 927                 unreachable("unknown blit src type\n");
 928                 break;
 929         }
 930
 931         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
 932         struct radv_shader_module fs = { .nir = NULL };
 933
 934         fs.nir = build_nir_copy_fragment_shader_depth(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
 935         vi_create_info = &normal_vi_create_info;
 936
 937         struct radv_shader_module vs = {
 938                 .nir = build_nir_vertex_shader(),
 939         };
 940
 941         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
 942                 {
 943                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 944                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
 945                         .module = radv_shader_module_to_handle(&vs),
 946                         .pName = "main",
 947                         .pSpecializationInfo = NULL
 948                 }, {
 949                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 950                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
 951                         .module = radv_shader_module_to_handle(&fs),
 952                         .pName = "main",
 953                         .pSpecializationInfo = NULL
 954                 },
 955         };
 956
 957         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
 958                 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
 959                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
 960                         result = radv_CreateRenderPass(radv_device_to_handle(device),
 961                                                        &(VkRenderPassCreateInfo) {
 962                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 963                                                                .attachmentCount = 1,
 964                                                                .pAttachments = &(VkAttachmentDescription) {
 965                                                                        .format = VK_FORMAT_D32_SFLOAT,
 966                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
 967                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
 968                                                                        .initialLayout = layout,
 969                                                                        .finalLayout = layout,
 970                                                                },
 971                                                                .subpassCount = 1,
 972                                                                .pSubpasses = &(VkSubpassDescription) {
 973                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 974                                                                        .inputAttachmentCount = 0,
 975                                                                        .colorAttachmentCount = 0,
 976                                                                        .pColorAttachments = NULL,
 977                                                                        .pResolveAttachments = NULL,
 978                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
 979                                                                                .attachment = 0,
 980                                                                                .layout = layout,
 981                                                                        },
 982                                                                        .preserveAttachmentCount = 1,
 983                                                                        .pPreserveAttachments = (uint32_t[]) { 0 },
 984                                                                },
 985                                                                .dependencyCount = 0,
 986                                                         }, &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
 987                 }
 988         }
 989
 990         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
 991                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 992                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
 993                 .pStages = pipeline_shader_stages,
 994                 .pVertexInputState = vi_create_info,
 995                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
 996                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 997                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
 998                         .primitiveRestartEnable = false,
 999                 },
1000                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1001                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1002                         .viewportCount = 1,
1003                         .scissorCount = 1,
1004                 },
1005                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1006                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1007                         .rasterizerDiscardEnable = false,
1008                         .polygonMode = VK_POLYGON_MODE_FILL,
1009                         .cullMode = VK_CULL_MODE_NONE,
1010                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1011                 },
1012                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1013                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1014                         .rasterizationSamples = 1 << log2_samples,
1015                         .sampleShadingEnable = false,
1016                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1017                 },
1018                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1019                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1020                         .attachmentCount = 0,
1021                         .pAttachments = NULL,
1022                 },
1023                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1024                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1025                         .depthTestEnable = true,
1026                         .depthWriteEnable = true,
1027                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1028                 },
1029                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1030                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1031                         .dynamicStateCount = 9,
1032                         .pDynamicStates = (VkDynamicState[]) {
1033                                 VK_DYNAMIC_STATE_VIEWPORT,
1034                                 VK_DYNAMIC_STATE_SCISSOR,
1035                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1036                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1037                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1038                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1039                                 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1040                                 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1041                                 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1042                         },
1043                 },
1044                 .flags = 0,
1045                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1046                 .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1047                 .subpass = 0,
1048         };
1049
1050         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1051                 .use_rectlist = true
1052         };
1053
1054         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1055                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1056                                                &vk_pipeline_info, &radv_pipeline_info,
1057                                                &device->meta_state.alloc,
1058                                                &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1059
1060
1061         ralloc_free(vs.nir);
1062         ralloc_free(fs.nir);
1063
1064         mtx_unlock(&device->meta_state.mtx);
1065         return result;
1066 }
1067
1068 static VkResult
1069 blit2d_init_stencil_only_pipeline(struct radv_device *device,
1070                                   enum blit2d_src_type src_type,
1071                                   uint32_t log2_samples)
1072 {
1073         VkResult result;
1074         const char *name;
1075
1076         mtx_lock(&device->meta_state.mtx);
1077         if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1078                 mtx_unlock(&device->meta_state.mtx);
1079                 return VK_SUCCESS;
1080         }
1081
1082         texel_fetch_build_func src_func;
1083         switch(src_type) {
1084         case BLIT2D_SRC_TYPE_IMAGE:
1085                 src_func = build_nir_texel_fetch;
1086                 name = "meta_blit2d_stencil_image_fs";
1087                 break;
1088         case BLIT2D_SRC_TYPE_IMAGE_3D:
1089                 src_func = build_nir_texel_fetch;
1090                 name = "meta_blit3d_stencil_image_fs";
1091                 break;
1092         case BLIT2D_SRC_TYPE_BUFFER:
1093                 src_func = build_nir_buffer_fetch;
1094                 name = "meta_blit2d_stencil_buffer_fs";
1095                 break;
1096         default:
1097                 unreachable("unknown blit src type\n");
1098                 break;
1099         }
1100
1101         const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1102         struct radv_shader_module fs = { .nir = NULL };
1103
1104         fs.nir = build_nir_copy_fragment_shader_stencil(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1105         vi_create_info = &normal_vi_create_info;
1106
1107         struct radv_shader_module vs = {
1108                 .nir = build_nir_vertex_shader(),
1109         };
1110
1111         VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1112                 {
1113                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1114                         .stage = VK_SHADER_STAGE_VERTEX_BIT,
1115                         .module = radv_shader_module_to_handle(&vs),
1116                         .pName = "main",
1117                         .pSpecializationInfo = NULL
1118                 }, {
1119                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1120                         .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1121                         .module = radv_shader_module_to_handle(&fs),
1122                         .pName = "main",
1123                         .pSpecializationInfo = NULL
1124                 },
1125         };
1126
1127         for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1128                 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1129                         VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1130                         result = radv_CreateRenderPass(radv_device_to_handle(device),
1131                                                        &(VkRenderPassCreateInfo) {
1132                                                                .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
1133                                                                .attachmentCount = 1,
1134                                                                .pAttachments = &(VkAttachmentDescription) {
1135                                                                        .format = VK_FORMAT_S8_UINT,
1136                                                                        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1137                                                                        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1138                                                                        .initialLayout = layout,
1139                                                                        .finalLayout = layout,
1140                                                                },
1141                                                                .subpassCount = 1,
1142                                                                .pSubpasses = &(VkSubpassDescription) {
1143                                                                        .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1144                                                                        .inputAttachmentCount = 0,
1145                                                                        .colorAttachmentCount = 0,
1146                                                                        .pColorAttachments = NULL,
1147                                                                        .pResolveAttachments = NULL,
1148                                                                        .pDepthStencilAttachment = &(VkAttachmentReference) {
1149                                                                                .attachment = 0,
1150                                                                                .layout = layout,
1151                                                                        },
1152                                                                        .preserveAttachmentCount = 1,
1153                                                                        .pPreserveAttachments = (uint32_t[]) { 0 },
1154                                                                },
1155                                                                .dependencyCount = 0,
1156                                                        }, &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1157                 }
1158         }
1159
1160         const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1161                 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1162                 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1163                 .pStages = pipeline_shader_stages,
1164                 .pVertexInputState = vi_create_info,
1165                 .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1166                         .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1167                         .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1168                         .primitiveRestartEnable = false,
1169                 },
1170                 .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1171                         .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1172                         .viewportCount = 1,
1173                         .scissorCount = 1,
1174                 },
1175                 .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1176                         .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1177                         .rasterizerDiscardEnable = false,
1178                         .polygonMode = VK_POLYGON_MODE_FILL,
1179                         .cullMode = VK_CULL_MODE_NONE,
1180                         .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1181                 },
1182                 .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1183                         .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1184                         .rasterizationSamples = 1 << log2_samples,
1185                         .sampleShadingEnable = false,
1186                         .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1187                 },
1188                 .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1189                         .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1190                         .attachmentCount = 0,
1191                         .pAttachments = NULL,
1192                 },
1193                 .pDepthStencilState = &(VkPipelineDepthStencilStateCreateInfo) {
1194                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1195                         .depthTestEnable = false,
1196                         .depthWriteEnable = false,
1197                         .stencilTestEnable = true,
1198                         .front = {
1199                                 .failOp = VK_STENCIL_OP_REPLACE,
1200                                 .passOp = VK_STENCIL_OP_REPLACE,
1201                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1202                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1203                                 .compareMask = 0xff,
1204                                 .writeMask = 0xff,
1205                                 .reference = 0
1206                         },
1207                         .back = {
1208                                 .failOp = VK_STENCIL_OP_REPLACE,
1209                                 .passOp = VK_STENCIL_OP_REPLACE,
1210                                 .depthFailOp = VK_STENCIL_OP_REPLACE,
1211                                 .compareOp = VK_COMPARE_OP_ALWAYS,
1212                                 .compareMask = 0xff,
1213                                 .writeMask = 0xff,
1214                                 .reference = 0
1215                         },
1216                         .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1217                 },
1218                 .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1219                         .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1220                         .dynamicStateCount = 6,
1221                         .pDynamicStates = (VkDynamicState[]) {
1222                                 VK_DYNAMIC_STATE_VIEWPORT,
1223                                 VK_DYNAMIC_STATE_SCISSOR,
1224                                 VK_DYNAMIC_STATE_LINE_WIDTH,
1225                                 VK_DYNAMIC_STATE_DEPTH_BIAS,
1226                                 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1227                                 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1228                         },
1229                 },
1230                 .flags = 0,
1231                 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1232                 .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1233                 .subpass = 0,
1234         };
1235
1236         const struct radv_graphics_pipeline_create_info radv_pipeline_info = {
1237                 .use_rectlist = true
1238         };
1239
1240         result = radv_graphics_pipeline_create(radv_device_to_handle(device),
1241                                                radv_pipeline_cache_to_handle(&device->meta_state.cache),
1242                                                &vk_pipeline_info, &radv_pipeline_info,
1243                                                &device->meta_state.alloc,
1244                                                &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1245
1246
1247         ralloc_free(vs.nir);
1248         ralloc_free(fs.nir);
1249
1250         mtx_unlock(&device->meta_state.mtx);
1251         return result;
1252 }
1253
1254 static VkResult
1255 meta_blit2d_create_pipe_layout(struct radv_device *device,
1256                                int idx,
1257                                uint32_t log2_samples)
1258 {
1259         VkResult result;
1260         VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER) ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1261         const VkPushConstantRange push_constant_ranges[] = {
1262                 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1263                 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1264         };
1265         int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1266
1267         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
1268                                                 &(VkDescriptorSetLayoutCreateInfo) {
1269                                                         .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1270                                                         .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1271                                                         .bindingCount = 1,
1272                                                         .pBindings = (VkDescriptorSetLayoutBinding[]) {
1273                                                         {
1274                                                                 .binding = 0,
1275                                                                 .descriptorType = desc_type,
1276                                                                 .descriptorCount = 1,
1277                                                                 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1278                                                                 .pImmutableSamplers = NULL
1279                                                         },
1280                                                         }
1281                                                 }, &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1282         if (result != VK_SUCCESS)
1283                 goto fail;
1284
1285         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
1286                                            &(VkPipelineLayoutCreateInfo) {
1287                                                    .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1288                                                            .setLayoutCount = 1,
1289                                                            .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1290                                                            .pushConstantRangeCount = num_push_constant_range,
1291                                                            .pPushConstantRanges = push_constant_ranges,
1292                                                            },
1293                                            &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1294         if (result != VK_SUCCESS)
1295                 goto fail;
1296         return VK_SUCCESS;
1297 fail:
1298         return result;
1299 }
1300
1301 VkResult
1302 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1303 {
1304         VkResult result;
1305         bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1306
1307         for (unsigned log2_samples = 0; log2_samples < 1 + MAX_SAMPLES_LOG2; log2_samples++) {
1308                 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1309                         if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1310                                 continue;
1311
1312                         /* Don't need to handle copies between buffers and multisample images. */
1313                         if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1314                                 continue;
1315
1316                         result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1317                         if (result != VK_SUCCESS)
1318                                 goto fail;
1319
1320                         if (on_demand)
1321                                 continue;
1322
1323                         for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1324                                 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j], log2_samples);
1325                                 if (result != VK_SUCCESS)
1326                                         goto fail;
1327                         }
1328
1329                         result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1330                         if (result != VK_SUCCESS)
1331                                 goto fail;
1332
1333                         result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1334                         if (result != VK_SUCCESS)
1335                                 goto fail;
1336                 }
1337         }
1338
1339         return VK_SUCCESS;
1340
1341 fail:
1342         radv_device_finish_meta_blit2d_state(device);
1343         return result;
1344 }