src/amd/vulkan/radv_meta_bufimage.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  */
  24 #include "radv_meta.h"
  25 #include "nir/nir_builder.h"
  26
  27 /*
  28  * GFX queue: Compute shader implementation of image->buffer copy
  29  * Compute queue: implementation also of buffer->image, image->image, and image clear.
  30  */
  31
  32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options
  33  * for that.
  34  */
  35 static nir_shader *
  36 build_nir_itob_compute_shader(struct radv_device *dev, bool is_3d)
  37 {
  38         nir_builder b;
  39         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
  40         const struct glsl_type *sampler_type = glsl_sampler_type(dim,
  41                                                                  false,
  42                                                                  false,
  43                                                                  GLSL_TYPE_FLOAT);
  44         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
  45                                                              false,
  46                                                              false,
  47                                                              GLSL_TYPE_FLOAT);
  48         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
  49         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itob_cs_3d" : "meta_itob_cs");
  50         b.shader->info.cs.local_size[0] = 16;
  51         b.shader->info.cs.local_size[1] = 16;
  52         b.shader->info.cs.local_size[2] = 1;
  53         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
  54                                                       sampler_type, "s_tex");
  55         input_img->data.descriptor_set = 0;
  56         input_img->data.binding = 0;
  57
  58         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
  59                                                        img_type, "out_img");
  60         output_img->data.descriptor_set = 0;
  61         output_img->data.binding = 1;
  62
  63         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
  64         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
  65         nir_ssa_def *block_size = nir_imm_ivec4(&b,
  66                                                 b.shader->info.cs.local_size[0],
  67                                                 b.shader->info.cs.local_size[1],
  68                                                 b.shader->info.cs.local_size[2], 0);
  69
  70         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
  71
  72
  73
  74         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  75         nir_intrinsic_set_base(offset, 0);
  76         nir_intrinsic_set_range(offset, 16);
  77         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
  78         offset->num_components = is_3d ? 3 : 2;
  79         nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
  80         nir_builder_instr_insert(&b, &offset->instr);
  81
  82         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  83         nir_intrinsic_set_base(stride, 0);
  84         nir_intrinsic_set_range(stride, 16);
  85         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
  86         stride->num_components = 1;
  87         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
  88         nir_builder_instr_insert(&b, &stride->instr);
  89
  90         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
  91         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
  92         tex->sampler_dim = dim;
  93         tex->op = nir_texop_txf;
  94         tex->src[0].src_type = nir_tex_src_coord;
  95         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, is_3d ? 0x7 : 0x3));
  96         tex->src[1].src_type = nir_tex_src_lod;
  97         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
  98         tex->dest_type = nir_type_float;
  99         tex->is_array = false;
 100         tex->coord_components = is_3d ? 3 : 2;
 101         tex->texture = nir_deref_var_create(tex, input_img);
 102         tex->sampler = NULL;
 103
 104         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 105         nir_builder_instr_insert(&b, &tex->instr);
 106
 107         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 108         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 109
 110         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 111         tmp = nir_iadd(&b, tmp, pos_x);
 112
 113         nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 114
 115         nir_ssa_def *outval = &tex->dest.ssa;
 116         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 117         store->src[0] = nir_src_for_ssa(coord);
 118         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 119         store->src[2] = nir_src_for_ssa(outval);
 120         store->variables[0] = nir_deref_var_create(store, output_img);
 121
 122         nir_builder_instr_insert(&b, &store->instr);
 123         return b.shader;
 124 }
 125
 126 /* Image to buffer - don't write use image accessors */
 127 static VkResult
 128 radv_device_init_meta_itob_state(struct radv_device *device)
 129 {
 130         VkResult result;
 131         struct radv_shader_module cs = { .nir = NULL };
 132         struct radv_shader_module cs_3d = { .nir = NULL };
 133
 134         cs.nir = build_nir_itob_compute_shader(device, false);
 135         if (device->physical_device->rad_info.chip_class >= GFX9)
 136                 cs_3d.nir = build_nir_itob_compute_shader(device, true);
 137
 138         /*
 139          * two descriptors one for the image being sampled
 140          * one for the buffer being written.
 141          */
 142         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 143                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 144                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 145                 .bindingCount = 2,
 146                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 147                         {
 148                                 .binding = 0,
 149                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 150                                 .descriptorCount = 1,
 151                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 152                                 .pImmutableSamplers = NULL
 153                         },
 154                         {
 155                                 .binding = 1,
 156                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 157                                 .descriptorCount = 1,
 158                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 159                                 .pImmutableSamplers = NULL
 160                         },
 161                 }
 162         };
 163
 164         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 165                                                 &ds_create_info,
 166                                                 &device->meta_state.alloc,
 167                                                 &device->meta_state.itob.img_ds_layout);
 168         if (result != VK_SUCCESS)
 169                 goto fail;
 170
 171
 172         VkPipelineLayoutCreateInfo pl_create_info = {
 173                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 174                 .setLayoutCount = 1,
 175                 .pSetLayouts = &device->meta_state.itob.img_ds_layout,
 176                 .pushConstantRangeCount = 1,
 177                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 178         };
 179
 180         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 181                                           &pl_create_info,
 182                                           &device->meta_state.alloc,
 183                                           &device->meta_state.itob.img_p_layout);
 184         if (result != VK_SUCCESS)
 185                 goto fail;
 186
 187         /* compute shader */
 188
 189         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 190                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 191                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 192                 .module = radv_shader_module_to_handle(&cs),
 193                 .pName = "main",
 194                 .pSpecializationInfo = NULL,
 195         };
 196
 197         VkComputePipelineCreateInfo vk_pipeline_info = {
 198                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 199                 .stage = pipeline_shader_stage,
 200                 .flags = 0,
 201                 .layout = device->meta_state.itob.img_p_layout,
 202         };
 203
 204         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 205                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 206                                              1, &vk_pipeline_info, NULL,
 207                                              &device->meta_state.itob.pipeline);
 208         if (result != VK_SUCCESS)
 209                 goto fail;
 210
 211         if (device->physical_device->rad_info.chip_class >= GFX9) {
 212                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 213                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 214                         .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 215                         .module = radv_shader_module_to_handle(&cs_3d),
 216                         .pName = "main",
 217                         .pSpecializationInfo = NULL,
 218                 };
 219
 220                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 221                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 222                         .stage = pipeline_shader_stage_3d,
 223                         .flags = 0,
 224                         .layout = device->meta_state.itob.img_p_layout,
 225                 };
 226
 227                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 228                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 229                                                      1, &vk_pipeline_info_3d, NULL,
 230                                                      &device->meta_state.itob.pipeline_3d);
 231                 if (result != VK_SUCCESS)
 232                         goto fail;
 233                 ralloc_free(cs_3d.nir);
 234         }
 235         ralloc_free(cs.nir);
 236
 237         return VK_SUCCESS;
 238 fail:
 239         ralloc_free(cs.nir);
 240         ralloc_free(cs_3d.nir);
 241         return result;
 242 }
 243
 244 static void
 245 radv_device_finish_meta_itob_state(struct radv_device *device)
 246 {
 247         struct radv_meta_state *state = &device->meta_state;
 248
 249         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 250                                    state->itob.img_p_layout, &state->alloc);
 251         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 252                                         state->itob.img_ds_layout,
 253                                         &state->alloc);
 254         radv_DestroyPipeline(radv_device_to_handle(device),
 255                              state->itob.pipeline, &state->alloc);
 256         if (device->physical_device->rad_info.chip_class >= GFX9)
 257                 radv_DestroyPipeline(radv_device_to_handle(device),
 258                                      state->itob.pipeline_3d, &state->alloc);
 259 }
 260
 261 static nir_shader *
 262 build_nir_btoi_compute_shader(struct radv_device *dev, bool is_3d)
 263 {
 264         nir_builder b;
 265         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
 266         const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
 267                                                              false,
 268                                                              false,
 269                                                              GLSL_TYPE_FLOAT);
 270         const struct glsl_type *img_type = glsl_sampler_type(dim,
 271                                                              false,
 272                                                              false,
 273                                                              GLSL_TYPE_FLOAT);
 274         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 275         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_btoi_cs_3d" : "meta_btoi_cs");
 276         b.shader->info.cs.local_size[0] = 16;
 277         b.shader->info.cs.local_size[1] = 16;
 278         b.shader->info.cs.local_size[2] = 1;
 279         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 280                                                       buf_type, "s_tex");
 281         input_img->data.descriptor_set = 0;
 282         input_img->data.binding = 0;
 283
 284         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 285                                                        img_type, "out_img");
 286         output_img->data.descriptor_set = 0;
 287         output_img->data.binding = 1;
 288
 289         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 290         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 291         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 292                                                 b.shader->info.cs.local_size[0],
 293                                                 b.shader->info.cs.local_size[1],
 294                                                 b.shader->info.cs.local_size[2], 0);
 295
 296         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 297
 298         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 299         nir_intrinsic_set_base(offset, 0);
 300         nir_intrinsic_set_range(offset, 16);
 301         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 302         offset->num_components = is_3d ? 3 : 2;
 303         nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
 304         nir_builder_instr_insert(&b, &offset->instr);
 305
 306         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 307         nir_intrinsic_set_base(stride, 0);
 308         nir_intrinsic_set_range(stride, 16);
 309         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
 310         stride->num_components = 1;
 311         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
 312         nir_builder_instr_insert(&b, &stride->instr);
 313
 314         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 315         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 316
 317         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 318         tmp = nir_iadd(&b, tmp, pos_x);
 319
 320         nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 321
 322         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
 323
 324         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 325         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 326         tex->op = nir_texop_txf;
 327         tex->src[0].src_type = nir_tex_src_coord;
 328         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1));
 329         tex->src[1].src_type = nir_tex_src_lod;
 330         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 331         tex->dest_type = nir_type_float;
 332         tex->is_array = false;
 333         tex->coord_components = 1;
 334         tex->texture = nir_deref_var_create(tex, input_img);
 335         tex->sampler = NULL;
 336
 337         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 338         nir_builder_instr_insert(&b, &tex->instr);
 339
 340         nir_ssa_def *outval = &tex->dest.ssa;
 341         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 342         store->src[0] = nir_src_for_ssa(img_coord);
 343         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 344         store->src[2] = nir_src_for_ssa(outval);
 345         store->variables[0] = nir_deref_var_create(store, output_img);
 346
 347         nir_builder_instr_insert(&b, &store->instr);
 348         return b.shader;
 349 }
 350
 351 /* Buffer to image - don't write use image accessors */
 352 static VkResult
 353 radv_device_init_meta_btoi_state(struct radv_device *device)
 354 {
 355         VkResult result;
 356         struct radv_shader_module cs = { .nir = NULL };
 357         struct radv_shader_module cs_3d = { .nir = NULL };
 358         cs.nir = build_nir_btoi_compute_shader(device, false);
 359         if (device->physical_device->rad_info.chip_class >= GFX9)
 360                 cs_3d.nir = build_nir_btoi_compute_shader(device, true);
 361         /*
 362          * two descriptors one for the image being sampled
 363          * one for the buffer being written.
 364          */
 365         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 366                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 367                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 368                 .bindingCount = 2,
 369                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 370                         {
 371                                 .binding = 0,
 372                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 373                                 .descriptorCount = 1,
 374                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 375                                 .pImmutableSamplers = NULL
 376                         },
 377                         {
 378                                 .binding = 1,
 379                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 380                                 .descriptorCount = 1,
 381                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 382                                 .pImmutableSamplers = NULL
 383                         },
 384                 }
 385         };
 386
 387         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 388                                                 &ds_create_info,
 389                                                 &device->meta_state.alloc,
 390                                                 &device->meta_state.btoi.img_ds_layout);
 391         if (result != VK_SUCCESS)
 392                 goto fail;
 393
 394
 395         VkPipelineLayoutCreateInfo pl_create_info = {
 396                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 397                 .setLayoutCount = 1,
 398                 .pSetLayouts = &device->meta_state.btoi.img_ds_layout,
 399                 .pushConstantRangeCount = 1,
 400                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 401         };
 402
 403         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 404                                           &pl_create_info,
 405                                           &device->meta_state.alloc,
 406                                           &device->meta_state.btoi.img_p_layout);
 407         if (result != VK_SUCCESS)
 408                 goto fail;
 409
 410         /* compute shader */
 411
 412         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 413                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 414                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 415                 .module = radv_shader_module_to_handle(&cs),
 416                 .pName = "main",
 417                 .pSpecializationInfo = NULL,
 418         };
 419
 420         VkComputePipelineCreateInfo vk_pipeline_info = {
 421                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 422                 .stage = pipeline_shader_stage,
 423                 .flags = 0,
 424                 .layout = device->meta_state.btoi.img_p_layout,
 425         };
 426
 427         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 428                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 429                                              1, &vk_pipeline_info, NULL,
 430                                              &device->meta_state.btoi.pipeline);
 431         if (result != VK_SUCCESS)
 432                 goto fail;
 433
 434         if (device->physical_device->rad_info.chip_class >= GFX9) {
 435                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 436                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 437                         .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 438                         .module = radv_shader_module_to_handle(&cs_3d),
 439                         .pName = "main",
 440                         .pSpecializationInfo = NULL,
 441                 };
 442
 443                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 444                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 445                         .stage = pipeline_shader_stage_3d,
 446                         .flags = 0,
 447                         .layout = device->meta_state.btoi.img_p_layout,
 448                 };
 449
 450                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 451                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 452                                                      1, &vk_pipeline_info_3d, NULL,
 453                                                      &device->meta_state.btoi.pipeline_3d);
 454                 ralloc_free(cs_3d.nir);
 455         }
 456         ralloc_free(cs.nir);
 457
 458         return VK_SUCCESS;
 459 fail:
 460         ralloc_free(cs_3d.nir);
 461         ralloc_free(cs.nir);
 462         return result;
 463 }
 464
 465 static void
 466 radv_device_finish_meta_btoi_state(struct radv_device *device)
 467 {
 468         struct radv_meta_state *state = &device->meta_state;
 469
 470         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 471                                    state->btoi.img_p_layout, &state->alloc);
 472         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 473                                         state->btoi.img_ds_layout,
 474                                         &state->alloc);
 475         radv_DestroyPipeline(radv_device_to_handle(device),
 476                              state->btoi.pipeline, &state->alloc);
 477         radv_DestroyPipeline(radv_device_to_handle(device),
 478                              state->btoi.pipeline_3d, &state->alloc);
 479 }
 480
 481 static nir_shader *
 482 build_nir_itoi_compute_shader(struct radv_device *dev, bool is_3d)
 483 {
 484         nir_builder b;
 485         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
 486         const struct glsl_type *buf_type = glsl_sampler_type(dim,
 487                                                              false,
 488                                                              false,
 489                                                              GLSL_TYPE_FLOAT);
 490         const struct glsl_type *img_type = glsl_sampler_type(dim,
 491                                                              false,
 492                                                              false,
 493                                                              GLSL_TYPE_FLOAT);
 494         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 495         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itoi_cs_3d" : "meta_itoi_cs");
 496         b.shader->info.cs.local_size[0] = 16;
 497         b.shader->info.cs.local_size[1] = 16;
 498         b.shader->info.cs.local_size[2] = 1;
 499         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 500                                                       buf_type, "s_tex");
 501         input_img->data.descriptor_set = 0;
 502         input_img->data.binding = 0;
 503
 504         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 505                                                        img_type, "out_img");
 506         output_img->data.descriptor_set = 0;
 507         output_img->data.binding = 1;
 508
 509         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 510         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 511         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 512                                                 b.shader->info.cs.local_size[0],
 513                                                 b.shader->info.cs.local_size[1],
 514                                                 b.shader->info.cs.local_size[2], 0);
 515
 516         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 517
 518         nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 519         nir_intrinsic_set_base(src_offset, 0);
 520         nir_intrinsic_set_range(src_offset, 24);
 521         src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 522         src_offset->num_components = is_3d ? 3 : 2;
 523         nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, is_3d ? 3 : 2, 32, "src_offset");
 524         nir_builder_instr_insert(&b, &src_offset->instr);
 525
 526         nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 527         nir_intrinsic_set_base(dst_offset, 0);
 528         nir_intrinsic_set_range(dst_offset, 24);
 529         dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
 530         dst_offset->num_components = is_3d ? 3 : 2;
 531         nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, is_3d ? 3 : 2, 32, "dst_offset");
 532         nir_builder_instr_insert(&b, &dst_offset->instr);
 533
 534         nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa);
 535
 536         nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
 537
 538         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 539         tex->sampler_dim = dim;
 540         tex->op = nir_texop_txf;
 541         tex->src[0].src_type = nir_tex_src_coord;
 542         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, is_3d ? 0x7 : 0x3));
 543         tex->src[1].src_type = nir_tex_src_lod;
 544         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 545         tex->dest_type = nir_type_float;
 546         tex->is_array = false;
 547         tex->coord_components = is_3d ? 3 : 2;
 548         tex->texture = nir_deref_var_create(tex, input_img);
 549         tex->sampler = NULL;
 550
 551         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 552         nir_builder_instr_insert(&b, &tex->instr);
 553
 554         nir_ssa_def *outval = &tex->dest.ssa;
 555         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 556         store->src[0] = nir_src_for_ssa(dst_coord);
 557         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 558         store->src[2] = nir_src_for_ssa(outval);
 559         store->variables[0] = nir_deref_var_create(store, output_img);
 560
 561         nir_builder_instr_insert(&b, &store->instr);
 562         return b.shader;
 563 }
 564
 565 /* image to image - don't write use image accessors */
 566 static VkResult
 567 radv_device_init_meta_itoi_state(struct radv_device *device)
 568 {
 569         VkResult result;
 570         struct radv_shader_module cs = { .nir = NULL };
 571         struct radv_shader_module cs_3d = { .nir = NULL };
 572         cs.nir = build_nir_itoi_compute_shader(device, false);
 573         if (device->physical_device->rad_info.chip_class >= GFX9)
 574                 cs_3d.nir = build_nir_itoi_compute_shader(device, true);
 575         /*
 576          * two descriptors one for the image being sampled
 577          * one for the buffer being written.
 578          */
 579         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 580                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 581                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 582                 .bindingCount = 2,
 583                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 584                         {
 585                                 .binding = 0,
 586                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 587                                 .descriptorCount = 1,
 588                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 589                                 .pImmutableSamplers = NULL
 590                         },
 591                         {
 592                                 .binding = 1,
 593                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 594                                 .descriptorCount = 1,
 595                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 596                                 .pImmutableSamplers = NULL
 597                         },
 598                 }
 599         };
 600
 601         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 602                                                 &ds_create_info,
 603                                                 &device->meta_state.alloc,
 604                                                 &device->meta_state.itoi.img_ds_layout);
 605         if (result != VK_SUCCESS)
 606                 goto fail;
 607
 608
 609         VkPipelineLayoutCreateInfo pl_create_info = {
 610                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 611                 .setLayoutCount = 1,
 612                 .pSetLayouts = &device->meta_state.itoi.img_ds_layout,
 613                 .pushConstantRangeCount = 1,
 614                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 24},
 615         };
 616
 617         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 618                                           &pl_create_info,
 619                                           &device->meta_state.alloc,
 620                                           &device->meta_state.itoi.img_p_layout);
 621         if (result != VK_SUCCESS)
 622                 goto fail;
 623
 624         /* compute shader */
 625
 626         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 627                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 628                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 629                 .module = radv_shader_module_to_handle(&cs),
 630                 .pName = "main",
 631                 .pSpecializationInfo = NULL,
 632         };
 633
 634         VkComputePipelineCreateInfo vk_pipeline_info = {
 635                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 636                 .stage = pipeline_shader_stage,
 637                 .flags = 0,
 638                 .layout = device->meta_state.itoi.img_p_layout,
 639         };
 640
 641         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 642                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 643                                              1, &vk_pipeline_info, NULL,
 644                                              &device->meta_state.itoi.pipeline);
 645         if (result != VK_SUCCESS)
 646                 goto fail;
 647
 648         if (device->physical_device->rad_info.chip_class >= GFX9) {
 649                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 650                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 651 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 652                         .module = radv_shader_module_to_handle(&cs_3d),
 653                         .pName = "main",
 654                         .pSpecializationInfo = NULL,
 655                 };
 656
 657                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 658                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 659                         .stage = pipeline_shader_stage_3d,
 660                         .flags = 0,
 661                         .layout = device->meta_state.itoi.img_p_layout,
 662                 };
 663
 664                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 665                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 666                                                      1, &vk_pipeline_info_3d, NULL,
 667                                                      &device->meta_state.itoi.pipeline_3d);
 668
 669                 ralloc_free(cs_3d.nir);
 670         }
 671         ralloc_free(cs.nir);
 672
 673         return VK_SUCCESS;
 674 fail:
 675         ralloc_free(cs.nir);
 676         ralloc_free(cs_3d.nir);
 677         return result;
 678 }
 679
 680 static void
 681 radv_device_finish_meta_itoi_state(struct radv_device *device)
 682 {
 683         struct radv_meta_state *state = &device->meta_state;
 684
 685         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 686                                    state->itoi.img_p_layout, &state->alloc);
 687         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 688                                         state->itoi.img_ds_layout,
 689                                         &state->alloc);
 690         radv_DestroyPipeline(radv_device_to_handle(device),
 691                              state->itoi.pipeline, &state->alloc);
 692         if (device->physical_device->rad_info.chip_class >= GFX9)
 693                 radv_DestroyPipeline(radv_device_to_handle(device),
 694                                      state->itoi.pipeline_3d, &state->alloc);
 695 }
 696
 697 static nir_shader *
 698 build_nir_cleari_compute_shader(struct radv_device *dev, bool is_3d)
 699 {
 700         nir_builder b;
 701         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
 702         const struct glsl_type *img_type = glsl_sampler_type(dim,
 703                                                              false,
 704                                                              false,
 705                                                              GLSL_TYPE_FLOAT);
 706         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 707         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_cleari_cs_3d" : "meta_cleari_cs");
 708         b.shader->info.cs.local_size[0] = 16;
 709         b.shader->info.cs.local_size[1] = 16;
 710         b.shader->info.cs.local_size[2] = 1;
 711
 712         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 713                                                        img_type, "out_img");
 714         output_img->data.descriptor_set = 0;
 715         output_img->data.binding = 0;
 716
 717         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 718         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 719         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 720                                                 b.shader->info.cs.local_size[0],
 721                                                 b.shader->info.cs.local_size[1],
 722                                                 b.shader->info.cs.local_size[2], 0);
 723
 724         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 725
 726         nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 727         nir_intrinsic_set_base(clear_val, 0);
 728         nir_intrinsic_set_range(clear_val, 20);
 729         clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 730         clear_val->num_components = 4;
 731         nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value");
 732         nir_builder_instr_insert(&b, &clear_val->instr);
 733
 734         nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 735         nir_intrinsic_set_base(layer, 0);
 736         nir_intrinsic_set_range(layer, 20);
 737         layer->src[0] = nir_src_for_ssa(nir_imm_int(&b, 16));
 738         layer->num_components = 1;
 739         nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
 740         nir_builder_instr_insert(&b, &layer->instr);
 741
 742         nir_ssa_def *global_z = nir_iadd(&b, nir_channel(&b, global_id, 2), &layer->dest.ssa);
 743
 744         nir_ssa_def *comps[4];
 745         comps[0] = nir_channel(&b, global_id, 0);
 746         comps[1] = nir_channel(&b, global_id, 1);
 747         comps[2] = global_z;
 748         comps[3] = nir_imm_int(&b, 0);
 749         global_id = nir_vec(&b, comps, 4);
 750
 751         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 752         store->src[0] = nir_src_for_ssa(global_id);
 753         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 754         store->src[2] = nir_src_for_ssa(&clear_val->dest.ssa);
 755         store->variables[0] = nir_deref_var_create(store, output_img);
 756
 757         nir_builder_instr_insert(&b, &store->instr);
 758         return b.shader;
 759 }
 760
 761 static VkResult
 762 radv_device_init_meta_cleari_state(struct radv_device *device)
 763 {
 764         VkResult result;
 765         struct radv_shader_module cs = { .nir = NULL };
 766         struct radv_shader_module cs_3d = { .nir = NULL };
 767         cs.nir = build_nir_cleari_compute_shader(device, false);
 768         if (device->physical_device->rad_info.chip_class >= GFX9)
 769                 cs_3d.nir = build_nir_cleari_compute_shader(device, true);
 770
 771         /*
 772          * two descriptors one for the image being sampled
 773          * one for the buffer being written.
 774          */
 775         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 776                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 777                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 778                 .bindingCount = 1,
 779                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 780                         {
 781                                 .binding = 0,
 782                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 783                                 .descriptorCount = 1,
 784                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 785                                 .pImmutableSamplers = NULL
 786                         },
 787                 }
 788         };
 789
 790         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 791                                                 &ds_create_info,
 792                                                 &device->meta_state.alloc,
 793                                                 &device->meta_state.cleari.img_ds_layout);
 794         if (result != VK_SUCCESS)
 795                 goto fail;
 796
 797
 798         VkPipelineLayoutCreateInfo pl_create_info = {
 799                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 800                 .setLayoutCount = 1,
 801                 .pSetLayouts = &device->meta_state.cleari.img_ds_layout,
 802                 .pushConstantRangeCount = 1,
 803                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 20},
 804         };
 805
 806         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 807                                           &pl_create_info,
 808                                           &device->meta_state.alloc,
 809                                           &device->meta_state.cleari.img_p_layout);
 810         if (result != VK_SUCCESS)
 811                 goto fail;
 812
 813         /* compute shader */
 814
 815         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 816                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 817                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 818                 .module = radv_shader_module_to_handle(&cs),
 819                 .pName = "main",
 820                 .pSpecializationInfo = NULL,
 821         };
 822
 823         VkComputePipelineCreateInfo vk_pipeline_info = {
 824                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 825                 .stage = pipeline_shader_stage,
 826                 .flags = 0,
 827                 .layout = device->meta_state.cleari.img_p_layout,
 828         };
 829
 830         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 831                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 832                                              1, &vk_pipeline_info, NULL,
 833                                              &device->meta_state.cleari.pipeline);
 834         if (result != VK_SUCCESS)
 835                 goto fail;
 836
 837
 838         if (device->physical_device->rad_info.chip_class >= GFX9) {
 839                 /* compute shader */
 840                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 841                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 842                         .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 843                         .module = radv_shader_module_to_handle(&cs_3d),
 844                         .pName = "main",
 845                         .pSpecializationInfo = NULL,
 846                 };
 847
 848                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 849                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 850                         .stage = pipeline_shader_stage_3d,
 851                         .flags = 0,
 852                         .layout = device->meta_state.cleari.img_p_layout,
 853                 };
 854
 855                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 856                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 857                                                      1, &vk_pipeline_info_3d, NULL,
 858                                                      &device->meta_state.cleari.pipeline_3d);
 859                 if (result != VK_SUCCESS)
 860                         goto fail;
 861
 862                 ralloc_free(cs_3d.nir);
 863         }
 864         ralloc_free(cs.nir);
 865         return VK_SUCCESS;
 866 fail:
 867         ralloc_free(cs.nir);
 868         ralloc_free(cs_3d.nir);
 869         return result;
 870 }
 871
 872 static void
 873 radv_device_finish_meta_cleari_state(struct radv_device *device)
 874 {
 875         struct radv_meta_state *state = &device->meta_state;
 876
 877         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 878                                    state->cleari.img_p_layout, &state->alloc);
 879         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 880                                         state->cleari.img_ds_layout,
 881                                         &state->alloc);
 882         radv_DestroyPipeline(radv_device_to_handle(device),
 883                              state->cleari.pipeline, &state->alloc);
 884         radv_DestroyPipeline(radv_device_to_handle(device),
 885                              state->cleari.pipeline_3d, &state->alloc);
 886 }
 887
 888 void
 889 radv_device_finish_meta_bufimage_state(struct radv_device *device)
 890 {
 891         radv_device_finish_meta_itob_state(device);
 892         radv_device_finish_meta_btoi_state(device);
 893         radv_device_finish_meta_itoi_state(device);
 894         radv_device_finish_meta_cleari_state(device);
 895 }
 896
 897 VkResult
 898 radv_device_init_meta_bufimage_state(struct radv_device *device)
 899 {
 900         VkResult result;
 901
 902         result = radv_device_init_meta_itob_state(device);
 903         if (result != VK_SUCCESS)
 904                 return result;
 905
 906         result = radv_device_init_meta_btoi_state(device);
 907         if (result != VK_SUCCESS)
 908                 goto fail_itob;
 909
 910         result = radv_device_init_meta_itoi_state(device);
 911         if (result != VK_SUCCESS)
 912                 goto fail_btoi;
 913
 914         result = radv_device_init_meta_cleari_state(device);
 915         if (result != VK_SUCCESS)
 916                 goto fail_itoi;
 917
 918         return VK_SUCCESS;
 919 fail_itoi:
 920         radv_device_finish_meta_itoi_state(device);
 921 fail_btoi:
 922         radv_device_finish_meta_btoi_state(device);
 923 fail_itob:
 924         radv_device_finish_meta_itob_state(device);
 925         return result;
 926 }
 927
 928 static void
 929 create_iview(struct radv_cmd_buffer *cmd_buffer,
 930              struct radv_meta_blit2d_surf *surf,
 931              struct radv_image_view *iview)
 932 {
 933         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
 934                 radv_meta_get_view_type(surf->image);
 935         radv_image_view_init(iview, cmd_buffer->device,
 936                              &(VkImageViewCreateInfo) {
 937                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 938                                              .image = radv_image_to_handle(surf->image),
 939                                              .viewType = view_type,
 940                                              .format = surf->format,
 941                                              .subresourceRange = {
 942                                              .aspectMask = surf->aspect_mask,
 943                                              .baseMipLevel = surf->level,
 944                                              .levelCount = 1,
 945                                              .baseArrayLayer = surf->layer,
 946                                              .layerCount = 1
 947                                      },
 948                              });
 949 }
 950
 951 static void
 952 create_bview(struct radv_cmd_buffer *cmd_buffer,
 953              struct radv_buffer *buffer,
 954              unsigned offset,
 955              VkFormat format,
 956              struct radv_buffer_view *bview)
 957 {
 958         radv_buffer_view_init(bview, cmd_buffer->device,
 959                               &(VkBufferViewCreateInfo) {
 960                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
 961                                       .flags = 0,
 962                                       .buffer = radv_buffer_to_handle(buffer),
 963                                       .format = format,
 964                                       .offset = offset,
 965                                       .range = VK_WHOLE_SIZE,
 966                               });
 967
 968 }
 969
 970 static void
 971 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
 972                       struct radv_image_view *src,
 973                       struct radv_buffer_view *dst)
 974 {
 975         struct radv_device *device = cmd_buffer->device;
 976
 977         radv_meta_push_descriptor_set(cmd_buffer,
 978                                       VK_PIPELINE_BIND_POINT_COMPUTE,
 979                                       device->meta_state.itob.img_p_layout,
 980                                       0, /* set */
 981                                       2, /* descriptorWriteCount */
 982                                       (VkWriteDescriptorSet[]) {
 983                                               {
 984                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 985                                                       .dstBinding = 0,
 986                                                       .dstArrayElement = 0,
 987                                                       .descriptorCount = 1,
 988                                                       .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 989                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 990                                                               {
 991                                                                       .sampler = VK_NULL_HANDLE,
 992                                                                       .imageView = radv_image_view_to_handle(src),
 993                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 994                                                               },
 995                                                       }
 996                                               },
 997                                               {
 998                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 999                                                       .dstBinding = 1,
1000                                                       .dstArrayElement = 0,
1001                                                       .descriptorCount = 1,
1002                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1003                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(dst) },
1004                                               }
1005                                       });
1006 }
1007
1008 void
1009 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
1010                           struct radv_meta_blit2d_surf *src,
1011                           struct radv_meta_blit2d_buffer *dst,
1012                           unsigned num_rects,
1013                           struct radv_meta_blit2d_rect *rects)
1014 {
1015         VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline;
1016         struct radv_device *device = cmd_buffer->device;
1017         struct radv_image_view src_view;
1018         struct radv_buffer_view dst_view;
1019
1020         create_iview(cmd_buffer, src, &src_view);
1021         create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &dst_view);
1022         itob_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1023
1024         if (device->physical_device->rad_info.chip_class >= GFX9 &&
1025             src->image->type == VK_IMAGE_TYPE_3D)
1026                 pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d;
1027
1028         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1029                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1030
1031         for (unsigned r = 0; r < num_rects; ++r) {
1032                 unsigned push_constants[4] = {
1033                         rects[r].src_x,
1034                         rects[r].src_y,
1035                         src->layer,
1036                         dst->pitch
1037                 };
1038                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1039                                       device->meta_state.itob.img_p_layout,
1040                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1041                                       push_constants);
1042
1043                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1044         }
1045 }
1046
1047 static void
1048 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1049                       struct radv_buffer_view *src,
1050                       struct radv_image_view *dst)
1051 {
1052         struct radv_device *device = cmd_buffer->device;
1053
1054         radv_meta_push_descriptor_set(cmd_buffer,
1055                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1056                                       device->meta_state.btoi.img_p_layout,
1057                                       0, /* set */
1058                                       2, /* descriptorWriteCount */
1059                                       (VkWriteDescriptorSet[]) {
1060                                               {
1061                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1062                                                       .dstBinding = 0,
1063                                                       .dstArrayElement = 0,
1064                                                       .descriptorCount = 1,
1065                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1066                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(src) },
1067                                               },
1068                                               {
1069                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1070                                                       .dstBinding = 1,
1071                                                       .dstArrayElement = 0,
1072                                                       .descriptorCount = 1,
1073                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1074                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1075                                                               {
1076                                                                       .sampler = VK_NULL_HANDLE,
1077                                                                       .imageView = radv_image_view_to_handle(dst),
1078                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1079                                                               },
1080                                                       }
1081                                               }
1082                                       });
1083 }
1084
1085 void
1086 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1087                              struct radv_meta_blit2d_buffer *src,
1088                              struct radv_meta_blit2d_surf *dst,
1089                              unsigned num_rects,
1090                              struct radv_meta_blit2d_rect *rects)
1091 {
1092         VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline;
1093         struct radv_device *device = cmd_buffer->device;
1094         struct radv_buffer_view src_view;
1095         struct radv_image_view dst_view;
1096
1097         create_bview(cmd_buffer, src->buffer, src->offset, src->format, &src_view);
1098         create_iview(cmd_buffer, dst, &dst_view);
1099         btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1100
1101         if (device->physical_device->rad_info.chip_class >= GFX9 &&
1102             dst->image->type == VK_IMAGE_TYPE_3D)
1103                 pipeline = cmd_buffer->device->meta_state.btoi.pipeline_3d;
1104         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1105                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1106
1107         for (unsigned r = 0; r < num_rects; ++r) {
1108                 unsigned push_constants[4] = {
1109                         rects[r].dst_x,
1110                         rects[r].dst_y,
1111                         dst->layer,
1112                         src->pitch,
1113                 };
1114                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1115                                       device->meta_state.btoi.img_p_layout,
1116                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1117                                       push_constants);
1118
1119                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1120         }
1121 }
1122
1123 static void
1124 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1125                       struct radv_image_view *src,
1126                       struct radv_image_view *dst)
1127 {
1128         struct radv_device *device = cmd_buffer->device;
1129
1130         radv_meta_push_descriptor_set(cmd_buffer,
1131                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1132                                       device->meta_state.itoi.img_p_layout,
1133                                       0, /* set */
1134                                       2, /* descriptorWriteCount */
1135                                       (VkWriteDescriptorSet[]) {
1136                                               {
1137                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1138                                                        .dstBinding = 0,
1139                                                        .dstArrayElement = 0,
1140                                                        .descriptorCount = 1,
1141                                                        .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
1142                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1143                                                                {
1144                                                                        .sampler = VK_NULL_HANDLE,
1145                                                                        .imageView = radv_image_view_to_handle(src),
1146                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1147                                                                },
1148                                                        }
1149                                               },
1150                                               {
1151                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1152                                                        .dstBinding = 1,
1153                                                        .dstArrayElement = 0,
1154                                                        .descriptorCount = 1,
1155                                                        .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1156                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1157                                                                {
1158                                                                        .sampler = VK_NULL_HANDLE,
1159                                                                        .imageView = radv_image_view_to_handle(dst),
1160                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1161                                                                },
1162                                                        }
1163                                               }
1164                                       });
1165 }
1166
1167 void
1168 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1169                             struct radv_meta_blit2d_surf *src,
1170                             struct radv_meta_blit2d_surf *dst,
1171                             unsigned num_rects,
1172                             struct radv_meta_blit2d_rect *rects)
1173 {
1174         VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline;
1175         struct radv_device *device = cmd_buffer->device;
1176         struct radv_image_view src_view, dst_view;
1177
1178         create_iview(cmd_buffer, src, &src_view);
1179         create_iview(cmd_buffer, dst, &dst_view);
1180
1181         itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1182
1183         if (device->physical_device->rad_info.chip_class >= GFX9 &&
1184             src->image->type == VK_IMAGE_TYPE_3D)
1185                 pipeline = cmd_buffer->device->meta_state.itoi.pipeline_3d;
1186         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1187                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1188
1189         for (unsigned r = 0; r < num_rects; ++r) {
1190                 unsigned push_constants[6] = {
1191                         rects[r].src_x,
1192                         rects[r].src_y,
1193                         src->layer,
1194                         rects[r].dst_x,
1195                         rects[r].dst_y,
1196                         dst->layer,
1197                 };
1198                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1199                                       device->meta_state.itoi.img_p_layout,
1200                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 24,
1201                                       push_constants);
1202
1203                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1204         }
1205 }
1206
1207 static void
1208 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1209                         struct radv_image_view *dst_iview)
1210 {
1211         struct radv_device *device = cmd_buffer->device;
1212
1213         radv_meta_push_descriptor_set(cmd_buffer,
1214                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1215                                       device->meta_state.cleari.img_p_layout,
1216                                       0, /* set */
1217                                       1, /* descriptorWriteCount */
1218                                       (VkWriteDescriptorSet[]) {
1219                                               {
1220                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1221                                                       .dstBinding = 0,
1222                                                       .dstArrayElement = 0,
1223                                                       .descriptorCount = 1,
1224                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1225                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1226                                                                {
1227                                                                       .sampler = VK_NULL_HANDLE,
1228                                                                       .imageView = radv_image_view_to_handle(dst_iview),
1229                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1230                                                                },
1231                                                       }
1232                                                },
1233                                       });
1234 }
1235
1236 void
1237 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer,
1238                          struct radv_meta_blit2d_surf *dst,
1239                          const VkClearColorValue *clear_color)
1240 {
1241         VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline;
1242         struct radv_device *device = cmd_buffer->device;
1243         struct radv_image_view dst_iview;
1244
1245         create_iview(cmd_buffer, dst, &dst_iview);
1246         cleari_bind_descriptors(cmd_buffer, &dst_iview);
1247
1248         if (device->physical_device->rad_info.chip_class >= GFX9 &&
1249             dst->image->type == VK_IMAGE_TYPE_3D)
1250                 pipeline = cmd_buffer->device->meta_state.cleari.pipeline_3d;
1251
1252         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1253                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1254
1255         unsigned push_constants[5] = {
1256                 clear_color->uint32[0],
1257                 clear_color->uint32[1],
1258                 clear_color->uint32[2],
1259                 clear_color->uint32[3],
1260                 dst->layer,
1261         };
1262
1263         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1264                               device->meta_state.cleari.img_p_layout,
1265                               VK_SHADER_STAGE_COMPUTE_BIT, 0, 20,
1266                               push_constants);
1267
1268         radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1);
1269 }