src/amd/vulkan/radv_meta_bufimage.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  */
  24 #include "radv_meta.h"
  25 #include "nir/nir_builder.h"
  26
  27 /*
  28  * GFX queue: Compute shader implementation of image->buffer copy
  29  * Compute queue: implementation also of buffer->image, image->image, and image clear.
  30  */
  31
  32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options
  33  * for that.
  34  */
  35 static nir_shader *
  36 build_nir_itob_compute_shader(struct radv_device *dev, bool is_3d)
  37 {
  38         nir_builder b;
  39         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
  40         const struct glsl_type *sampler_type = glsl_sampler_type(dim,
  41                                                                  false,
  42                                                                  false,
  43                                                                  GLSL_TYPE_FLOAT);
  44         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
  45                                                              false,
  46                                                              false,
  47                                                              GLSL_TYPE_FLOAT);
  48         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
  49         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itob_cs_3d" : "meta_itob_cs");
  50         b.shader->info.cs.local_size[0] = 16;
  51         b.shader->info.cs.local_size[1] = 16;
  52         b.shader->info.cs.local_size[2] = 1;
  53         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
  54                                                       sampler_type, "s_tex");
  55         input_img->data.descriptor_set = 0;
  56         input_img->data.binding = 0;
  57
  58         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
  59                                                        img_type, "out_img");
  60         output_img->data.descriptor_set = 0;
  61         output_img->data.binding = 1;
  62
  63         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
  64         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
  65         nir_ssa_def *block_size = nir_imm_ivec4(&b,
  66                                                 b.shader->info.cs.local_size[0],
  67                                                 b.shader->info.cs.local_size[1],
  68                                                 b.shader->info.cs.local_size[2], 0);
  69
  70         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
  71
  72
  73
  74         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  75         nir_intrinsic_set_base(offset, 0);
  76         nir_intrinsic_set_range(offset, 16);
  77         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
  78         offset->num_components = is_3d ? 3 : 2;
  79         nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
  80         nir_builder_instr_insert(&b, &offset->instr);
  81
  82         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  83         nir_intrinsic_set_base(stride, 0);
  84         nir_intrinsic_set_range(stride, 16);
  85         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
  86         stride->num_components = 1;
  87         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
  88         nir_builder_instr_insert(&b, &stride->instr);
  89
  90         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
  91         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
  92         tex->sampler_dim = dim;
  93         tex->op = nir_texop_txf;
  94         tex->src[0].src_type = nir_tex_src_coord;
  95         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, is_3d ? 0x7 : 0x3));
  96         tex->src[1].src_type = nir_tex_src_lod;
  97         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
  98         tex->dest_type = nir_type_float;
  99         tex->is_array = false;
 100         tex->coord_components = is_3d ? 3 : 2;
 101         tex->texture = nir_deref_var_create(tex, input_img);
 102         tex->sampler = NULL;
 103
 104         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 105         nir_builder_instr_insert(&b, &tex->instr);
 106
 107         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 108         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 109
 110         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 111         tmp = nir_iadd(&b, tmp, pos_x);
 112
 113         nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 114
 115         nir_ssa_def *outval = &tex->dest.ssa;
 116         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 117         store->src[0] = nir_src_for_ssa(coord);
 118         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 119         store->src[2] = nir_src_for_ssa(outval);
 120         store->variables[0] = nir_deref_var_create(store, output_img);
 121
 122         nir_builder_instr_insert(&b, &store->instr);
 123         return b.shader;
 124 }
 125
 126 /* Image to buffer - don't write use image accessors */
 127 static VkResult
 128 radv_device_init_meta_itob_state(struct radv_device *device)
 129 {
 130         VkResult result;
 131         struct radv_shader_module cs = { .nir = NULL };
 132         struct radv_shader_module cs_3d = { .nir = NULL };
 133
 134         cs.nir = build_nir_itob_compute_shader(device, false);
 135         if (device->physical_device->rad_info.chip_class >= GFX9)
 136                 cs_3d.nir = build_nir_itob_compute_shader(device, true);
 137
 138         /*
 139          * two descriptors one for the image being sampled
 140          * one for the buffer being written.
 141          */
 142         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 143                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 144                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 145                 .bindingCount = 2,
 146                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 147                         {
 148                                 .binding = 0,
 149                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 150                                 .descriptorCount = 1,
 151                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 152                                 .pImmutableSamplers = NULL
 153                         },
 154                         {
 155                                 .binding = 1,
 156                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 157                                 .descriptorCount = 1,
 158                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 159                                 .pImmutableSamplers = NULL
 160                         },
 161                 }
 162         };
 163
 164         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 165                                                 &ds_create_info,
 166                                                 &device->meta_state.alloc,
 167                                                 &device->meta_state.itob.img_ds_layout);
 168         if (result != VK_SUCCESS)
 169                 goto fail;
 170
 171
 172         VkPipelineLayoutCreateInfo pl_create_info = {
 173                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 174                 .setLayoutCount = 1,
 175                 .pSetLayouts = &device->meta_state.itob.img_ds_layout,
 176                 .pushConstantRangeCount = 1,
 177                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 178         };
 179
 180         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 181                                           &pl_create_info,
 182                                           &device->meta_state.alloc,
 183                                           &device->meta_state.itob.img_p_layout);
 184         if (result != VK_SUCCESS)
 185                 goto fail;
 186
 187         /* compute shader */
 188
 189         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 190                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 191                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 192                 .module = radv_shader_module_to_handle(&cs),
 193                 .pName = "main",
 194                 .pSpecializationInfo = NULL,
 195         };
 196
 197         VkComputePipelineCreateInfo vk_pipeline_info = {
 198                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 199                 .stage = pipeline_shader_stage,
 200                 .flags = 0,
 201                 .layout = device->meta_state.itob.img_p_layout,
 202         };
 203
 204         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 205                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 206                                              1, &vk_pipeline_info, NULL,
 207                                              &device->meta_state.itob.pipeline);
 208         if (result != VK_SUCCESS)
 209                 goto fail;
 210
 211         if (device->physical_device->rad_info.chip_class >= GFX9) {
 212                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 213                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 214                         .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 215                         .module = radv_shader_module_to_handle(&cs_3d),
 216                         .pName = "main",
 217                         .pSpecializationInfo = NULL,
 218                 };
 219
 220                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 221                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 222                         .stage = pipeline_shader_stage_3d,
 223                         .flags = 0,
 224                         .layout = device->meta_state.itob.img_p_layout,
 225                 };
 226
 227                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 228                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 229                                                      1, &vk_pipeline_info_3d, NULL,
 230                                                      &device->meta_state.itob.pipeline_3d);
 231                 if (result != VK_SUCCESS)
 232                         goto fail;
 233                 ralloc_free(cs_3d.nir);
 234         }
 235         ralloc_free(cs.nir);
 236
 237         return VK_SUCCESS;
 238 fail:
 239         ralloc_free(cs.nir);
 240         ralloc_free(cs_3d.nir);
 241         return result;
 242 }
 243
 244 static void
 245 radv_device_finish_meta_itob_state(struct radv_device *device)
 246 {
 247         struct radv_meta_state *state = &device->meta_state;
 248
 249         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 250                                    state->itob.img_p_layout, &state->alloc);
 251         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 252                                         state->itob.img_ds_layout,
 253                                         &state->alloc);
 254         radv_DestroyPipeline(radv_device_to_handle(device),
 255                              state->itob.pipeline, &state->alloc);
 256         if (device->physical_device->rad_info.chip_class >= GFX9)
 257                 radv_DestroyPipeline(radv_device_to_handle(device),
 258                                      state->itob.pipeline_3d, &state->alloc);
 259 }
 260
 261 static nir_shader *
 262 build_nir_btoi_compute_shader(struct radv_device *dev)
 263 {
 264         nir_builder b;
 265         const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
 266                                                              false,
 267                                                              false,
 268                                                              GLSL_TYPE_FLOAT);
 269         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 270                                                              false,
 271                                                              false,
 272                                                              GLSL_TYPE_FLOAT);
 273         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 274         b.shader->info.name = ralloc_strdup(b.shader, "meta_btoi_cs");
 275         b.shader->info.cs.local_size[0] = 16;
 276         b.shader->info.cs.local_size[1] = 16;
 277         b.shader->info.cs.local_size[2] = 1;
 278         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 279                                                       buf_type, "s_tex");
 280         input_img->data.descriptor_set = 0;
 281         input_img->data.binding = 0;
 282
 283         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 284                                                        img_type, "out_img");
 285         output_img->data.descriptor_set = 0;
 286         output_img->data.binding = 1;
 287
 288         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 289         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 290         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 291                                                 b.shader->info.cs.local_size[0],
 292                                                 b.shader->info.cs.local_size[1],
 293                                                 b.shader->info.cs.local_size[2], 0);
 294
 295         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 296
 297         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 298         nir_intrinsic_set_base(offset, 0);
 299         nir_intrinsic_set_range(offset, 12);
 300         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 301         offset->num_components = 2;
 302         nir_ssa_dest_init(&offset->instr, &offset->dest, 2, 32, "offset");
 303         nir_builder_instr_insert(&b, &offset->instr);
 304
 305         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 306         nir_intrinsic_set_base(stride, 0);
 307         nir_intrinsic_set_range(stride, 12);
 308         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 8));
 309         stride->num_components = 1;
 310         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
 311         nir_builder_instr_insert(&b, &stride->instr);
 312
 313         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 314         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 315
 316         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 317         tmp = nir_iadd(&b, tmp, pos_x);
 318
 319         nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 320
 321         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
 322
 323         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 324         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 325         tex->op = nir_texop_txf;
 326         tex->src[0].src_type = nir_tex_src_coord;
 327         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1));
 328         tex->src[1].src_type = nir_tex_src_lod;
 329         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 330         tex->dest_type = nir_type_float;
 331         tex->is_array = false;
 332         tex->coord_components = 1;
 333         tex->texture = nir_deref_var_create(tex, input_img);
 334         tex->sampler = NULL;
 335
 336         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 337         nir_builder_instr_insert(&b, &tex->instr);
 338
 339         nir_ssa_def *outval = &tex->dest.ssa;
 340         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 341         store->src[0] = nir_src_for_ssa(img_coord);
 342         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 343         store->src[2] = nir_src_for_ssa(outval);
 344         store->variables[0] = nir_deref_var_create(store, output_img);
 345
 346         nir_builder_instr_insert(&b, &store->instr);
 347         return b.shader;
 348 }
 349
 350 /* Buffer to image - don't write use image accessors */
 351 static VkResult
 352 radv_device_init_meta_btoi_state(struct radv_device *device)
 353 {
 354         VkResult result;
 355         struct radv_shader_module cs = { .nir = NULL };
 356
 357         cs.nir = build_nir_btoi_compute_shader(device);
 358
 359         /*
 360          * two descriptors one for the image being sampled
 361          * one for the buffer being written.
 362          */
 363         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 364                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 365                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 366                 .bindingCount = 2,
 367                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 368                         {
 369                                 .binding = 0,
 370                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 371                                 .descriptorCount = 1,
 372                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 373                                 .pImmutableSamplers = NULL
 374                         },
 375                         {
 376                                 .binding = 1,
 377                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 378                                 .descriptorCount = 1,
 379                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 380                                 .pImmutableSamplers = NULL
 381                         },
 382                 }
 383         };
 384
 385         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 386                                                 &ds_create_info,
 387                                                 &device->meta_state.alloc,
 388                                                 &device->meta_state.btoi.img_ds_layout);
 389         if (result != VK_SUCCESS)
 390                 goto fail;
 391
 392
 393         VkPipelineLayoutCreateInfo pl_create_info = {
 394                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 395                 .setLayoutCount = 1,
 396                 .pSetLayouts = &device->meta_state.btoi.img_ds_layout,
 397                 .pushConstantRangeCount = 1,
 398                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 12},
 399         };
 400
 401         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 402                                           &pl_create_info,
 403                                           &device->meta_state.alloc,
 404                                           &device->meta_state.btoi.img_p_layout);
 405         if (result != VK_SUCCESS)
 406                 goto fail;
 407
 408         /* compute shader */
 409
 410         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 411                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 412                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 413                 .module = radv_shader_module_to_handle(&cs),
 414                 .pName = "main",
 415                 .pSpecializationInfo = NULL,
 416         };
 417
 418         VkComputePipelineCreateInfo vk_pipeline_info = {
 419                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 420                 .stage = pipeline_shader_stage,
 421                 .flags = 0,
 422                 .layout = device->meta_state.btoi.img_p_layout,
 423         };
 424
 425         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 426                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 427                                              1, &vk_pipeline_info, NULL,
 428                                              &device->meta_state.btoi.pipeline);
 429         if (result != VK_SUCCESS)
 430                 goto fail;
 431
 432         ralloc_free(cs.nir);
 433         return VK_SUCCESS;
 434 fail:
 435         ralloc_free(cs.nir);
 436         return result;
 437 }
 438
 439 static void
 440 radv_device_finish_meta_btoi_state(struct radv_device *device)
 441 {
 442         struct radv_meta_state *state = &device->meta_state;
 443
 444         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 445                                    state->btoi.img_p_layout, &state->alloc);
 446         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 447                                         state->btoi.img_ds_layout,
 448                                         &state->alloc);
 449         radv_DestroyPipeline(radv_device_to_handle(device),
 450                              state->btoi.pipeline, &state->alloc);
 451 }
 452
 453 static nir_shader *
 454 build_nir_itoi_compute_shader(struct radv_device *dev, bool is_3d)
 455 {
 456         nir_builder b;
 457         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
 458         const struct glsl_type *buf_type = glsl_sampler_type(dim,
 459                                                              false,
 460                                                              false,
 461                                                              GLSL_TYPE_FLOAT);
 462         const struct glsl_type *img_type = glsl_sampler_type(dim,
 463                                                              false,
 464                                                              false,
 465                                                              GLSL_TYPE_FLOAT);
 466         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 467         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itoi_cs_3d" : "meta_itoi_cs");
 468         b.shader->info.cs.local_size[0] = 16;
 469         b.shader->info.cs.local_size[1] = 16;
 470         b.shader->info.cs.local_size[2] = 1;
 471         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 472                                                       buf_type, "s_tex");
 473         input_img->data.descriptor_set = 0;
 474         input_img->data.binding = 0;
 475
 476         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 477                                                        img_type, "out_img");
 478         output_img->data.descriptor_set = 0;
 479         output_img->data.binding = 1;
 480
 481         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 482         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 483         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 484                                                 b.shader->info.cs.local_size[0],
 485                                                 b.shader->info.cs.local_size[1],
 486                                                 b.shader->info.cs.local_size[2], 0);
 487
 488         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 489
 490         nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 491         nir_intrinsic_set_base(src_offset, 0);
 492         nir_intrinsic_set_range(src_offset, 24);
 493         src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 494         src_offset->num_components = is_3d ? 3 : 2;
 495         nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, is_3d ? 3 : 2, 32, "src_offset");
 496         nir_builder_instr_insert(&b, &src_offset->instr);
 497
 498         nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 499         nir_intrinsic_set_base(dst_offset, 0);
 500         nir_intrinsic_set_range(dst_offset, 24);
 501         dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
 502         dst_offset->num_components = is_3d ? 3 : 2;
 503         nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, is_3d ? 3 : 2, 32, "dst_offset");
 504         nir_builder_instr_insert(&b, &dst_offset->instr);
 505
 506         nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa);
 507
 508         nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
 509
 510         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 511         tex->sampler_dim = dim;
 512         tex->op = nir_texop_txf;
 513         tex->src[0].src_type = nir_tex_src_coord;
 514         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, is_3d ? 0x7 : 0x3));
 515         tex->src[1].src_type = nir_tex_src_lod;
 516         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 517         tex->dest_type = nir_type_float;
 518         tex->is_array = false;
 519         tex->coord_components = is_3d ? 3 : 2;
 520         tex->texture = nir_deref_var_create(tex, input_img);
 521         tex->sampler = NULL;
 522
 523         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 524         nir_builder_instr_insert(&b, &tex->instr);
 525
 526         nir_ssa_def *outval = &tex->dest.ssa;
 527         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 528         store->src[0] = nir_src_for_ssa(dst_coord);
 529         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 530         store->src[2] = nir_src_for_ssa(outval);
 531         store->variables[0] = nir_deref_var_create(store, output_img);
 532
 533         nir_builder_instr_insert(&b, &store->instr);
 534         return b.shader;
 535 }
 536
 537 /* image to image - don't write use image accessors */
 538 static VkResult
 539 radv_device_init_meta_itoi_state(struct radv_device *device)
 540 {
 541         VkResult result;
 542         struct radv_shader_module cs = { .nir = NULL };
 543         struct radv_shader_module cs_3d = { .nir = NULL };
 544         cs.nir = build_nir_itoi_compute_shader(device, false);
 545         if (device->physical_device->rad_info.chip_class >= GFX9)
 546                 cs_3d.nir = build_nir_itoi_compute_shader(device, true);
 547         /*
 548          * two descriptors one for the image being sampled
 549          * one for the buffer being written.
 550          */
 551         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 552                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 553                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 554                 .bindingCount = 2,
 555                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 556                         {
 557                                 .binding = 0,
 558                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 559                                 .descriptorCount = 1,
 560                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 561                                 .pImmutableSamplers = NULL
 562                         },
 563                         {
 564                                 .binding = 1,
 565                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 566                                 .descriptorCount = 1,
 567                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 568                                 .pImmutableSamplers = NULL
 569                         },
 570                 }
 571         };
 572
 573         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 574                                                 &ds_create_info,
 575                                                 &device->meta_state.alloc,
 576                                                 &device->meta_state.itoi.img_ds_layout);
 577         if (result != VK_SUCCESS)
 578                 goto fail;
 579
 580
 581         VkPipelineLayoutCreateInfo pl_create_info = {
 582                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 583                 .setLayoutCount = 1,
 584                 .pSetLayouts = &device->meta_state.itoi.img_ds_layout,
 585                 .pushConstantRangeCount = 1,
 586                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 24},
 587         };
 588
 589         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 590                                           &pl_create_info,
 591                                           &device->meta_state.alloc,
 592                                           &device->meta_state.itoi.img_p_layout);
 593         if (result != VK_SUCCESS)
 594                 goto fail;
 595
 596         /* compute shader */
 597
 598         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 599                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 600                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 601                 .module = radv_shader_module_to_handle(&cs),
 602                 .pName = "main",
 603                 .pSpecializationInfo = NULL,
 604         };
 605
 606         VkComputePipelineCreateInfo vk_pipeline_info = {
 607                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 608                 .stage = pipeline_shader_stage,
 609                 .flags = 0,
 610                 .layout = device->meta_state.itoi.img_p_layout,
 611         };
 612
 613         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 614                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 615                                              1, &vk_pipeline_info, NULL,
 616                                              &device->meta_state.itoi.pipeline);
 617         if (result != VK_SUCCESS)
 618                 goto fail;
 619
 620         if (device->physical_device->rad_info.chip_class >= GFX9) {
 621                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 622                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 623 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 624                         .module = radv_shader_module_to_handle(&cs_3d),
 625                         .pName = "main",
 626                         .pSpecializationInfo = NULL,
 627                 };
 628
 629                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 630                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 631                         .stage = pipeline_shader_stage_3d,
 632                         .flags = 0,
 633                         .layout = device->meta_state.itoi.img_p_layout,
 634                 };
 635
 636                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 637                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 638                                                      1, &vk_pipeline_info_3d, NULL,
 639                                                      &device->meta_state.itoi.pipeline_3d);
 640
 641                 ralloc_free(cs_3d.nir);
 642         }
 643         ralloc_free(cs.nir);
 644
 645         return VK_SUCCESS;
 646 fail:
 647         ralloc_free(cs.nir);
 648         ralloc_free(cs_3d.nir);
 649         return result;
 650 }
 651
 652 static void
 653 radv_device_finish_meta_itoi_state(struct radv_device *device)
 654 {
 655         struct radv_meta_state *state = &device->meta_state;
 656
 657         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 658                                    state->itoi.img_p_layout, &state->alloc);
 659         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 660                                         state->itoi.img_ds_layout,
 661                                         &state->alloc);
 662         radv_DestroyPipeline(radv_device_to_handle(device),
 663                              state->itoi.pipeline, &state->alloc);
 664         if (device->physical_device->rad_info.chip_class >= GFX9)
 665                 radv_DestroyPipeline(radv_device_to_handle(device),
 666                                      state->itoi.pipeline_3d, &state->alloc);
 667 }
 668
 669 static nir_shader *
 670 build_nir_cleari_compute_shader(struct radv_device *dev, bool is_3d)
 671 {
 672         nir_builder b;
 673         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
 674         const struct glsl_type *img_type = glsl_sampler_type(dim,
 675                                                              false,
 676                                                              false,
 677                                                              GLSL_TYPE_FLOAT);
 678         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 679         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_cleari_cs_3d" : "meta_cleari_cs");
 680         b.shader->info.cs.local_size[0] = 16;
 681         b.shader->info.cs.local_size[1] = 16;
 682         b.shader->info.cs.local_size[2] = 1;
 683
 684         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 685                                                        img_type, "out_img");
 686         output_img->data.descriptor_set = 0;
 687         output_img->data.binding = 0;
 688
 689         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 690         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 691         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 692                                                 b.shader->info.cs.local_size[0],
 693                                                 b.shader->info.cs.local_size[1],
 694                                                 b.shader->info.cs.local_size[2], 0);
 695
 696         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 697
 698         nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 699         nir_intrinsic_set_base(clear_val, 0);
 700         nir_intrinsic_set_range(clear_val, 20);
 701         clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 702         clear_val->num_components = 4;
 703         nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value");
 704         nir_builder_instr_insert(&b, &clear_val->instr);
 705
 706         nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 707         nir_intrinsic_set_base(layer, 0);
 708         nir_intrinsic_set_range(layer, 20);
 709         layer->src[0] = nir_src_for_ssa(nir_imm_int(&b, 16));
 710         layer->num_components = 1;
 711         nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
 712         nir_builder_instr_insert(&b, &layer->instr);
 713
 714         nir_ssa_def *global_z = nir_iadd(&b, nir_channel(&b, global_id, 2), &layer->dest.ssa);
 715
 716         nir_ssa_def *comps[4];
 717         comps[0] = nir_channel(&b, global_id, 0);
 718         comps[1] = nir_channel(&b, global_id, 1);
 719         comps[2] = global_z;
 720         comps[3] = nir_imm_int(&b, 0);
 721         global_id = nir_vec(&b, comps, 4);
 722
 723         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 724         store->src[0] = nir_src_for_ssa(global_id);
 725         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 726         store->src[2] = nir_src_for_ssa(&clear_val->dest.ssa);
 727         store->variables[0] = nir_deref_var_create(store, output_img);
 728
 729         nir_builder_instr_insert(&b, &store->instr);
 730         return b.shader;
 731 }
 732
 733 static VkResult
 734 radv_device_init_meta_cleari_state(struct radv_device *device)
 735 {
 736         VkResult result;
 737         struct radv_shader_module cs = { .nir = NULL };
 738         struct radv_shader_module cs_3d = { .nir = NULL };
 739         cs.nir = build_nir_cleari_compute_shader(device, false);
 740         if (device->physical_device->rad_info.chip_class >= GFX9)
 741                 cs_3d.nir = build_nir_cleari_compute_shader(device, true);
 742
 743         /*
 744          * two descriptors one for the image being sampled
 745          * one for the buffer being written.
 746          */
 747         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 748                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 749                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 750                 .bindingCount = 1,
 751                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 752                         {
 753                                 .binding = 0,
 754                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 755                                 .descriptorCount = 1,
 756                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 757                                 .pImmutableSamplers = NULL
 758                         },
 759                 }
 760         };
 761
 762         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 763                                                 &ds_create_info,
 764                                                 &device->meta_state.alloc,
 765                                                 &device->meta_state.cleari.img_ds_layout);
 766         if (result != VK_SUCCESS)
 767                 goto fail;
 768
 769
 770         VkPipelineLayoutCreateInfo pl_create_info = {
 771                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 772                 .setLayoutCount = 1,
 773                 .pSetLayouts = &device->meta_state.cleari.img_ds_layout,
 774                 .pushConstantRangeCount = 1,
 775                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 20},
 776         };
 777
 778         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 779                                           &pl_create_info,
 780                                           &device->meta_state.alloc,
 781                                           &device->meta_state.cleari.img_p_layout);
 782         if (result != VK_SUCCESS)
 783                 goto fail;
 784
 785         /* compute shader */
 786
 787         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 788                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 789                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 790                 .module = radv_shader_module_to_handle(&cs),
 791                 .pName = "main",
 792                 .pSpecializationInfo = NULL,
 793         };
 794
 795         VkComputePipelineCreateInfo vk_pipeline_info = {
 796                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 797                 .stage = pipeline_shader_stage,
 798                 .flags = 0,
 799                 .layout = device->meta_state.cleari.img_p_layout,
 800         };
 801
 802         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 803                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 804                                              1, &vk_pipeline_info, NULL,
 805                                              &device->meta_state.cleari.pipeline);
 806         if (result != VK_SUCCESS)
 807                 goto fail;
 808
 809
 810         if (device->physical_device->rad_info.chip_class >= GFX9) {
 811                 /* compute shader */
 812                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 813                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 814                         .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 815                         .module = radv_shader_module_to_handle(&cs_3d),
 816                         .pName = "main",
 817                         .pSpecializationInfo = NULL,
 818                 };
 819
 820                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 821                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 822                         .stage = pipeline_shader_stage_3d,
 823                         .flags = 0,
 824                         .layout = device->meta_state.cleari.img_p_layout,
 825                 };
 826
 827                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 828                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 829                                                      1, &vk_pipeline_info_3d, NULL,
 830                                                      &device->meta_state.cleari.pipeline_3d);
 831                 if (result != VK_SUCCESS)
 832                         goto fail;
 833
 834                 ralloc_free(cs_3d.nir);
 835         }
 836         ralloc_free(cs.nir);
 837         return VK_SUCCESS;
 838 fail:
 839         ralloc_free(cs.nir);
 840         ralloc_free(cs_3d.nir);
 841         return result;
 842 }
 843
 844 static void
 845 radv_device_finish_meta_cleari_state(struct radv_device *device)
 846 {
 847         struct radv_meta_state *state = &device->meta_state;
 848
 849         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 850                                    state->cleari.img_p_layout, &state->alloc);
 851         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 852                                         state->cleari.img_ds_layout,
 853                                         &state->alloc);
 854         radv_DestroyPipeline(radv_device_to_handle(device),
 855                              state->cleari.pipeline, &state->alloc);
 856         radv_DestroyPipeline(radv_device_to_handle(device),
 857                              state->cleari.pipeline_3d, &state->alloc);
 858 }
 859
 860 void
 861 radv_device_finish_meta_bufimage_state(struct radv_device *device)
 862 {
 863         radv_device_finish_meta_itob_state(device);
 864         radv_device_finish_meta_btoi_state(device);
 865         radv_device_finish_meta_itoi_state(device);
 866         radv_device_finish_meta_cleari_state(device);
 867 }
 868
 869 VkResult
 870 radv_device_init_meta_bufimage_state(struct radv_device *device)
 871 {
 872         VkResult result;
 873
 874         result = radv_device_init_meta_itob_state(device);
 875         if (result != VK_SUCCESS)
 876                 return result;
 877
 878         result = radv_device_init_meta_btoi_state(device);
 879         if (result != VK_SUCCESS)
 880                 goto fail_itob;
 881
 882         result = radv_device_init_meta_itoi_state(device);
 883         if (result != VK_SUCCESS)
 884                 goto fail_btoi;
 885
 886         result = radv_device_init_meta_cleari_state(device);
 887         if (result != VK_SUCCESS)
 888                 goto fail_itoi;
 889
 890         return VK_SUCCESS;
 891 fail_itoi:
 892         radv_device_finish_meta_itoi_state(device);
 893 fail_btoi:
 894         radv_device_finish_meta_btoi_state(device);
 895 fail_itob:
 896         radv_device_finish_meta_itob_state(device);
 897         return result;
 898 }
 899
 900 static void
 901 create_iview(struct radv_cmd_buffer *cmd_buffer,
 902              struct radv_meta_blit2d_surf *surf,
 903              struct radv_image_view *iview)
 904 {
 905         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
 906                 radv_meta_get_view_type(surf->image);
 907         radv_image_view_init(iview, cmd_buffer->device,
 908                              &(VkImageViewCreateInfo) {
 909                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 910                                              .image = radv_image_to_handle(surf->image),
 911                                              .viewType = view_type,
 912                                              .format = surf->format,
 913                                              .subresourceRange = {
 914                                              .aspectMask = surf->aspect_mask,
 915                                              .baseMipLevel = surf->level,
 916                                              .levelCount = 1,
 917                                              .baseArrayLayer = surf->layer,
 918                                              .layerCount = 1
 919                                      },
 920                              });
 921 }
 922
 923 static void
 924 create_bview(struct radv_cmd_buffer *cmd_buffer,
 925              struct radv_buffer *buffer,
 926              unsigned offset,
 927              VkFormat format,
 928              struct radv_buffer_view *bview)
 929 {
 930         radv_buffer_view_init(bview, cmd_buffer->device,
 931                               &(VkBufferViewCreateInfo) {
 932                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
 933                                       .flags = 0,
 934                                       .buffer = radv_buffer_to_handle(buffer),
 935                                       .format = format,
 936                                       .offset = offset,
 937                                       .range = VK_WHOLE_SIZE,
 938                               });
 939
 940 }
 941
 942 static void
 943 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
 944                       struct radv_image_view *src,
 945                       struct radv_buffer_view *dst)
 946 {
 947         struct radv_device *device = cmd_buffer->device;
 948
 949         radv_meta_push_descriptor_set(cmd_buffer,
 950                                       VK_PIPELINE_BIND_POINT_COMPUTE,
 951                                       device->meta_state.itob.img_p_layout,
 952                                       0, /* set */
 953                                       2, /* descriptorWriteCount */
 954                                       (VkWriteDescriptorSet[]) {
 955                                               {
 956                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 957                                                       .dstBinding = 0,
 958                                                       .dstArrayElement = 0,
 959                                                       .descriptorCount = 1,
 960                                                       .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 961                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 962                                                               {
 963                                                                       .sampler = VK_NULL_HANDLE,
 964                                                                       .imageView = radv_image_view_to_handle(src),
 965                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 966                                                               },
 967                                                       }
 968                                               },
 969                                               {
 970                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 971                                                       .dstBinding = 1,
 972                                                       .dstArrayElement = 0,
 973                                                       .descriptorCount = 1,
 974                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 975                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(dst) },
 976                                               }
 977                                       });
 978 }
 979
 980 void
 981 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 982                           struct radv_meta_blit2d_surf *src,
 983                           struct radv_meta_blit2d_buffer *dst,
 984                           unsigned num_rects,
 985                           struct radv_meta_blit2d_rect *rects)
 986 {
 987         VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline;
 988         struct radv_device *device = cmd_buffer->device;
 989         struct radv_image_view src_view;
 990         struct radv_buffer_view dst_view;
 991
 992         create_iview(cmd_buffer, src, &src_view);
 993         create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &dst_view);
 994         itob_bind_descriptors(cmd_buffer, &src_view, &dst_view);
 995
 996         if (device->physical_device->rad_info.chip_class >= GFX9 &&
 997             src->image->type == VK_IMAGE_TYPE_3D)
 998                 pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d;
 999
1000         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1001                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1002
1003         for (unsigned r = 0; r < num_rects; ++r) {
1004                 unsigned push_constants[4] = {
1005                         rects[r].src_x,
1006                         rects[r].src_y,
1007                         src->layer,
1008                         dst->pitch
1009                 };
1010                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1011                                       device->meta_state.itob.img_p_layout,
1012                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1013                                       push_constants);
1014
1015                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1016         }
1017 }
1018
1019 static void
1020 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1021                       struct radv_buffer_view *src,
1022                       struct radv_image_view *dst)
1023 {
1024         struct radv_device *device = cmd_buffer->device;
1025
1026         radv_meta_push_descriptor_set(cmd_buffer,
1027                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1028                                       device->meta_state.btoi.img_p_layout,
1029                                       0, /* set */
1030                                       2, /* descriptorWriteCount */
1031                                       (VkWriteDescriptorSet[]) {
1032                                               {
1033                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1034                                                       .dstBinding = 0,
1035                                                       .dstArrayElement = 0,
1036                                                       .descriptorCount = 1,
1037                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1038                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(src) },
1039                                               },
1040                                               {
1041                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1042                                                       .dstBinding = 1,
1043                                                       .dstArrayElement = 0,
1044                                                       .descriptorCount = 1,
1045                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1046                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1047                                                               {
1048                                                                       .sampler = VK_NULL_HANDLE,
1049                                                                       .imageView = radv_image_view_to_handle(dst),
1050                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1051                                                               },
1052                                                       }
1053                                               }
1054                                       });
1055 }
1056
1057 void
1058 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1059                              struct radv_meta_blit2d_buffer *src,
1060                              struct radv_meta_blit2d_surf *dst,
1061                              unsigned num_rects,
1062                              struct radv_meta_blit2d_rect *rects)
1063 {
1064         VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline;
1065         struct radv_device *device = cmd_buffer->device;
1066         struct radv_buffer_view src_view;
1067         struct radv_image_view dst_view;
1068
1069         create_bview(cmd_buffer, src->buffer, src->offset, src->format, &src_view);
1070         create_iview(cmd_buffer, dst, &dst_view);
1071         btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1072
1073         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1074                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1075
1076         for (unsigned r = 0; r < num_rects; ++r) {
1077                 unsigned push_constants[3] = {
1078                         rects[r].dst_x,
1079                         rects[r].dst_y,
1080                         src->pitch
1081                 };
1082                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1083                                       device->meta_state.btoi.img_p_layout,
1084                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 12,
1085                                       push_constants);
1086
1087                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1088         }
1089 }
1090
1091 static void
1092 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1093                       struct radv_image_view *src,
1094                       struct radv_image_view *dst)
1095 {
1096         struct radv_device *device = cmd_buffer->device;
1097
1098         radv_meta_push_descriptor_set(cmd_buffer,
1099                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1100                                       device->meta_state.itoi.img_p_layout,
1101                                       0, /* set */
1102                                       2, /* descriptorWriteCount */
1103                                       (VkWriteDescriptorSet[]) {
1104                                               {
1105                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1106                                                        .dstBinding = 0,
1107                                                        .dstArrayElement = 0,
1108                                                        .descriptorCount = 1,
1109                                                        .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
1110                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1111                                                                {
1112                                                                        .sampler = VK_NULL_HANDLE,
1113                                                                        .imageView = radv_image_view_to_handle(src),
1114                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1115                                                                },
1116                                                        }
1117                                               },
1118                                               {
1119                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1120                                                        .dstBinding = 1,
1121                                                        .dstArrayElement = 0,
1122                                                        .descriptorCount = 1,
1123                                                        .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1124                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1125                                                                {
1126                                                                        .sampler = VK_NULL_HANDLE,
1127                                                                        .imageView = radv_image_view_to_handle(dst),
1128                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1129                                                                },
1130                                                        }
1131                                               }
1132                                       });
1133 }
1134
1135 void
1136 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1137                             struct radv_meta_blit2d_surf *src,
1138                             struct radv_meta_blit2d_surf *dst,
1139                             unsigned num_rects,
1140                             struct radv_meta_blit2d_rect *rects)
1141 {
1142         VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline;
1143         struct radv_device *device = cmd_buffer->device;
1144         struct radv_image_view src_view, dst_view;
1145
1146         create_iview(cmd_buffer, src, &src_view);
1147         create_iview(cmd_buffer, dst, &dst_view);
1148
1149         itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1150
1151         if (device->physical_device->rad_info.chip_class >= GFX9 &&
1152             src->image->type == VK_IMAGE_TYPE_3D)
1153                 pipeline = cmd_buffer->device->meta_state.itoi.pipeline_3d;
1154         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1155                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1156
1157         for (unsigned r = 0; r < num_rects; ++r) {
1158                 unsigned push_constants[6] = {
1159                         rects[r].src_x,
1160                         rects[r].src_y,
1161                         src->layer,
1162                         rects[r].dst_x,
1163                         rects[r].dst_y,
1164                         dst->layer,
1165                 };
1166                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1167                                       device->meta_state.itoi.img_p_layout,
1168                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 24,
1169                                       push_constants);
1170
1171                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1172         }
1173 }
1174
1175 static void
1176 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1177                         struct radv_image_view *dst_iview)
1178 {
1179         struct radv_device *device = cmd_buffer->device;
1180
1181         radv_meta_push_descriptor_set(cmd_buffer,
1182                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1183                                       device->meta_state.cleari.img_p_layout,
1184                                       0, /* set */
1185                                       1, /* descriptorWriteCount */
1186                                       (VkWriteDescriptorSet[]) {
1187                                               {
1188                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1189                                                       .dstBinding = 0,
1190                                                       .dstArrayElement = 0,
1191                                                       .descriptorCount = 1,
1192                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1193                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1194                                                                {
1195                                                                       .sampler = VK_NULL_HANDLE,
1196                                                                       .imageView = radv_image_view_to_handle(dst_iview),
1197                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1198                                                                },
1199                                                       }
1200                                                },
1201                                       });
1202 }
1203
1204 void
1205 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer,
1206                          struct radv_meta_blit2d_surf *dst,
1207                          const VkClearColorValue *clear_color)
1208 {
1209         VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline;
1210         struct radv_device *device = cmd_buffer->device;
1211         struct radv_image_view dst_iview;
1212
1213         create_iview(cmd_buffer, dst, &dst_iview);
1214         cleari_bind_descriptors(cmd_buffer, &dst_iview);
1215
1216         if (device->physical_device->rad_info.chip_class >= GFX9 &&
1217             dst->image->type == VK_IMAGE_TYPE_3D)
1218                 pipeline = cmd_buffer->device->meta_state.cleari.pipeline_3d;
1219
1220         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1221                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1222
1223         unsigned push_constants[5] = {
1224                 clear_color->uint32[0],
1225                 clear_color->uint32[1],
1226                 clear_color->uint32[2],
1227                 clear_color->uint32[3],
1228                 dst->layer,
1229         };
1230
1231         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1232                               device->meta_state.cleari.img_p_layout,
1233                               VK_SHADER_STAGE_COMPUTE_BIT, 0, 20,
1234                               push_constants);
1235
1236         radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1);
1237 }