src/amd/vulkan/radv_meta_bufimage.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  */
  24 #include "radv_meta.h"
  25 #include "nir/nir_builder.h"
  26
  27 /*
  28  * GFX queue: Compute shader implementation of image->buffer copy
  29  * Compute queue: implementation also of buffer->image, image->image, and image clear.
  30  */
  31
  32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options
  33  * for that.
  34  */
  35 static nir_shader *
  36 build_nir_itob_compute_shader(struct radv_device *dev, bool is_3d)
  37 {
  38         nir_builder b;
  39         enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
  40         const struct glsl_type *sampler_type = glsl_sampler_type(dim,
  41                                                                  false,
  42                                                                  false,
  43                                                                  GLSL_TYPE_FLOAT);
  44         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
  45                                                              false,
  46                                                              false,
  47                                                              GLSL_TYPE_FLOAT);
  48         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
  49         b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itob_cs_3d" : "meta_itob_cs");
  50         b.shader->info.cs.local_size[0] = 16;
  51         b.shader->info.cs.local_size[1] = 16;
  52         b.shader->info.cs.local_size[2] = 1;
  53         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
  54                                                       sampler_type, "s_tex");
  55         input_img->data.descriptor_set = 0;
  56         input_img->data.binding = 0;
  57
  58         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
  59                                                        img_type, "out_img");
  60         output_img->data.descriptor_set = 0;
  61         output_img->data.binding = 1;
  62
  63         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
  64         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
  65         nir_ssa_def *block_size = nir_imm_ivec4(&b,
  66                                                 b.shader->info.cs.local_size[0],
  67                                                 b.shader->info.cs.local_size[1],
  68                                                 b.shader->info.cs.local_size[2], 0);
  69
  70         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
  71
  72
  73
  74         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  75         nir_intrinsic_set_base(offset, 0);
  76         nir_intrinsic_set_range(offset, 16);
  77         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
  78         offset->num_components = is_3d ? 3 : 2;
  79         nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
  80         nir_builder_instr_insert(&b, &offset->instr);
  81
  82         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  83         nir_intrinsic_set_base(stride, 0);
  84         nir_intrinsic_set_range(stride, 16);
  85         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
  86         stride->num_components = 1;
  87         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
  88         nir_builder_instr_insert(&b, &stride->instr);
  89
  90         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
  91         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
  92         tex->sampler_dim = dim;
  93         tex->op = nir_texop_txf;
  94         tex->src[0].src_type = nir_tex_src_coord;
  95         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, is_3d ? 0x7 : 0x3));
  96         tex->src[1].src_type = nir_tex_src_lod;
  97         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
  98         tex->dest_type = nir_type_float;
  99         tex->is_array = false;
 100         tex->coord_components = is_3d ? 3 : 2;
 101         tex->texture = nir_deref_var_create(tex, input_img);
 102         tex->sampler = NULL;
 103
 104         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 105         nir_builder_instr_insert(&b, &tex->instr);
 106
 107         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 108         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 109
 110         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 111         tmp = nir_iadd(&b, tmp, pos_x);
 112
 113         nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 114
 115         nir_ssa_def *outval = &tex->dest.ssa;
 116         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 117         store->src[0] = nir_src_for_ssa(coord);
 118         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 119         store->src[2] = nir_src_for_ssa(outval);
 120         store->variables[0] = nir_deref_var_create(store, output_img);
 121
 122         nir_builder_instr_insert(&b, &store->instr);
 123         return b.shader;
 124 }
 125
 126 /* Image to buffer - don't write use image accessors */
 127 static VkResult
 128 radv_device_init_meta_itob_state(struct radv_device *device)
 129 {
 130         VkResult result;
 131         struct radv_shader_module cs = { .nir = NULL };
 132         struct radv_shader_module cs_3d = { .nir = NULL };
 133
 134         cs.nir = build_nir_itob_compute_shader(device, false);
 135         if (device->physical_device->rad_info.chip_class >= GFX9)
 136                 cs_3d.nir = build_nir_itob_compute_shader(device, true);
 137
 138         /*
 139          * two descriptors one for the image being sampled
 140          * one for the buffer being written.
 141          */
 142         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 143                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 144                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 145                 .bindingCount = 2,
 146                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 147                         {
 148                                 .binding = 0,
 149                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 150                                 .descriptorCount = 1,
 151                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 152                                 .pImmutableSamplers = NULL
 153                         },
 154                         {
 155                                 .binding = 1,
 156                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 157                                 .descriptorCount = 1,
 158                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 159                                 .pImmutableSamplers = NULL
 160                         },
 161                 }
 162         };
 163
 164         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 165                                                 &ds_create_info,
 166                                                 &device->meta_state.alloc,
 167                                                 &device->meta_state.itob.img_ds_layout);
 168         if (result != VK_SUCCESS)
 169                 goto fail;
 170
 171
 172         VkPipelineLayoutCreateInfo pl_create_info = {
 173                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 174                 .setLayoutCount = 1,
 175                 .pSetLayouts = &device->meta_state.itob.img_ds_layout,
 176                 .pushConstantRangeCount = 1,
 177                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 178         };
 179
 180         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 181                                           &pl_create_info,
 182                                           &device->meta_state.alloc,
 183                                           &device->meta_state.itob.img_p_layout);
 184         if (result != VK_SUCCESS)
 185                 goto fail;
 186
 187         /* compute shader */
 188
 189         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 190                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 191                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 192                 .module = radv_shader_module_to_handle(&cs),
 193                 .pName = "main",
 194                 .pSpecializationInfo = NULL,
 195         };
 196
 197         VkComputePipelineCreateInfo vk_pipeline_info = {
 198                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 199                 .stage = pipeline_shader_stage,
 200                 .flags = 0,
 201                 .layout = device->meta_state.itob.img_p_layout,
 202         };
 203
 204         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 205                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 206                                              1, &vk_pipeline_info, NULL,
 207                                              &device->meta_state.itob.pipeline);
 208         if (result != VK_SUCCESS)
 209                 goto fail;
 210
 211         if (device->physical_device->rad_info.chip_class >= GFX9) {
 212                 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
 213                         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 214                         .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 215                         .module = radv_shader_module_to_handle(&cs_3d),
 216                         .pName = "main",
 217                         .pSpecializationInfo = NULL,
 218                 };
 219
 220                 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
 221                         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 222                         .stage = pipeline_shader_stage_3d,
 223                         .flags = 0,
 224                         .layout = device->meta_state.itob.img_p_layout,
 225                 };
 226
 227                 result = radv_CreateComputePipelines(radv_device_to_handle(device),
 228                                                      radv_pipeline_cache_to_handle(&device->meta_state.cache),
 229                                                      1, &vk_pipeline_info_3d, NULL,
 230                                                      &device->meta_state.itob.pipeline_3d);
 231                 if (result != VK_SUCCESS)
 232                         goto fail;
 233                 ralloc_free(cs_3d.nir);
 234         }
 235         ralloc_free(cs.nir);
 236
 237         return VK_SUCCESS;
 238 fail:
 239         ralloc_free(cs.nir);
 240         ralloc_free(cs_3d.nir);
 241         return result;
 242 }
 243
 244 static void
 245 radv_device_finish_meta_itob_state(struct radv_device *device)
 246 {
 247         struct radv_meta_state *state = &device->meta_state;
 248
 249         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 250                                    state->itob.img_p_layout, &state->alloc);
 251         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 252                                         state->itob.img_ds_layout,
 253                                         &state->alloc);
 254         radv_DestroyPipeline(radv_device_to_handle(device),
 255                              state->itob.pipeline, &state->alloc);
 256         if (device->physical_device->rad_info.chip_class >= GFX9)
 257                 radv_DestroyPipeline(radv_device_to_handle(device),
 258                                      state->itob.pipeline_3d, &state->alloc);
 259 }
 260
 261 static nir_shader *
 262 build_nir_btoi_compute_shader(struct radv_device *dev)
 263 {
 264         nir_builder b;
 265         const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
 266                                                              false,
 267                                                              false,
 268                                                              GLSL_TYPE_FLOAT);
 269         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 270                                                              false,
 271                                                              false,
 272                                                              GLSL_TYPE_FLOAT);
 273         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 274         b.shader->info.name = ralloc_strdup(b.shader, "meta_btoi_cs");
 275         b.shader->info.cs.local_size[0] = 16;
 276         b.shader->info.cs.local_size[1] = 16;
 277         b.shader->info.cs.local_size[2] = 1;
 278         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 279                                                       buf_type, "s_tex");
 280         input_img->data.descriptor_set = 0;
 281         input_img->data.binding = 0;
 282
 283         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 284                                                        img_type, "out_img");
 285         output_img->data.descriptor_set = 0;
 286         output_img->data.binding = 1;
 287
 288         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 289         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 290         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 291                                                 b.shader->info.cs.local_size[0],
 292                                                 b.shader->info.cs.local_size[1],
 293                                                 b.shader->info.cs.local_size[2], 0);
 294
 295         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 296
 297         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 298         nir_intrinsic_set_base(offset, 0);
 299         nir_intrinsic_set_range(offset, 12);
 300         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 301         offset->num_components = 2;
 302         nir_ssa_dest_init(&offset->instr, &offset->dest, 2, 32, "offset");
 303         nir_builder_instr_insert(&b, &offset->instr);
 304
 305         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 306         nir_intrinsic_set_base(stride, 0);
 307         nir_intrinsic_set_range(stride, 12);
 308         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 8));
 309         stride->num_components = 1;
 310         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
 311         nir_builder_instr_insert(&b, &stride->instr);
 312
 313         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 314         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 315
 316         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 317         tmp = nir_iadd(&b, tmp, pos_x);
 318
 319         nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 320
 321         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
 322
 323         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 324         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 325         tex->op = nir_texop_txf;
 326         tex->src[0].src_type = nir_tex_src_coord;
 327         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1));
 328         tex->src[1].src_type = nir_tex_src_lod;
 329         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 330         tex->dest_type = nir_type_float;
 331         tex->is_array = false;
 332         tex->coord_components = 1;
 333         tex->texture = nir_deref_var_create(tex, input_img);
 334         tex->sampler = NULL;
 335
 336         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 337         nir_builder_instr_insert(&b, &tex->instr);
 338
 339         nir_ssa_def *outval = &tex->dest.ssa;
 340         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 341         store->src[0] = nir_src_for_ssa(img_coord);
 342         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 343         store->src[2] = nir_src_for_ssa(outval);
 344         store->variables[0] = nir_deref_var_create(store, output_img);
 345
 346         nir_builder_instr_insert(&b, &store->instr);
 347         return b.shader;
 348 }
 349
 350 /* Buffer to image - don't write use image accessors */
 351 static VkResult
 352 radv_device_init_meta_btoi_state(struct radv_device *device)
 353 {
 354         VkResult result;
 355         struct radv_shader_module cs = { .nir = NULL };
 356
 357         cs.nir = build_nir_btoi_compute_shader(device);
 358
 359         /*
 360          * two descriptors one for the image being sampled
 361          * one for the buffer being written.
 362          */
 363         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 364                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 365                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 366                 .bindingCount = 2,
 367                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 368                         {
 369                                 .binding = 0,
 370                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 371                                 .descriptorCount = 1,
 372                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 373                                 .pImmutableSamplers = NULL
 374                         },
 375                         {
 376                                 .binding = 1,
 377                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 378                                 .descriptorCount = 1,
 379                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 380                                 .pImmutableSamplers = NULL
 381                         },
 382                 }
 383         };
 384
 385         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 386                                                 &ds_create_info,
 387                                                 &device->meta_state.alloc,
 388                                                 &device->meta_state.btoi.img_ds_layout);
 389         if (result != VK_SUCCESS)
 390                 goto fail;
 391
 392
 393         VkPipelineLayoutCreateInfo pl_create_info = {
 394                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 395                 .setLayoutCount = 1,
 396                 .pSetLayouts = &device->meta_state.btoi.img_ds_layout,
 397                 .pushConstantRangeCount = 1,
 398                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 12},
 399         };
 400
 401         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 402                                           &pl_create_info,
 403                                           &device->meta_state.alloc,
 404                                           &device->meta_state.btoi.img_p_layout);
 405         if (result != VK_SUCCESS)
 406                 goto fail;
 407
 408         /* compute shader */
 409
 410         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 411                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 412                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 413                 .module = radv_shader_module_to_handle(&cs),
 414                 .pName = "main",
 415                 .pSpecializationInfo = NULL,
 416         };
 417
 418         VkComputePipelineCreateInfo vk_pipeline_info = {
 419                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 420                 .stage = pipeline_shader_stage,
 421                 .flags = 0,
 422                 .layout = device->meta_state.btoi.img_p_layout,
 423         };
 424
 425         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 426                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 427                                              1, &vk_pipeline_info, NULL,
 428                                              &device->meta_state.btoi.pipeline);
 429         if (result != VK_SUCCESS)
 430                 goto fail;
 431
 432         ralloc_free(cs.nir);
 433         return VK_SUCCESS;
 434 fail:
 435         ralloc_free(cs.nir);
 436         return result;
 437 }
 438
 439 static void
 440 radv_device_finish_meta_btoi_state(struct radv_device *device)
 441 {
 442         struct radv_meta_state *state = &device->meta_state;
 443
 444         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 445                                    state->btoi.img_p_layout, &state->alloc);
 446         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 447                                         state->btoi.img_ds_layout,
 448                                         &state->alloc);
 449         radv_DestroyPipeline(radv_device_to_handle(device),
 450                              state->btoi.pipeline, &state->alloc);
 451 }
 452
 453 static nir_shader *
 454 build_nir_itoi_compute_shader(struct radv_device *dev)
 455 {
 456         nir_builder b;
 457         const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 458                                                              false,
 459                                                              false,
 460                                                              GLSL_TYPE_FLOAT);
 461         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 462                                                              false,
 463                                                              false,
 464                                                              GLSL_TYPE_FLOAT);
 465         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 466         b.shader->info.name = ralloc_strdup(b.shader, "meta_itoi_cs");
 467         b.shader->info.cs.local_size[0] = 16;
 468         b.shader->info.cs.local_size[1] = 16;
 469         b.shader->info.cs.local_size[2] = 1;
 470         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 471                                                       buf_type, "s_tex");
 472         input_img->data.descriptor_set = 0;
 473         input_img->data.binding = 0;
 474
 475         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 476                                                        img_type, "out_img");
 477         output_img->data.descriptor_set = 0;
 478         output_img->data.binding = 1;
 479
 480         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 481         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 482         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 483                                                 b.shader->info.cs.local_size[0],
 484                                                 b.shader->info.cs.local_size[1],
 485                                                 b.shader->info.cs.local_size[2], 0);
 486
 487         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 488
 489         nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 490         nir_intrinsic_set_base(src_offset, 0);
 491         nir_intrinsic_set_range(src_offset, 16);
 492         src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 493         src_offset->num_components = 2;
 494         nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
 495         nir_builder_instr_insert(&b, &src_offset->instr);
 496
 497         nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 498         nir_intrinsic_set_base(dst_offset, 0);
 499         nir_intrinsic_set_range(dst_offset, 16);
 500         dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 8));
 501         dst_offset->num_components = 2;
 502         nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, 2, 32, "dst_offset");
 503         nir_builder_instr_insert(&b, &dst_offset->instr);
 504
 505         nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa);
 506
 507         nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
 508
 509         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 510         tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
 511         tex->op = nir_texop_txf;
 512         tex->src[0].src_type = nir_tex_src_coord;
 513         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, 3));
 514         tex->src[1].src_type = nir_tex_src_lod;
 515         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 516         tex->dest_type = nir_type_float;
 517         tex->is_array = false;
 518         tex->coord_components = 2;
 519         tex->texture = nir_deref_var_create(tex, input_img);
 520         tex->sampler = NULL;
 521
 522         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 523         nir_builder_instr_insert(&b, &tex->instr);
 524
 525         nir_ssa_def *outval = &tex->dest.ssa;
 526         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 527         store->src[0] = nir_src_for_ssa(dst_coord);
 528         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 529         store->src[2] = nir_src_for_ssa(outval);
 530         store->variables[0] = nir_deref_var_create(store, output_img);
 531
 532         nir_builder_instr_insert(&b, &store->instr);
 533         return b.shader;
 534 }
 535
 536 /* image to image - don't write use image accessors */
 537 static VkResult
 538 radv_device_init_meta_itoi_state(struct radv_device *device)
 539 {
 540         VkResult result;
 541         struct radv_shader_module cs = { .nir = NULL };
 542
 543         cs.nir = build_nir_itoi_compute_shader(device);
 544
 545         /*
 546          * two descriptors one for the image being sampled
 547          * one for the buffer being written.
 548          */
 549         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 550                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 551                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 552                 .bindingCount = 2,
 553                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 554                         {
 555                                 .binding = 0,
 556                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 557                                 .descriptorCount = 1,
 558                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 559                                 .pImmutableSamplers = NULL
 560                         },
 561                         {
 562                                 .binding = 1,
 563                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 564                                 .descriptorCount = 1,
 565                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 566                                 .pImmutableSamplers = NULL
 567                         },
 568                 }
 569         };
 570
 571         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 572                                                 &ds_create_info,
 573                                                 &device->meta_state.alloc,
 574                                                 &device->meta_state.itoi.img_ds_layout);
 575         if (result != VK_SUCCESS)
 576                 goto fail;
 577
 578
 579         VkPipelineLayoutCreateInfo pl_create_info = {
 580                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 581                 .setLayoutCount = 1,
 582                 .pSetLayouts = &device->meta_state.itoi.img_ds_layout,
 583                 .pushConstantRangeCount = 1,
 584                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 585         };
 586
 587         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 588                                           &pl_create_info,
 589                                           &device->meta_state.alloc,
 590                                           &device->meta_state.itoi.img_p_layout);
 591         if (result != VK_SUCCESS)
 592                 goto fail;
 593
 594         /* compute shader */
 595
 596         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 597                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 598                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 599                 .module = radv_shader_module_to_handle(&cs),
 600                 .pName = "main",
 601                 .pSpecializationInfo = NULL,
 602         };
 603
 604         VkComputePipelineCreateInfo vk_pipeline_info = {
 605                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 606                 .stage = pipeline_shader_stage,
 607                 .flags = 0,
 608                 .layout = device->meta_state.itoi.img_p_layout,
 609         };
 610
 611         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 612                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 613                                              1, &vk_pipeline_info, NULL,
 614                                              &device->meta_state.itoi.pipeline);
 615         if (result != VK_SUCCESS)
 616                 goto fail;
 617
 618         ralloc_free(cs.nir);
 619         return VK_SUCCESS;
 620 fail:
 621         ralloc_free(cs.nir);
 622         return result;
 623 }
 624
 625 static void
 626 radv_device_finish_meta_itoi_state(struct radv_device *device)
 627 {
 628         struct radv_meta_state *state = &device->meta_state;
 629
 630         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 631                                    state->itoi.img_p_layout, &state->alloc);
 632         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 633                                         state->itoi.img_ds_layout,
 634                                         &state->alloc);
 635         radv_DestroyPipeline(radv_device_to_handle(device),
 636                              state->itoi.pipeline, &state->alloc);
 637 }
 638
 639 static nir_shader *
 640 build_nir_cleari_compute_shader(struct radv_device *dev)
 641 {
 642         nir_builder b;
 643         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 644                                                              false,
 645                                                              false,
 646                                                              GLSL_TYPE_FLOAT);
 647         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 648         b.shader->info.name = ralloc_strdup(b.shader, "meta_cleari_cs");
 649         b.shader->info.cs.local_size[0] = 16;
 650         b.shader->info.cs.local_size[1] = 16;
 651         b.shader->info.cs.local_size[2] = 1;
 652
 653         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 654                                                        img_type, "out_img");
 655         output_img->data.descriptor_set = 0;
 656         output_img->data.binding = 0;
 657
 658         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 659         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 660         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 661                                                 b.shader->info.cs.local_size[0],
 662                                                 b.shader->info.cs.local_size[1],
 663                                                 b.shader->info.cs.local_size[2], 0);
 664
 665         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 666
 667         nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 668         nir_intrinsic_set_base(clear_val, 0);
 669         nir_intrinsic_set_range(clear_val, 16);
 670         clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 671         clear_val->num_components = 4;
 672         nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value");
 673         nir_builder_instr_insert(&b, &clear_val->instr);
 674
 675         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 676         store->src[0] = nir_src_for_ssa(global_id);
 677         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 678         store->src[2] = nir_src_for_ssa(&clear_val->dest.ssa);
 679         store->variables[0] = nir_deref_var_create(store, output_img);
 680
 681         nir_builder_instr_insert(&b, &store->instr);
 682         return b.shader;
 683 }
 684
 685 static VkResult
 686 radv_device_init_meta_cleari_state(struct radv_device *device)
 687 {
 688         VkResult result;
 689         struct radv_shader_module cs = { .nir = NULL };
 690
 691         cs.nir = build_nir_cleari_compute_shader(device);
 692
 693         /*
 694          * two descriptors one for the image being sampled
 695          * one for the buffer being written.
 696          */
 697         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 698                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 699                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 700                 .bindingCount = 1,
 701                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 702                         {
 703                                 .binding = 0,
 704                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 705                                 .descriptorCount = 1,
 706                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 707                                 .pImmutableSamplers = NULL
 708                         },
 709                 }
 710         };
 711
 712         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 713                                                 &ds_create_info,
 714                                                 &device->meta_state.alloc,
 715                                                 &device->meta_state.cleari.img_ds_layout);
 716         if (result != VK_SUCCESS)
 717                 goto fail;
 718
 719
 720         VkPipelineLayoutCreateInfo pl_create_info = {
 721                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 722                 .setLayoutCount = 1,
 723                 .pSetLayouts = &device->meta_state.cleari.img_ds_layout,
 724                 .pushConstantRangeCount = 1,
 725                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 726         };
 727
 728         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 729                                           &pl_create_info,
 730                                           &device->meta_state.alloc,
 731                                           &device->meta_state.cleari.img_p_layout);
 732         if (result != VK_SUCCESS)
 733                 goto fail;
 734
 735         /* compute shader */
 736
 737         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 738                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 739                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 740                 .module = radv_shader_module_to_handle(&cs),
 741                 .pName = "main",
 742                 .pSpecializationInfo = NULL,
 743         };
 744
 745         VkComputePipelineCreateInfo vk_pipeline_info = {
 746                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 747                 .stage = pipeline_shader_stage,
 748                 .flags = 0,
 749                 .layout = device->meta_state.cleari.img_p_layout,
 750         };
 751
 752         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 753                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 754                                              1, &vk_pipeline_info, NULL,
 755                                              &device->meta_state.cleari.pipeline);
 756         if (result != VK_SUCCESS)
 757                 goto fail;
 758
 759         ralloc_free(cs.nir);
 760         return VK_SUCCESS;
 761 fail:
 762         ralloc_free(cs.nir);
 763         return result;
 764 }
 765
 766 static void
 767 radv_device_finish_meta_cleari_state(struct radv_device *device)
 768 {
 769         struct radv_meta_state *state = &device->meta_state;
 770
 771         radv_DestroyPipelineLayout(radv_device_to_handle(device),
 772                                    state->cleari.img_p_layout, &state->alloc);
 773         radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 774                                         state->cleari.img_ds_layout,
 775                                         &state->alloc);
 776         radv_DestroyPipeline(radv_device_to_handle(device),
 777                              state->cleari.pipeline, &state->alloc);
 778 }
 779
 780 void
 781 radv_device_finish_meta_bufimage_state(struct radv_device *device)
 782 {
 783         radv_device_finish_meta_itob_state(device);
 784         radv_device_finish_meta_btoi_state(device);
 785         radv_device_finish_meta_itoi_state(device);
 786         radv_device_finish_meta_cleari_state(device);
 787 }
 788
 789 VkResult
 790 radv_device_init_meta_bufimage_state(struct radv_device *device)
 791 {
 792         VkResult result;
 793
 794         result = radv_device_init_meta_itob_state(device);
 795         if (result != VK_SUCCESS)
 796                 return result;
 797
 798         result = radv_device_init_meta_btoi_state(device);
 799         if (result != VK_SUCCESS)
 800                 goto fail_itob;
 801
 802         result = radv_device_init_meta_itoi_state(device);
 803         if (result != VK_SUCCESS)
 804                 goto fail_btoi;
 805
 806         result = radv_device_init_meta_cleari_state(device);
 807         if (result != VK_SUCCESS)
 808                 goto fail_itoi;
 809
 810         return VK_SUCCESS;
 811 fail_itoi:
 812         radv_device_finish_meta_itoi_state(device);
 813 fail_btoi:
 814         radv_device_finish_meta_btoi_state(device);
 815 fail_itob:
 816         radv_device_finish_meta_itob_state(device);
 817         return result;
 818 }
 819
 820 static void
 821 create_iview(struct radv_cmd_buffer *cmd_buffer,
 822              struct radv_meta_blit2d_surf *surf,
 823              struct radv_image_view *iview)
 824 {
 825         VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
 826                 radv_meta_get_view_type(surf->image);
 827         radv_image_view_init(iview, cmd_buffer->device,
 828                              &(VkImageViewCreateInfo) {
 829                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 830                                              .image = radv_image_to_handle(surf->image),
 831                                              .viewType = view_type,
 832                                              .format = surf->format,
 833                                              .subresourceRange = {
 834                                              .aspectMask = surf->aspect_mask,
 835                                              .baseMipLevel = surf->level,
 836                                              .levelCount = 1,
 837                                              .baseArrayLayer = surf->layer,
 838                                              .layerCount = 1
 839                                      },
 840                              });
 841 }
 842
 843 static void
 844 create_bview(struct radv_cmd_buffer *cmd_buffer,
 845              struct radv_buffer *buffer,
 846              unsigned offset,
 847              VkFormat format,
 848              struct radv_buffer_view *bview)
 849 {
 850         radv_buffer_view_init(bview, cmd_buffer->device,
 851                               &(VkBufferViewCreateInfo) {
 852                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
 853                                       .flags = 0,
 854                                       .buffer = radv_buffer_to_handle(buffer),
 855                                       .format = format,
 856                                       .offset = offset,
 857                                       .range = VK_WHOLE_SIZE,
 858                               });
 859
 860 }
 861
 862 static void
 863 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
 864                       struct radv_image_view *src,
 865                       struct radv_buffer_view *dst)
 866 {
 867         struct radv_device *device = cmd_buffer->device;
 868
 869         radv_meta_push_descriptor_set(cmd_buffer,
 870                                       VK_PIPELINE_BIND_POINT_COMPUTE,
 871                                       device->meta_state.itob.img_p_layout,
 872                                       0, /* set */
 873                                       2, /* descriptorWriteCount */
 874                                       (VkWriteDescriptorSet[]) {
 875                                               {
 876                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 877                                                       .dstBinding = 0,
 878                                                       .dstArrayElement = 0,
 879                                                       .descriptorCount = 1,
 880                                                       .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 881                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 882                                                               {
 883                                                                       .sampler = VK_NULL_HANDLE,
 884                                                                       .imageView = radv_image_view_to_handle(src),
 885                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 886                                                               },
 887                                                       }
 888                                               },
 889                                               {
 890                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 891                                                       .dstBinding = 1,
 892                                                       .dstArrayElement = 0,
 893                                                       .descriptorCount = 1,
 894                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 895                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(dst) },
 896                                               }
 897                                       });
 898 }
 899
 900 void
 901 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 902                           struct radv_meta_blit2d_surf *src,
 903                           struct radv_meta_blit2d_buffer *dst,
 904                           unsigned num_rects,
 905                           struct radv_meta_blit2d_rect *rects)
 906 {
 907         VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline;
 908         struct radv_device *device = cmd_buffer->device;
 909         struct radv_image_view src_view;
 910         struct radv_buffer_view dst_view;
 911
 912         create_iview(cmd_buffer, src, &src_view);
 913         create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &dst_view);
 914         itob_bind_descriptors(cmd_buffer, &src_view, &dst_view);
 915
 916         if (device->physical_device->rad_info.chip_class >= GFX9 &&
 917             src->image->type == VK_IMAGE_TYPE_3D)
 918                 pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d;
 919
 920         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 921                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
 922
 923         for (unsigned r = 0; r < num_rects; ++r) {
 924                 unsigned push_constants[4] = {
 925                         rects[r].src_x,
 926                         rects[r].src_y,
 927                         src->layer,
 928                         dst->pitch
 929                 };
 930                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 931                                       device->meta_state.itob.img_p_layout,
 932                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
 933                                       push_constants);
 934
 935                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
 936         }
 937 }
 938
 939 static void
 940 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
 941                       struct radv_buffer_view *src,
 942                       struct radv_image_view *dst)
 943 {
 944         struct radv_device *device = cmd_buffer->device;
 945
 946         radv_meta_push_descriptor_set(cmd_buffer,
 947                                       VK_PIPELINE_BIND_POINT_COMPUTE,
 948                                       device->meta_state.btoi.img_p_layout,
 949                                       0, /* set */
 950                                       2, /* descriptorWriteCount */
 951                                       (VkWriteDescriptorSet[]) {
 952                                               {
 953                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 954                                                       .dstBinding = 0,
 955                                                       .dstArrayElement = 0,
 956                                                       .descriptorCount = 1,
 957                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 958                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(src) },
 959                                               },
 960                                               {
 961                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 962                                                       .dstBinding = 1,
 963                                                       .dstArrayElement = 0,
 964                                                       .descriptorCount = 1,
 965                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 966                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 967                                                               {
 968                                                                       .sampler = VK_NULL_HANDLE,
 969                                                                       .imageView = radv_image_view_to_handle(dst),
 970                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 971                                                               },
 972                                                       }
 973                                               }
 974                                       });
 975 }
 976
 977 void
 978 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
 979                              struct radv_meta_blit2d_buffer *src,
 980                              struct radv_meta_blit2d_surf *dst,
 981                              unsigned num_rects,
 982                              struct radv_meta_blit2d_rect *rects)
 983 {
 984         VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline;
 985         struct radv_device *device = cmd_buffer->device;
 986         struct radv_buffer_view src_view;
 987         struct radv_image_view dst_view;
 988
 989         create_bview(cmd_buffer, src->buffer, src->offset, src->format, &src_view);
 990         create_iview(cmd_buffer, dst, &dst_view);
 991         btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
 992
 993         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 994                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
 995
 996         for (unsigned r = 0; r < num_rects; ++r) {
 997                 unsigned push_constants[3] = {
 998                         rects[r].dst_x,
 999                         rects[r].dst_y,
1000                         src->pitch
1001                 };
1002                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1003                                       device->meta_state.btoi.img_p_layout,
1004                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 12,
1005                                       push_constants);
1006
1007                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1008         }
1009 }
1010
1011 static void
1012 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1013                       struct radv_image_view *src,
1014                       struct radv_image_view *dst)
1015 {
1016         struct radv_device *device = cmd_buffer->device;
1017
1018         radv_meta_push_descriptor_set(cmd_buffer,
1019                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1020                                       device->meta_state.itoi.img_p_layout,
1021                                       0, /* set */
1022                                       2, /* descriptorWriteCount */
1023                                       (VkWriteDescriptorSet[]) {
1024                                               {
1025                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1026                                                        .dstBinding = 0,
1027                                                        .dstArrayElement = 0,
1028                                                        .descriptorCount = 1,
1029                                                        .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
1030                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1031                                                                {
1032                                                                        .sampler = VK_NULL_HANDLE,
1033                                                                        .imageView = radv_image_view_to_handle(src),
1034                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1035                                                                },
1036                                                        }
1037                                               },
1038                                               {
1039                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1040                                                        .dstBinding = 1,
1041                                                        .dstArrayElement = 0,
1042                                                        .descriptorCount = 1,
1043                                                        .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1044                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1045                                                                {
1046                                                                        .sampler = VK_NULL_HANDLE,
1047                                                                        .imageView = radv_image_view_to_handle(dst),
1048                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1049                                                                },
1050                                                        }
1051                                               }
1052                                       });
1053 }
1054
1055 void
1056 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1057                             struct radv_meta_blit2d_surf *src,
1058                             struct radv_meta_blit2d_surf *dst,
1059                             unsigned num_rects,
1060                             struct radv_meta_blit2d_rect *rects)
1061 {
1062         VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline;
1063         struct radv_device *device = cmd_buffer->device;
1064         struct radv_image_view src_view, dst_view;
1065
1066         create_iview(cmd_buffer, src, &src_view);
1067         create_iview(cmd_buffer, dst, &dst_view);
1068
1069         itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1070
1071         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1072                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1073
1074         for (unsigned r = 0; r < num_rects; ++r) {
1075                 unsigned push_constants[4] = {
1076                         rects[r].src_x,
1077                         rects[r].src_y,
1078                         rects[r].dst_x,
1079                         rects[r].dst_y,
1080                 };
1081                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1082                                       device->meta_state.itoi.img_p_layout,
1083                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1084                                       push_constants);
1085
1086                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1087         }
1088 }
1089
1090 static void
1091 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1092                         struct radv_image_view *dst_iview)
1093 {
1094         struct radv_device *device = cmd_buffer->device;
1095
1096         radv_meta_push_descriptor_set(cmd_buffer,
1097                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1098                                       device->meta_state.cleari.img_p_layout,
1099                                       0, /* set */
1100                                       1, /* descriptorWriteCount */
1101                                       (VkWriteDescriptorSet[]) {
1102                                               {
1103                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1104                                                       .dstBinding = 0,
1105                                                       .dstArrayElement = 0,
1106                                                       .descriptorCount = 1,
1107                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1108                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1109                                                                {
1110                                                                       .sampler = VK_NULL_HANDLE,
1111                                                                       .imageView = radv_image_view_to_handle(dst_iview),
1112                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1113                                                                },
1114                                                       }
1115                                                },
1116                                       });
1117 }
1118
1119 void
1120 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer,
1121                          struct radv_meta_blit2d_surf *dst,
1122                          const VkClearColorValue *clear_color)
1123 {
1124         VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline;
1125         struct radv_device *device = cmd_buffer->device;
1126         struct radv_image_view dst_iview;
1127
1128         create_iview(cmd_buffer, dst, &dst_iview);
1129         cleari_bind_descriptors(cmd_buffer, &dst_iview);
1130
1131         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1132                              VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1133
1134         unsigned push_constants[4] = {
1135                 clear_color->uint32[0],
1136                 clear_color->uint32[1],
1137                 clear_color->uint32[2],
1138                 clear_color->uint32[3],
1139         };
1140
1141         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1142                               device->meta_state.cleari.img_p_layout,
1143                               VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1144                               push_constants);
1145
1146         radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1);
1147 }