src/amd/vulkan/radv_meta_bufimage.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  */
  24 #include "radv_meta.h"
  25 #include "nir/nir_builder.h"
  26
  27 /*
  28  * GFX queue: Compute shader implementation of image->buffer copy
  29  * Compute queue: implementation also of buffer->image, image->image, and image clear.
  30  */
  31
  32 static nir_shader *
  33 build_nir_itob_compute_shader(struct radv_device *dev)
  34 {
  35         nir_builder b;
  36         const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
  37                                                                  false,
  38                                                                  false,
  39                                                                  GLSL_TYPE_FLOAT);
  40         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
  41                                                              false,
  42                                                              false,
  43                                                              GLSL_TYPE_FLOAT);
  44         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
  45         b.shader->info.name = ralloc_strdup(b.shader, "meta_itob_cs");
  46         b.shader->info.cs.local_size[0] = 16;
  47         b.shader->info.cs.local_size[1] = 16;
  48         b.shader->info.cs.local_size[2] = 1;
  49         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
  50                                                       sampler_type, "s_tex");
  51         input_img->data.descriptor_set = 0;
  52         input_img->data.binding = 0;
  53
  54         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
  55                                                        img_type, "out_img");
  56         output_img->data.descriptor_set = 0;
  57         output_img->data.binding = 1;
  58
  59         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
  60         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
  61         nir_ssa_def *block_size = nir_imm_ivec4(&b,
  62                                                 b.shader->info.cs.local_size[0],
  63                                                 b.shader->info.cs.local_size[1],
  64                                                 b.shader->info.cs.local_size[2], 0);
  65
  66         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
  67
  68
  69
  70         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  71         nir_intrinsic_set_base(offset, 0);
  72         nir_intrinsic_set_range(offset, 12);
  73         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
  74         offset->num_components = 2;
  75         nir_ssa_dest_init(&offset->instr, &offset->dest, 2, 32, "offset");
  76         nir_builder_instr_insert(&b, &offset->instr);
  77
  78         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
  79         nir_intrinsic_set_base(stride, 0);
  80         nir_intrinsic_set_range(stride, 12);
  81         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 8));
  82         stride->num_components = 1;
  83         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
  84         nir_builder_instr_insert(&b, &stride->instr);
  85
  86         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
  87
  88         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
  89         tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
  90         tex->op = nir_texop_txf;
  91         tex->src[0].src_type = nir_tex_src_coord;
  92         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, 0x3));
  93         tex->src[1].src_type = nir_tex_src_lod;
  94         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
  95         tex->dest_type = nir_type_float;
  96         tex->is_array = false;
  97         tex->coord_components = 2;
  98         tex->texture = nir_deref_var_create(tex, input_img);
  99         tex->sampler = NULL;
 100
 101         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 102         nir_builder_instr_insert(&b, &tex->instr);
 103
 104         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 105         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 106
 107         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 108         tmp = nir_iadd(&b, tmp, pos_x);
 109
 110         nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 111
 112         nir_ssa_def *outval = &tex->dest.ssa;
 113         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 114         store->src[0] = nir_src_for_ssa(coord);
 115         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 116         store->src[2] = nir_src_for_ssa(outval);
 117         store->variables[0] = nir_deref_var_create(store, output_img);
 118
 119         nir_builder_instr_insert(&b, &store->instr);
 120         return b.shader;
 121 }
 122
 123 /* Image to buffer - don't write use image accessors */
 124 static VkResult
 125 radv_device_init_meta_itob_state(struct radv_device *device)
 126 {
 127         VkResult result;
 128         struct radv_shader_module cs = { .nir = NULL };
 129
 130         zero(device->meta_state.itob);
 131
 132         cs.nir = build_nir_itob_compute_shader(device);
 133
 134         /*
 135          * two descriptors one for the image being sampled
 136          * one for the buffer being written.
 137          */
 138         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 139                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 140                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 141                 .bindingCount = 2,
 142                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 143                         {
 144                                 .binding = 0,
 145                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 146                                 .descriptorCount = 1,
 147                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 148                                 .pImmutableSamplers = NULL
 149                         },
 150                         {
 151                                 .binding = 1,
 152                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 153                                 .descriptorCount = 1,
 154                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 155                                 .pImmutableSamplers = NULL
 156                         },
 157                 }
 158         };
 159
 160         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 161                                                 &ds_create_info,
 162                                                 &device->meta_state.alloc,
 163                                                 &device->meta_state.itob.img_ds_layout);
 164         if (result != VK_SUCCESS)
 165                 goto fail;
 166
 167
 168         VkPipelineLayoutCreateInfo pl_create_info = {
 169                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 170                 .setLayoutCount = 1,
 171                 .pSetLayouts = &device->meta_state.itob.img_ds_layout,
 172                 .pushConstantRangeCount = 1,
 173                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 12},
 174         };
 175
 176         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 177                                           &pl_create_info,
 178                                           &device->meta_state.alloc,
 179                                           &device->meta_state.itob.img_p_layout);
 180         if (result != VK_SUCCESS)
 181                 goto fail;
 182
 183         /* compute shader */
 184
 185         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 186                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 187                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 188                 .module = radv_shader_module_to_handle(&cs),
 189                 .pName = "main",
 190                 .pSpecializationInfo = NULL,
 191         };
 192
 193         VkComputePipelineCreateInfo vk_pipeline_info = {
 194                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 195                 .stage = pipeline_shader_stage,
 196                 .flags = 0,
 197                 .layout = device->meta_state.itob.img_p_layout,
 198         };
 199
 200         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 201                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 202                                              1, &vk_pipeline_info, NULL,
 203                                              &device->meta_state.itob.pipeline);
 204         if (result != VK_SUCCESS)
 205                 goto fail;
 206
 207         ralloc_free(cs.nir);
 208         return VK_SUCCESS;
 209 fail:
 210         ralloc_free(cs.nir);
 211         return result;
 212 }
 213
 214 static void
 215 radv_device_finish_meta_itob_state(struct radv_device *device)
 216 {
 217         if (device->meta_state.itob.img_p_layout) {
 218                 radv_DestroyPipelineLayout(radv_device_to_handle(device),
 219                                            device->meta_state.itob.img_p_layout,
 220                                            &device->meta_state.alloc);
 221         }
 222         if (device->meta_state.itob.img_ds_layout) {
 223                 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 224                                                 device->meta_state.itob.img_ds_layout,
 225                                                 &device->meta_state.alloc);
 226         }
 227         if (device->meta_state.itob.pipeline) {
 228                 radv_DestroyPipeline(radv_device_to_handle(device),
 229                                      device->meta_state.itob.pipeline,
 230                                      &device->meta_state.alloc);
 231         }
 232 }
 233
 234 static nir_shader *
 235 build_nir_btoi_compute_shader(struct radv_device *dev)
 236 {
 237         nir_builder b;
 238         const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
 239                                                              false,
 240                                                              false,
 241                                                              GLSL_TYPE_FLOAT);
 242         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 243                                                              false,
 244                                                              false,
 245                                                              GLSL_TYPE_FLOAT);
 246         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 247         b.shader->info.name = ralloc_strdup(b.shader, "meta_btoi_cs");
 248         b.shader->info.cs.local_size[0] = 16;
 249         b.shader->info.cs.local_size[1] = 16;
 250         b.shader->info.cs.local_size[2] = 1;
 251         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 252                                                       buf_type, "s_tex");
 253         input_img->data.descriptor_set = 0;
 254         input_img->data.binding = 0;
 255
 256         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 257                                                        img_type, "out_img");
 258         output_img->data.descriptor_set = 0;
 259         output_img->data.binding = 1;
 260
 261         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 262         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 263         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 264                                                 b.shader->info.cs.local_size[0],
 265                                                 b.shader->info.cs.local_size[1],
 266                                                 b.shader->info.cs.local_size[2], 0);
 267
 268         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 269
 270         nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 271         nir_intrinsic_set_base(offset, 0);
 272         nir_intrinsic_set_range(offset, 12);
 273         offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 274         offset->num_components = 2;
 275         nir_ssa_dest_init(&offset->instr, &offset->dest, 2, 32, "offset");
 276         nir_builder_instr_insert(&b, &offset->instr);
 277
 278         nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 279         nir_intrinsic_set_base(stride, 0);
 280         nir_intrinsic_set_range(stride, 12);
 281         stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 8));
 282         stride->num_components = 1;
 283         nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
 284         nir_builder_instr_insert(&b, &stride->instr);
 285
 286         nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
 287         nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
 288
 289         nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
 290         tmp = nir_iadd(&b, tmp, pos_x);
 291
 292         nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
 293
 294         nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
 295
 296         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 297         tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
 298         tex->op = nir_texop_txf;
 299         tex->src[0].src_type = nir_tex_src_coord;
 300         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1));
 301         tex->src[1].src_type = nir_tex_src_lod;
 302         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 303         tex->dest_type = nir_type_float;
 304         tex->is_array = false;
 305         tex->coord_components = 1;
 306         tex->texture = nir_deref_var_create(tex, input_img);
 307         tex->sampler = NULL;
 308
 309         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 310         nir_builder_instr_insert(&b, &tex->instr);
 311
 312         nir_ssa_def *outval = &tex->dest.ssa;
 313         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 314         store->src[0] = nir_src_for_ssa(img_coord);
 315         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 316         store->src[2] = nir_src_for_ssa(outval);
 317         store->variables[0] = nir_deref_var_create(store, output_img);
 318
 319         nir_builder_instr_insert(&b, &store->instr);
 320         return b.shader;
 321 }
 322
 323 /* Buffer to image - don't write use image accessors */
 324 static VkResult
 325 radv_device_init_meta_btoi_state(struct radv_device *device)
 326 {
 327         VkResult result;
 328         struct radv_shader_module cs = { .nir = NULL };
 329
 330         zero(device->meta_state.btoi);
 331
 332         cs.nir = build_nir_btoi_compute_shader(device);
 333
 334         /*
 335          * two descriptors one for the image being sampled
 336          * one for the buffer being written.
 337          */
 338         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 339                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 340                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 341                 .bindingCount = 2,
 342                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 343                         {
 344                                 .binding = 0,
 345                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 346                                 .descriptorCount = 1,
 347                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 348                                 .pImmutableSamplers = NULL
 349                         },
 350                         {
 351                                 .binding = 1,
 352                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 353                                 .descriptorCount = 1,
 354                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 355                                 .pImmutableSamplers = NULL
 356                         },
 357                 }
 358         };
 359
 360         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 361                                                 &ds_create_info,
 362                                                 &device->meta_state.alloc,
 363                                                 &device->meta_state.btoi.img_ds_layout);
 364         if (result != VK_SUCCESS)
 365                 goto fail;
 366
 367
 368         VkPipelineLayoutCreateInfo pl_create_info = {
 369                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 370                 .setLayoutCount = 1,
 371                 .pSetLayouts = &device->meta_state.btoi.img_ds_layout,
 372                 .pushConstantRangeCount = 1,
 373                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 12},
 374         };
 375
 376         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 377                                           &pl_create_info,
 378                                           &device->meta_state.alloc,
 379                                           &device->meta_state.btoi.img_p_layout);
 380         if (result != VK_SUCCESS)
 381                 goto fail;
 382
 383         /* compute shader */
 384
 385         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 386                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 387                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 388                 .module = radv_shader_module_to_handle(&cs),
 389                 .pName = "main",
 390                 .pSpecializationInfo = NULL,
 391         };
 392
 393         VkComputePipelineCreateInfo vk_pipeline_info = {
 394                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 395                 .stage = pipeline_shader_stage,
 396                 .flags = 0,
 397                 .layout = device->meta_state.btoi.img_p_layout,
 398         };
 399
 400         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 401                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 402                                              1, &vk_pipeline_info, NULL,
 403                                              &device->meta_state.btoi.pipeline);
 404         if (result != VK_SUCCESS)
 405                 goto fail;
 406
 407         ralloc_free(cs.nir);
 408         return VK_SUCCESS;
 409 fail:
 410         ralloc_free(cs.nir);
 411         return result;
 412 }
 413
 414 static void
 415 radv_device_finish_meta_btoi_state(struct radv_device *device)
 416 {
 417         if (device->meta_state.btoi.img_p_layout) {
 418                 radv_DestroyPipelineLayout(radv_device_to_handle(device),
 419                                            device->meta_state.btoi.img_p_layout,
 420                                            &device->meta_state.alloc);
 421         }
 422         if (device->meta_state.btoi.img_ds_layout) {
 423                 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 424                                                 device->meta_state.btoi.img_ds_layout,
 425                                                 &device->meta_state.alloc);
 426         }
 427         if (device->meta_state.btoi.pipeline) {
 428                 radv_DestroyPipeline(radv_device_to_handle(device),
 429                                      device->meta_state.btoi.pipeline,
 430                                      &device->meta_state.alloc);
 431         }
 432 }
 433
 434 static nir_shader *
 435 build_nir_itoi_compute_shader(struct radv_device *dev)
 436 {
 437         nir_builder b;
 438         const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 439                                                              false,
 440                                                              false,
 441                                                              GLSL_TYPE_FLOAT);
 442         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 443                                                              false,
 444                                                              false,
 445                                                              GLSL_TYPE_FLOAT);
 446         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 447         b.shader->info.name = ralloc_strdup(b.shader, "meta_itoi_cs");
 448         b.shader->info.cs.local_size[0] = 16;
 449         b.shader->info.cs.local_size[1] = 16;
 450         b.shader->info.cs.local_size[2] = 1;
 451         nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
 452                                                       buf_type, "s_tex");
 453         input_img->data.descriptor_set = 0;
 454         input_img->data.binding = 0;
 455
 456         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 457                                                        img_type, "out_img");
 458         output_img->data.descriptor_set = 0;
 459         output_img->data.binding = 1;
 460
 461         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 462         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 463         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 464                                                 b.shader->info.cs.local_size[0],
 465                                                 b.shader->info.cs.local_size[1],
 466                                                 b.shader->info.cs.local_size[2], 0);
 467
 468         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 469
 470         nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 471         nir_intrinsic_set_base(src_offset, 0);
 472         nir_intrinsic_set_range(src_offset, 16);
 473         src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 474         src_offset->num_components = 2;
 475         nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
 476         nir_builder_instr_insert(&b, &src_offset->instr);
 477
 478         nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 479         nir_intrinsic_set_base(dst_offset, 0);
 480         nir_intrinsic_set_range(dst_offset, 16);
 481         dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 8));
 482         dst_offset->num_components = 2;
 483         nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, 2, 32, "dst_offset");
 484         nir_builder_instr_insert(&b, &dst_offset->instr);
 485
 486         nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa);
 487
 488         nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
 489
 490         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
 491         tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
 492         tex->op = nir_texop_txf;
 493         tex->src[0].src_type = nir_tex_src_coord;
 494         tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, 3));
 495         tex->src[1].src_type = nir_tex_src_lod;
 496         tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
 497         tex->dest_type = nir_type_float;
 498         tex->is_array = false;
 499         tex->coord_components = 2;
 500         tex->texture = nir_deref_var_create(tex, input_img);
 501         tex->sampler = NULL;
 502
 503         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 504         nir_builder_instr_insert(&b, &tex->instr);
 505
 506         nir_ssa_def *outval = &tex->dest.ssa;
 507         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 508         store->src[0] = nir_src_for_ssa(dst_coord);
 509         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 510         store->src[2] = nir_src_for_ssa(outval);
 511         store->variables[0] = nir_deref_var_create(store, output_img);
 512
 513         nir_builder_instr_insert(&b, &store->instr);
 514         return b.shader;
 515 }
 516
 517 /* image to image - don't write use image accessors */
 518 static VkResult
 519 radv_device_init_meta_itoi_state(struct radv_device *device)
 520 {
 521         VkResult result;
 522         struct radv_shader_module cs = { .nir = NULL };
 523
 524         zero(device->meta_state.itoi);
 525
 526         cs.nir = build_nir_itoi_compute_shader(device);
 527
 528         /*
 529          * two descriptors one for the image being sampled
 530          * one for the buffer being written.
 531          */
 532         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 533                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 534                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 535                 .bindingCount = 2,
 536                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 537                         {
 538                                 .binding = 0,
 539                                 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 540                                 .descriptorCount = 1,
 541                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 542                                 .pImmutableSamplers = NULL
 543                         },
 544                         {
 545                                 .binding = 1,
 546                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 547                                 .descriptorCount = 1,
 548                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 549                                 .pImmutableSamplers = NULL
 550                         },
 551                 }
 552         };
 553
 554         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 555                                                 &ds_create_info,
 556                                                 &device->meta_state.alloc,
 557                                                 &device->meta_state.itoi.img_ds_layout);
 558         if (result != VK_SUCCESS)
 559                 goto fail;
 560
 561
 562         VkPipelineLayoutCreateInfo pl_create_info = {
 563                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 564                 .setLayoutCount = 1,
 565                 .pSetLayouts = &device->meta_state.itoi.img_ds_layout,
 566                 .pushConstantRangeCount = 1,
 567                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 568         };
 569
 570         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 571                                           &pl_create_info,
 572                                           &device->meta_state.alloc,
 573                                           &device->meta_state.itoi.img_p_layout);
 574         if (result != VK_SUCCESS)
 575                 goto fail;
 576
 577         /* compute shader */
 578
 579         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 580                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 581                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 582                 .module = radv_shader_module_to_handle(&cs),
 583                 .pName = "main",
 584                 .pSpecializationInfo = NULL,
 585         };
 586
 587         VkComputePipelineCreateInfo vk_pipeline_info = {
 588                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 589                 .stage = pipeline_shader_stage,
 590                 .flags = 0,
 591                 .layout = device->meta_state.itoi.img_p_layout,
 592         };
 593
 594         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 595                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 596                                              1, &vk_pipeline_info, NULL,
 597                                              &device->meta_state.itoi.pipeline);
 598         if (result != VK_SUCCESS)
 599                 goto fail;
 600
 601         ralloc_free(cs.nir);
 602         return VK_SUCCESS;
 603 fail:
 604         ralloc_free(cs.nir);
 605         return result;
 606 }
 607
 608 static void
 609 radv_device_finish_meta_itoi_state(struct radv_device *device)
 610 {
 611         if (device->meta_state.itoi.img_p_layout) {
 612                 radv_DestroyPipelineLayout(radv_device_to_handle(device),
 613                                            device->meta_state.itoi.img_p_layout,
 614                                            &device->meta_state.alloc);
 615         }
 616         if (device->meta_state.itoi.img_ds_layout) {
 617                 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 618                                                 device->meta_state.itoi.img_ds_layout,
 619                                                 &device->meta_state.alloc);
 620         }
 621         if (device->meta_state.itoi.pipeline) {
 622                 radv_DestroyPipeline(radv_device_to_handle(device),
 623                                      device->meta_state.itoi.pipeline,
 624                                      &device->meta_state.alloc);
 625         }
 626 }
 627
 628 static nir_shader *
 629 build_nir_cleari_compute_shader(struct radv_device *dev)
 630 {
 631         nir_builder b;
 632         const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_2D,
 633                                                              false,
 634                                                              false,
 635                                                              GLSL_TYPE_FLOAT);
 636         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
 637         b.shader->info.name = ralloc_strdup(b.shader, "meta_cleari_cs");
 638         b.shader->info.cs.local_size[0] = 16;
 639         b.shader->info.cs.local_size[1] = 16;
 640         b.shader->info.cs.local_size[2] = 1;
 641
 642         nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
 643                                                        img_type, "out_img");
 644         output_img->data.descriptor_set = 0;
 645         output_img->data.binding = 0;
 646
 647         nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
 648         nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
 649         nir_ssa_def *block_size = nir_imm_ivec4(&b,
 650                                                 b.shader->info.cs.local_size[0],
 651                                                 b.shader->info.cs.local_size[1],
 652                                                 b.shader->info.cs.local_size[2], 0);
 653
 654         nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
 655
 656         nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
 657         nir_intrinsic_set_base(clear_val, 0);
 658         nir_intrinsic_set_range(clear_val, 16);
 659         clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
 660         clear_val->num_components = 4;
 661         nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value");
 662         nir_builder_instr_insert(&b, &clear_val->instr);
 663
 664         nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store);
 665         store->src[0] = nir_src_for_ssa(global_id);
 666         store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
 667         store->src[2] = nir_src_for_ssa(&clear_val->dest.ssa);
 668         store->variables[0] = nir_deref_var_create(store, output_img);
 669
 670         nir_builder_instr_insert(&b, &store->instr);
 671         return b.shader;
 672 }
 673
 674 static VkResult
 675 radv_device_init_meta_cleari_state(struct radv_device *device)
 676 {
 677         VkResult result;
 678         struct radv_shader_module cs = { .nir = NULL };
 679
 680         zero(device->meta_state.cleari);
 681
 682         cs.nir = build_nir_cleari_compute_shader(device);
 683
 684         /*
 685          * two descriptors one for the image being sampled
 686          * one for the buffer being written.
 687          */
 688         VkDescriptorSetLayoutCreateInfo ds_create_info = {
 689                 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 690                 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
 691                 .bindingCount = 1,
 692                 .pBindings = (VkDescriptorSetLayoutBinding[]) {
 693                         {
 694                                 .binding = 0,
 695                                 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 696                                 .descriptorCount = 1,
 697                                 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
 698                                 .pImmutableSamplers = NULL
 699                         },
 700                 }
 701         };
 702
 703         result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
 704                                                 &ds_create_info,
 705                                                 &device->meta_state.alloc,
 706                                                 &device->meta_state.cleari.img_ds_layout);
 707         if (result != VK_SUCCESS)
 708                 goto fail;
 709
 710
 711         VkPipelineLayoutCreateInfo pl_create_info = {
 712                 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 713                 .setLayoutCount = 1,
 714                 .pSetLayouts = &device->meta_state.cleari.img_ds_layout,
 715                 .pushConstantRangeCount = 1,
 716                 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
 717         };
 718
 719         result = radv_CreatePipelineLayout(radv_device_to_handle(device),
 720                                           &pl_create_info,
 721                                           &device->meta_state.alloc,
 722                                           &device->meta_state.cleari.img_p_layout);
 723         if (result != VK_SUCCESS)
 724                 goto fail;
 725
 726         /* compute shader */
 727
 728         VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
 729                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 730                 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
 731                 .module = radv_shader_module_to_handle(&cs),
 732                 .pName = "main",
 733                 .pSpecializationInfo = NULL,
 734         };
 735
 736         VkComputePipelineCreateInfo vk_pipeline_info = {
 737                 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
 738                 .stage = pipeline_shader_stage,
 739                 .flags = 0,
 740                 .layout = device->meta_state.cleari.img_p_layout,
 741         };
 742
 743         result = radv_CreateComputePipelines(radv_device_to_handle(device),
 744                                              radv_pipeline_cache_to_handle(&device->meta_state.cache),
 745                                              1, &vk_pipeline_info, NULL,
 746                                              &device->meta_state.cleari.pipeline);
 747         if (result != VK_SUCCESS)
 748                 goto fail;
 749
 750         ralloc_free(cs.nir);
 751         return VK_SUCCESS;
 752 fail:
 753         ralloc_free(cs.nir);
 754         return result;
 755 }
 756
 757 static void
 758 radv_device_finish_meta_cleari_state(struct radv_device *device)
 759 {
 760         if (device->meta_state.cleari.img_p_layout) {
 761                 radv_DestroyPipelineLayout(radv_device_to_handle(device),
 762                                            device->meta_state.cleari.img_p_layout,
 763                                            &device->meta_state.alloc);
 764         }
 765         if (device->meta_state.cleari.img_ds_layout) {
 766                 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
 767                                                 device->meta_state.cleari.img_ds_layout,
 768                                                 &device->meta_state.alloc);
 769         }
 770         if (device->meta_state.cleari.pipeline) {
 771                 radv_DestroyPipeline(radv_device_to_handle(device),
 772                                      device->meta_state.cleari.pipeline,
 773                                      &device->meta_state.alloc);
 774         }
 775 }
 776
 777 void
 778 radv_device_finish_meta_bufimage_state(struct radv_device *device)
 779 {
 780         radv_device_finish_meta_itob_state(device);
 781         radv_device_finish_meta_btoi_state(device);
 782         radv_device_finish_meta_itoi_state(device);
 783         radv_device_finish_meta_cleari_state(device);
 784 }
 785
 786 VkResult
 787 radv_device_init_meta_bufimage_state(struct radv_device *device)
 788 {
 789         VkResult result;
 790
 791         result = radv_device_init_meta_itob_state(device);
 792         if (result != VK_SUCCESS)
 793                 return result;
 794
 795         result = radv_device_init_meta_btoi_state(device);
 796         if (result != VK_SUCCESS)
 797                 goto fail_itob;
 798
 799         result = radv_device_init_meta_itoi_state(device);
 800         if (result != VK_SUCCESS)
 801                 goto fail_btoi;
 802
 803         result = radv_device_init_meta_cleari_state(device);
 804         if (result != VK_SUCCESS)
 805                 goto fail_itoi;
 806
 807         return VK_SUCCESS;
 808 fail_itoi:
 809         radv_device_finish_meta_itoi_state(device);
 810 fail_btoi:
 811         radv_device_finish_meta_btoi_state(device);
 812 fail_itob:
 813         radv_device_finish_meta_itob_state(device);
 814         return result;
 815 }
 816
 817 void
 818 radv_meta_begin_itoi(struct radv_cmd_buffer *cmd_buffer,
 819                      struct radv_meta_saved_compute_state *save)
 820 {
 821         radv_meta_save_compute(save, cmd_buffer, 16);
 822 }
 823
 824 void
 825 radv_meta_end_itoi(struct radv_cmd_buffer *cmd_buffer,
 826                    struct radv_meta_saved_compute_state *save)
 827 {
 828         radv_meta_restore_compute(save, cmd_buffer, 16);
 829 }
 830
 831 void
 832 radv_meta_begin_bufimage(struct radv_cmd_buffer *cmd_buffer,
 833                          struct radv_meta_saved_compute_state *save)
 834 {
 835         radv_meta_save_compute(save, cmd_buffer, 12);
 836 }
 837
 838 void
 839 radv_meta_end_bufimage(struct radv_cmd_buffer *cmd_buffer,
 840                        struct radv_meta_saved_compute_state *save)
 841 {
 842         radv_meta_restore_compute(save, cmd_buffer, 12);
 843 }
 844
 845 void
 846 radv_meta_begin_cleari(struct radv_cmd_buffer *cmd_buffer,
 847                        struct radv_meta_saved_compute_state *save)
 848 {
 849         radv_meta_save_compute(save, cmd_buffer, 16);
 850 }
 851
 852 void
 853 radv_meta_end_cleari(struct radv_cmd_buffer *cmd_buffer,
 854                      struct radv_meta_saved_compute_state *save)
 855 {
 856         radv_meta_restore_compute(save, cmd_buffer, 16);
 857 }
 858
 859 static void
 860 create_iview(struct radv_cmd_buffer *cmd_buffer,
 861              struct radv_meta_blit2d_surf *surf,
 862              struct radv_image_view *iview)
 863 {
 864
 865         radv_image_view_init(iview, cmd_buffer->device,
 866                              &(VkImageViewCreateInfo) {
 867                                      .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 868                                              .image = radv_image_to_handle(surf->image),
 869                                              .viewType = VK_IMAGE_VIEW_TYPE_2D,
 870                                              .format = surf->format,
 871                                              .subresourceRange = {
 872                                              .aspectMask = surf->aspect_mask,
 873                                              .baseMipLevel = surf->level,
 874                                              .levelCount = 1,
 875                                              .baseArrayLayer = surf->layer,
 876                                              .layerCount = 1
 877                                      },
 878                              });
 879 }
 880
 881 static void
 882 create_bview(struct radv_cmd_buffer *cmd_buffer,
 883              struct radv_buffer *buffer,
 884              unsigned offset,
 885              VkFormat format,
 886              struct radv_buffer_view *bview)
 887 {
 888         radv_buffer_view_init(bview, cmd_buffer->device,
 889                               &(VkBufferViewCreateInfo) {
 890                                       .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
 891                                       .flags = 0,
 892                                       .buffer = radv_buffer_to_handle(buffer),
 893                                       .format = format,
 894                                       .offset = offset,
 895                                       .range = VK_WHOLE_SIZE,
 896                               }, cmd_buffer);
 897
 898 }
 899
 900 struct itob_temps {
 901         struct radv_image_view src_iview;
 902         struct radv_buffer_view dst_bview;
 903 };
 904
 905 static void
 906 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
 907                       struct itob_temps *tmp)
 908 {
 909         struct radv_device *device = cmd_buffer->device;
 910
 911         radv_meta_push_descriptor_set(cmd_buffer,
 912                                       VK_PIPELINE_BIND_POINT_COMPUTE,
 913                                       device->meta_state.itob.img_p_layout,
 914                                       0, /* set */
 915                                       2, /* descriptorWriteCount */
 916                                       (VkWriteDescriptorSet[]) {
 917                                               {
 918                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 919                                                       .dstBinding = 0,
 920                                                       .dstArrayElement = 0,
 921                                                       .descriptorCount = 1,
 922                                                       .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
 923                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
 924                                                               {
 925                                                                       .sampler = VK_NULL_HANDLE,
 926                                                                       .imageView = radv_image_view_to_handle(&tmp->src_iview),
 927                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 928                                                               },
 929                                                       }
 930                                               },
 931                                               {
 932                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 933                                                       .dstBinding = 1,
 934                                                       .dstArrayElement = 0,
 935                                                       .descriptorCount = 1,
 936                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
 937                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(&tmp->dst_bview) },
 938                                               }
 939                                       });
 940 }
 941
 942 static void
 943 itob_bind_pipeline(struct radv_cmd_buffer *cmd_buffer)
 944 {
 945         VkPipeline pipeline =
 946                 cmd_buffer->device->meta_state.itob.pipeline;
 947
 948         if (cmd_buffer->state.compute_pipeline != radv_pipeline_from_handle(pipeline)) {
 949                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 950                                      VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
 951         }
 952 }
 953
 954 void
 955 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
 956                           struct radv_meta_blit2d_surf *src,
 957                           struct radv_meta_blit2d_buffer *dst,
 958                           unsigned num_rects,
 959                           struct radv_meta_blit2d_rect *rects)
 960 {
 961         struct radv_device *device = cmd_buffer->device;
 962         struct itob_temps temps;
 963
 964         create_iview(cmd_buffer, src, &temps.src_iview);
 965         create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &temps.dst_bview);
 966         itob_bind_descriptors(cmd_buffer, &temps);
 967
 968         itob_bind_pipeline(cmd_buffer);
 969
 970         for (unsigned r = 0; r < num_rects; ++r) {
 971                 unsigned push_constants[3] = {
 972                         rects[r].src_x,
 973                         rects[r].src_y,
 974                         dst->pitch
 975                 };
 976                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 977                                       device->meta_state.itob.img_p_layout,
 978                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 12,
 979                                       push_constants);
 980
 981                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
 982         }
 983 }
 984
 985 struct btoi_temps {
 986         struct radv_buffer_view src_bview;
 987         struct radv_image_view dst_iview;
 988 };
 989
 990 static void
 991 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
 992                       struct btoi_temps *tmp)
 993 {
 994         struct radv_device *device = cmd_buffer->device;
 995
 996         radv_meta_push_descriptor_set(cmd_buffer,
 997                                       VK_PIPELINE_BIND_POINT_COMPUTE,
 998                                       device->meta_state.btoi.img_p_layout,
 999                                       0, /* set */
1000                                       2, /* descriptorWriteCount */
1001                                       (VkWriteDescriptorSet[]) {
1002                                               {
1003                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1004                                                       .dstBinding = 0,
1005                                                       .dstArrayElement = 0,
1006                                                       .descriptorCount = 1,
1007                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1008                                                       .pTexelBufferView = (VkBufferView[])  { radv_buffer_view_to_handle(&tmp->src_bview) },
1009                                               },
1010                                               {
1011                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1012                                                       .dstBinding = 1,
1013                                                       .dstArrayElement = 0,
1014                                                       .descriptorCount = 1,
1015                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1016                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1017                                                               {
1018                                                                       .sampler = VK_NULL_HANDLE,
1019                                                                       .imageView = radv_image_view_to_handle(&tmp->dst_iview),
1020                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1021                                                               },
1022                                                       }
1023                                               }
1024                                       });
1025 }
1026
1027 static void
1028 btoi_bind_pipeline(struct radv_cmd_buffer *cmd_buffer)
1029 {
1030         VkPipeline pipeline =
1031                 cmd_buffer->device->meta_state.btoi.pipeline;
1032
1033         if (cmd_buffer->state.compute_pipeline != radv_pipeline_from_handle(pipeline)) {
1034                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1035                                      VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1036         }
1037 }
1038
1039 void
1040 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1041                              struct radv_meta_blit2d_buffer *src,
1042                              struct radv_meta_blit2d_surf *dst,
1043                              unsigned num_rects,
1044                              struct radv_meta_blit2d_rect *rects)
1045 {
1046         struct radv_device *device = cmd_buffer->device;
1047         struct btoi_temps temps;
1048
1049         create_bview(cmd_buffer, src->buffer, src->offset, src->format, &temps.src_bview);
1050         create_iview(cmd_buffer, dst, &temps.dst_iview);
1051         btoi_bind_descriptors(cmd_buffer, &temps);
1052
1053         btoi_bind_pipeline(cmd_buffer);
1054
1055         for (unsigned r = 0; r < num_rects; ++r) {
1056                 unsigned push_constants[3] = {
1057                         rects[r].dst_x,
1058                         rects[r].dst_y,
1059                         src->pitch
1060                 };
1061                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1062                                       device->meta_state.btoi.img_p_layout,
1063                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 12,
1064                                       push_constants);
1065
1066                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1067         }
1068 }
1069
1070 struct itoi_temps {
1071         struct radv_image_view src_iview;
1072         struct radv_image_view dst_iview;
1073 };
1074
1075 static void
1076 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1077                       struct itoi_temps *tmp)
1078 {
1079         struct radv_device *device = cmd_buffer->device;
1080
1081         radv_meta_push_descriptor_set(cmd_buffer,
1082                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1083                                       device->meta_state.itoi.img_p_layout,
1084                                       0, /* set */
1085                                       2, /* descriptorWriteCount */
1086                                       (VkWriteDescriptorSet[]) {
1087                                               {
1088                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1089                                                        .dstBinding = 0,
1090                                                        .dstArrayElement = 0,
1091                                                        .descriptorCount = 1,
1092                                                        .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
1093                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1094                                                                {
1095                                                                        .sampler = VK_NULL_HANDLE,
1096                                                                        .imageView = radv_image_view_to_handle(&tmp->src_iview),
1097                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1098                                                                },
1099                                                        }
1100                                               },
1101                                               {
1102                                                        .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1103                                                        .dstBinding = 1,
1104                                                        .dstArrayElement = 0,
1105                                                        .descriptorCount = 1,
1106                                                        .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1107                                                        .pImageInfo = (VkDescriptorImageInfo[]) {
1108                                                                {
1109                                                                        .sampler = VK_NULL_HANDLE,
1110                                                                        .imageView = radv_image_view_to_handle(&tmp->dst_iview),
1111                                                                        .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1112                                                                },
1113                                                        }
1114                                               }
1115                                       });
1116 }
1117
1118 static void
1119 itoi_bind_pipeline(struct radv_cmd_buffer *cmd_buffer)
1120 {
1121         VkPipeline pipeline =
1122                 cmd_buffer->device->meta_state.itoi.pipeline;
1123
1124         if (cmd_buffer->state.compute_pipeline != radv_pipeline_from_handle(pipeline)) {
1125                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1126                                      VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1127         }
1128 }
1129
1130 void
1131 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1132                             struct radv_meta_blit2d_surf *src,
1133                             struct radv_meta_blit2d_surf *dst,
1134                             unsigned num_rects,
1135                             struct radv_meta_blit2d_rect *rects)
1136 {
1137         struct radv_device *device = cmd_buffer->device;
1138         struct itoi_temps temps;
1139
1140         create_iview(cmd_buffer, src, &temps.src_iview);
1141         create_iview(cmd_buffer, dst, &temps.dst_iview);
1142
1143         itoi_bind_descriptors(cmd_buffer, &temps);
1144
1145         itoi_bind_pipeline(cmd_buffer);
1146
1147         for (unsigned r = 0; r < num_rects; ++r) {
1148                 unsigned push_constants[4] = {
1149                         rects[r].src_x,
1150                         rects[r].src_y,
1151                         rects[r].dst_x,
1152                         rects[r].dst_y,
1153                 };
1154                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1155                                       device->meta_state.itoi.img_p_layout,
1156                                       VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1157                                       push_constants);
1158
1159                 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1160         }
1161 }
1162
1163 static void
1164 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1165                         struct radv_image_view *dst_iview)
1166 {
1167         struct radv_device *device = cmd_buffer->device;
1168
1169         radv_meta_push_descriptor_set(cmd_buffer,
1170                                       VK_PIPELINE_BIND_POINT_COMPUTE,
1171                                       device->meta_state.cleari.img_p_layout,
1172                                       0, /* set */
1173                                       1, /* descriptorWriteCount */
1174                                       (VkWriteDescriptorSet[]) {
1175                                               {
1176                                                       .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1177                                                       .dstBinding = 0,
1178                                                       .dstArrayElement = 0,
1179                                                       .descriptorCount = 1,
1180                                                       .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1181                                                       .pImageInfo = (VkDescriptorImageInfo[]) {
1182                                                                {
1183                                                                       .sampler = VK_NULL_HANDLE,
1184                                                                       .imageView = radv_image_view_to_handle(dst_iview),
1185                                                                       .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1186                                                                },
1187                                                       }
1188                                                },
1189                                       });
1190 }
1191
1192 static void
1193 cleari_bind_pipeline(struct radv_cmd_buffer *cmd_buffer)
1194 {
1195         VkPipeline pipeline =
1196                 cmd_buffer->device->meta_state.cleari.pipeline;
1197
1198         if (cmd_buffer->state.compute_pipeline != radv_pipeline_from_handle(pipeline)) {
1199                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1200                                      VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1201         }
1202 }
1203
1204 void
1205 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer,
1206                          struct radv_meta_blit2d_surf *dst,
1207                          const VkClearColorValue *clear_color)
1208 {
1209         struct radv_device *device = cmd_buffer->device;
1210         struct radv_image_view dst_iview;
1211
1212         create_iview(cmd_buffer, dst, &dst_iview);
1213         cleari_bind_descriptors(cmd_buffer, &dst_iview);
1214
1215         cleari_bind_pipeline(cmd_buffer);
1216
1217         unsigned push_constants[4] = {
1218                 clear_color->uint32[0],
1219                 clear_color->uint32[1],
1220                 clear_color->uint32[2],
1221                 clear_color->uint32[3],
1222         };
1223
1224         radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1225                               device->meta_state.cleari.img_p_layout,
1226                               VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1227                               push_constants);
1228
1229         radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1);
1230 }