projects / mesa.git / blob
? search:


aa17c25833b9af144d282b5f9c815a5d3cf76efc
[mesa.git] / src / amd / vulkan / radv_meta_bufimage.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24 #include "radv_meta.h"
25 #include "nir/nir_builder.h"
26
27 /*
28 * GFX queue: Compute shader implementation of image->buffer copy
29 * Compute queue: implementation also of buffer->image, image->image, and image clear.
30 */
31
32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options
33 * for that.
34 */
35 static nir_shader *
36 build_nir_itob_compute_shader(struct radv_device *dev, bool is_3d)
37 {
38 nir_builder b;
39 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
40 const struct glsl_type *sampler_type = glsl_sampler_type(dim,
41 false,
42 false,
43 GLSL_TYPE_FLOAT);
44 const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
45 false,
46 false,
47 GLSL_TYPE_FLOAT);
48 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
49 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itob_cs_3d" : "meta_itob_cs");
50 b.shader->info.cs.local_size[0] = 16;
51 b.shader->info.cs.local_size[1] = 16;
52 b.shader->info.cs.local_size[2] = 1;
53 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
54 sampler_type, "s_tex");
55 input_img->data.descriptor_set = 0;
56 input_img->data.binding = 0;
57
58 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
59 img_type, "out_img");
60 output_img->data.descriptor_set = 0;
61 output_img->data.binding = 1;
62
63 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
64 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
65 nir_ssa_def *block_size = nir_imm_ivec4(&b,
66 b.shader->info.cs.local_size[0],
67 b.shader->info.cs.local_size[1],
68 b.shader->info.cs.local_size[2], 0);
69
70 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
71
72
73
74 nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
75 nir_intrinsic_set_base(offset, 0);
76 nir_intrinsic_set_range(offset, 16);
77 offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
78 offset->num_components = is_3d ? 3 : 2;
79 nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
80 nir_builder_instr_insert(&b, &offset->instr);
81
82 nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
83 nir_intrinsic_set_base(stride, 0);
84 nir_intrinsic_set_range(stride, 16);
85 stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
86 stride->num_components = 1;
87 nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
88 nir_builder_instr_insert(&b, &stride->instr);
89
90 nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
91 nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
92
93 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
94 tex->sampler_dim = dim;
95 tex->op = nir_texop_txf;
96 tex->src[0].src_type = nir_tex_src_coord;
97 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, is_3d ? 0x7 : 0x3));
98 tex->src[1].src_type = nir_tex_src_lod;
99 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
100 tex->src[2].src_type = nir_tex_src_texture_deref;
101 tex->src[2].src = nir_src_for_ssa(input_img_deref);
102 tex->dest_type = nir_type_float;
103 tex->is_array = false;
104 tex->coord_components = is_3d ? 3 : 2;
105
106 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
107 nir_builder_instr_insert(&b, &tex->instr);
108
109 nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
110 nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
111
112 nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
113 tmp = nir_iadd(&b, tmp, pos_x);
114
115 nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
116
117 nir_ssa_def *outval = &tex->dest.ssa;
118 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
119 store->src[0] = nir_src_for_ssa(&nir_build_deref_var(&b, output_img)->dest.ssa);
120 store->src[1] = nir_src_for_ssa(coord);
121 store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
122 store->src[3] = nir_src_for_ssa(outval);
123
124 nir_builder_instr_insert(&b, &store->instr);
125 return b.shader;
126 }
127
128 /* Image to buffer - don't write use image accessors */
129 static VkResult
130 radv_device_init_meta_itob_state(struct radv_device *device)
131 {
132 VkResult result;
133 struct radv_shader_module cs = { .nir = NULL };
134 struct radv_shader_module cs_3d = { .nir = NULL };
135
136 cs.nir = build_nir_itob_compute_shader(device, false);
137 if (device->physical_device->rad_info.chip_class >= GFX9)
138 cs_3d.nir = build_nir_itob_compute_shader(device, true);
139
140 /*
141 * two descriptors one for the image being sampled
142 * one for the buffer being written.
143 */
144 VkDescriptorSetLayoutCreateInfo ds_create_info = {
145 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
146 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
147 .bindingCount = 2,
148 .pBindings = (VkDescriptorSetLayoutBinding[]) {
149 {
150 .binding = 0,
151 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
152 .descriptorCount = 1,
153 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
154 .pImmutableSamplers = NULL
155 },
156 {
157 .binding = 1,
158 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
159 .descriptorCount = 1,
160 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
161 .pImmutableSamplers = NULL
162 },
163 }
164 };
165
166 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
167 &ds_create_info,
168 &device->meta_state.alloc,
169 &device->meta_state.itob.img_ds_layout);
170 if (result != VK_SUCCESS)
171 goto fail;
172
173
174 VkPipelineLayoutCreateInfo pl_create_info = {
175 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
176 .setLayoutCount = 1,
177 .pSetLayouts = &device->meta_state.itob.img_ds_layout,
178 .pushConstantRangeCount = 1,
179 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
180 };
181
182 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
183 &pl_create_info,
184 &device->meta_state.alloc,
185 &device->meta_state.itob.img_p_layout);
186 if (result != VK_SUCCESS)
187 goto fail;
188
189 /* compute shader */
190
191 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
192 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
193 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
194 .module = radv_shader_module_to_handle(&cs),
195 .pName = "main",
196 .pSpecializationInfo = NULL,
197 };
198
199 VkComputePipelineCreateInfo vk_pipeline_info = {
200 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
201 .stage = pipeline_shader_stage,
202 .flags = 0,
203 .layout = device->meta_state.itob.img_p_layout,
204 };
205
206 result = radv_CreateComputePipelines(radv_device_to_handle(device),
207 radv_pipeline_cache_to_handle(&device->meta_state.cache),
208 1, &vk_pipeline_info, NULL,
209 &device->meta_state.itob.pipeline);
210 if (result != VK_SUCCESS)
211 goto fail;
212
213 if (device->physical_device->rad_info.chip_class >= GFX9) {
214 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
215 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
216 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
217 .module = radv_shader_module_to_handle(&cs_3d),
218 .pName = "main",
219 .pSpecializationInfo = NULL,
220 };
221
222 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
223 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
224 .stage = pipeline_shader_stage_3d,
225 .flags = 0,
226 .layout = device->meta_state.itob.img_p_layout,
227 };
228
229 result = radv_CreateComputePipelines(radv_device_to_handle(device),
230 radv_pipeline_cache_to_handle(&device->meta_state.cache),
231 1, &vk_pipeline_info_3d, NULL,
232 &device->meta_state.itob.pipeline_3d);
233 if (result != VK_SUCCESS)
234 goto fail;
235 ralloc_free(cs_3d.nir);
236 }
237 ralloc_free(cs.nir);
238
239 return VK_SUCCESS;
240 fail:
241 ralloc_free(cs.nir);
242 ralloc_free(cs_3d.nir);
243 return result;
244 }
245
246 static void
247 radv_device_finish_meta_itob_state(struct radv_device *device)
248 {
249 struct radv_meta_state *state = &device->meta_state;
250
251 radv_DestroyPipelineLayout(radv_device_to_handle(device),
252 state->itob.img_p_layout, &state->alloc);
253 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
254 state->itob.img_ds_layout,
255 &state->alloc);
256 radv_DestroyPipeline(radv_device_to_handle(device),
257 state->itob.pipeline, &state->alloc);
258 if (device->physical_device->rad_info.chip_class >= GFX9)
259 radv_DestroyPipeline(radv_device_to_handle(device),
260 state->itob.pipeline_3d, &state->alloc);
261 }
262
263 static nir_shader *
264 build_nir_btoi_compute_shader(struct radv_device *dev, bool is_3d)
265 {
266 nir_builder b;
267 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
268 const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF,
269 false,
270 false,
271 GLSL_TYPE_FLOAT);
272 const struct glsl_type *img_type = glsl_sampler_type(dim,
273 false,
274 false,
275 GLSL_TYPE_FLOAT);
276 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
277 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_btoi_cs_3d" : "meta_btoi_cs");
278 b.shader->info.cs.local_size[0] = 16;
279 b.shader->info.cs.local_size[1] = 16;
280 b.shader->info.cs.local_size[2] = 1;
281 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
282 buf_type, "s_tex");
283 input_img->data.descriptor_set = 0;
284 input_img->data.binding = 0;
285
286 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
287 img_type, "out_img");
288 output_img->data.descriptor_set = 0;
289 output_img->data.binding = 1;
290
291 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
292 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
293 nir_ssa_def *block_size = nir_imm_ivec4(&b,
294 b.shader->info.cs.local_size[0],
295 b.shader->info.cs.local_size[1],
296 b.shader->info.cs.local_size[2], 0);
297
298 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
299
300 nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
301 nir_intrinsic_set_base(offset, 0);
302 nir_intrinsic_set_range(offset, 16);
303 offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
304 offset->num_components = is_3d ? 3 : 2;
305 nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset");
306 nir_builder_instr_insert(&b, &offset->instr);
307
308 nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
309 nir_intrinsic_set_base(stride, 0);
310 nir_intrinsic_set_range(stride, 16);
311 stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
312 stride->num_components = 1;
313 nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride");
314 nir_builder_instr_insert(&b, &stride->instr);
315
316 nir_ssa_def *pos_x = nir_channel(&b, global_id, 0);
317 nir_ssa_def *pos_y = nir_channel(&b, global_id, 1);
318
319 nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa);
320 tmp = nir_iadd(&b, tmp, pos_x);
321
322 nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp);
323
324 nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa);
325 nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
326
327 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
328 tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
329 tex->op = nir_texop_txf;
330 tex->src[0].src_type = nir_tex_src_coord;
331 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1));
332 tex->src[1].src_type = nir_tex_src_lod;
333 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
334 tex->src[2].src_type = nir_tex_src_texture_deref;
335 tex->src[2].src = nir_src_for_ssa(input_img_deref);
336 tex->dest_type = nir_type_float;
337 tex->is_array = false;
338 tex->coord_components = 1;
339
340 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
341 nir_builder_instr_insert(&b, &tex->instr);
342
343 nir_ssa_def *outval = &tex->dest.ssa;
344 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
345 store->src[0] = nir_src_for_ssa(&nir_build_deref_var(&b, output_img)->dest.ssa);
346 store->src[1] = nir_src_for_ssa(img_coord);
347 store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
348 store->src[3] = nir_src_for_ssa(outval);
349
350 nir_builder_instr_insert(&b, &store->instr);
351 return b.shader;
352 }
353
354 /* Buffer to image - don't write use image accessors */
355 static VkResult
356 radv_device_init_meta_btoi_state(struct radv_device *device)
357 {
358 VkResult result;
359 struct radv_shader_module cs = { .nir = NULL };
360 struct radv_shader_module cs_3d = { .nir = NULL };
361 cs.nir = build_nir_btoi_compute_shader(device, false);
362 if (device->physical_device->rad_info.chip_class >= GFX9)
363 cs_3d.nir = build_nir_btoi_compute_shader(device, true);
364 /*
365 * two descriptors one for the image being sampled
366 * one for the buffer being written.
367 */
368 VkDescriptorSetLayoutCreateInfo ds_create_info = {
369 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
370 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
371 .bindingCount = 2,
372 .pBindings = (VkDescriptorSetLayoutBinding[]) {
373 {
374 .binding = 0,
375 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
376 .descriptorCount = 1,
377 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
378 .pImmutableSamplers = NULL
379 },
380 {
381 .binding = 1,
382 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
383 .descriptorCount = 1,
384 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
385 .pImmutableSamplers = NULL
386 },
387 }
388 };
389
390 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
391 &ds_create_info,
392 &device->meta_state.alloc,
393 &device->meta_state.btoi.img_ds_layout);
394 if (result != VK_SUCCESS)
395 goto fail;
396
397
398 VkPipelineLayoutCreateInfo pl_create_info = {
399 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
400 .setLayoutCount = 1,
401 .pSetLayouts = &device->meta_state.btoi.img_ds_layout,
402 .pushConstantRangeCount = 1,
403 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16},
404 };
405
406 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
407 &pl_create_info,
408 &device->meta_state.alloc,
409 &device->meta_state.btoi.img_p_layout);
410 if (result != VK_SUCCESS)
411 goto fail;
412
413 /* compute shader */
414
415 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
416 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
417 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
418 .module = radv_shader_module_to_handle(&cs),
419 .pName = "main",
420 .pSpecializationInfo = NULL,
421 };
422
423 VkComputePipelineCreateInfo vk_pipeline_info = {
424 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
425 .stage = pipeline_shader_stage,
426 .flags = 0,
427 .layout = device->meta_state.btoi.img_p_layout,
428 };
429
430 result = radv_CreateComputePipelines(radv_device_to_handle(device),
431 radv_pipeline_cache_to_handle(&device->meta_state.cache),
432 1, &vk_pipeline_info, NULL,
433 &device->meta_state.btoi.pipeline);
434 if (result != VK_SUCCESS)
435 goto fail;
436
437 if (device->physical_device->rad_info.chip_class >= GFX9) {
438 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
439 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
440 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
441 .module = radv_shader_module_to_handle(&cs_3d),
442 .pName = "main",
443 .pSpecializationInfo = NULL,
444 };
445
446 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
447 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
448 .stage = pipeline_shader_stage_3d,
449 .flags = 0,
450 .layout = device->meta_state.btoi.img_p_layout,
451 };
452
453 result = radv_CreateComputePipelines(radv_device_to_handle(device),
454 radv_pipeline_cache_to_handle(&device->meta_state.cache),
455 1, &vk_pipeline_info_3d, NULL,
456 &device->meta_state.btoi.pipeline_3d);
457 ralloc_free(cs_3d.nir);
458 }
459 ralloc_free(cs.nir);
460
461 return VK_SUCCESS;
462 fail:
463 ralloc_free(cs_3d.nir);
464 ralloc_free(cs.nir);
465 return result;
466 }
467
468 static void
469 radv_device_finish_meta_btoi_state(struct radv_device *device)
470 {
471 struct radv_meta_state *state = &device->meta_state;
472
473 radv_DestroyPipelineLayout(radv_device_to_handle(device),
474 state->btoi.img_p_layout, &state->alloc);
475 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
476 state->btoi.img_ds_layout,
477 &state->alloc);
478 radv_DestroyPipeline(radv_device_to_handle(device),
479 state->btoi.pipeline, &state->alloc);
480 radv_DestroyPipeline(radv_device_to_handle(device),
481 state->btoi.pipeline_3d, &state->alloc);
482 }
483
484 static nir_shader *
485 build_nir_itoi_compute_shader(struct radv_device *dev, bool is_3d)
486 {
487 nir_builder b;
488 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
489 const struct glsl_type *buf_type = glsl_sampler_type(dim,
490 false,
491 false,
492 GLSL_TYPE_FLOAT);
493 const struct glsl_type *img_type = glsl_sampler_type(dim,
494 false,
495 false,
496 GLSL_TYPE_FLOAT);
497 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
498 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itoi_cs_3d" : "meta_itoi_cs");
499 b.shader->info.cs.local_size[0] = 16;
500 b.shader->info.cs.local_size[1] = 16;
501 b.shader->info.cs.local_size[2] = 1;
502 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
503 buf_type, "s_tex");
504 input_img->data.descriptor_set = 0;
505 input_img->data.binding = 0;
506
507 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
508 img_type, "out_img");
509 output_img->data.descriptor_set = 0;
510 output_img->data.binding = 1;
511
512 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
513 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
514 nir_ssa_def *block_size = nir_imm_ivec4(&b,
515 b.shader->info.cs.local_size[0],
516 b.shader->info.cs.local_size[1],
517 b.shader->info.cs.local_size[2], 0);
518
519 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
520
521 nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
522 nir_intrinsic_set_base(src_offset, 0);
523 nir_intrinsic_set_range(src_offset, 24);
524 src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
525 src_offset->num_components = is_3d ? 3 : 2;
526 nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, is_3d ? 3 : 2, 32, "src_offset");
527 nir_builder_instr_insert(&b, &src_offset->instr);
528
529 nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
530 nir_intrinsic_set_base(dst_offset, 0);
531 nir_intrinsic_set_range(dst_offset, 24);
532 dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12));
533 dst_offset->num_components = is_3d ? 3 : 2;
534 nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, is_3d ? 3 : 2, 32, "dst_offset");
535 nir_builder_instr_insert(&b, &dst_offset->instr);
536
537 nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa);
538 nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
539
540 nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
541
542 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
543 tex->sampler_dim = dim;
544 tex->op = nir_texop_txf;
545 tex->src[0].src_type = nir_tex_src_coord;
546 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, is_3d ? 0x7 : 0x3));
547 tex->src[1].src_type = nir_tex_src_lod;
548 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
549 tex->src[2].src_type = nir_tex_src_texture_deref;
550 tex->src[2].src = nir_src_for_ssa(input_img_deref);
551 tex->dest_type = nir_type_float;
552 tex->is_array = false;
553 tex->coord_components = is_3d ? 3 : 2;
554
555 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
556 nir_builder_instr_insert(&b, &tex->instr);
557
558 nir_ssa_def *outval = &tex->dest.ssa;
559 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
560 store->src[0] = nir_src_for_ssa(&nir_build_deref_var(&b, output_img)->dest.ssa);
561 store->src[1] = nir_src_for_ssa(dst_coord);
562 store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
563 store->src[3] = nir_src_for_ssa(outval);
564
565 nir_builder_instr_insert(&b, &store->instr);
566 return b.shader;
567 }
568
569 /* image to image - don't write use image accessors */
570 static VkResult
571 radv_device_init_meta_itoi_state(struct radv_device *device)
572 {
573 VkResult result;
574 struct radv_shader_module cs = { .nir = NULL };
575 struct radv_shader_module cs_3d = { .nir = NULL };
576 cs.nir = build_nir_itoi_compute_shader(device, false);
577 if (device->physical_device->rad_info.chip_class >= GFX9)
578 cs_3d.nir = build_nir_itoi_compute_shader(device, true);
579 /*
580 * two descriptors one for the image being sampled
581 * one for the buffer being written.
582 */
583 VkDescriptorSetLayoutCreateInfo ds_create_info = {
584 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
585 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
586 .bindingCount = 2,
587 .pBindings = (VkDescriptorSetLayoutBinding[]) {
588 {
589 .binding = 0,
590 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
591 .descriptorCount = 1,
592 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
593 .pImmutableSamplers = NULL
594 },
595 {
596 .binding = 1,
597 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
598 .descriptorCount = 1,
599 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
600 .pImmutableSamplers = NULL
601 },
602 }
603 };
604
605 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
606 &ds_create_info,
607 &device->meta_state.alloc,
608 &device->meta_state.itoi.img_ds_layout);
609 if (result != VK_SUCCESS)
610 goto fail;
611
612
613 VkPipelineLayoutCreateInfo pl_create_info = {
614 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
615 .setLayoutCount = 1,
616 .pSetLayouts = &device->meta_state.itoi.img_ds_layout,
617 .pushConstantRangeCount = 1,
618 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 24},
619 };
620
621 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
622 &pl_create_info,
623 &device->meta_state.alloc,
624 &device->meta_state.itoi.img_p_layout);
625 if (result != VK_SUCCESS)
626 goto fail;
627
628 /* compute shader */
629
630 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
631 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
632 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
633 .module = radv_shader_module_to_handle(&cs),
634 .pName = "main",
635 .pSpecializationInfo = NULL,
636 };
637
638 VkComputePipelineCreateInfo vk_pipeline_info = {
639 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
640 .stage = pipeline_shader_stage,
641 .flags = 0,
642 .layout = device->meta_state.itoi.img_p_layout,
643 };
644
645 result = radv_CreateComputePipelines(radv_device_to_handle(device),
646 radv_pipeline_cache_to_handle(&device->meta_state.cache),
647 1, &vk_pipeline_info, NULL,
648 &device->meta_state.itoi.pipeline);
649 if (result != VK_SUCCESS)
650 goto fail;
651
652 if (device->physical_device->rad_info.chip_class >= GFX9) {
653 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
654 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
655 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
656 .module = radv_shader_module_to_handle(&cs_3d),
657 .pName = "main",
658 .pSpecializationInfo = NULL,
659 };
660
661 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
662 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
663 .stage = pipeline_shader_stage_3d,
664 .flags = 0,
665 .layout = device->meta_state.itoi.img_p_layout,
666 };
667
668 result = radv_CreateComputePipelines(radv_device_to_handle(device),
669 radv_pipeline_cache_to_handle(&device->meta_state.cache),
670 1, &vk_pipeline_info_3d, NULL,
671 &device->meta_state.itoi.pipeline_3d);
672
673 ralloc_free(cs_3d.nir);
674 }
675 ralloc_free(cs.nir);
676
677 return VK_SUCCESS;
678 fail:
679 ralloc_free(cs.nir);
680 ralloc_free(cs_3d.nir);
681 return result;
682 }
683
684 static void
685 radv_device_finish_meta_itoi_state(struct radv_device *device)
686 {
687 struct radv_meta_state *state = &device->meta_state;
688
689 radv_DestroyPipelineLayout(radv_device_to_handle(device),
690 state->itoi.img_p_layout, &state->alloc);
691 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
692 state->itoi.img_ds_layout,
693 &state->alloc);
694 radv_DestroyPipeline(radv_device_to_handle(device),
695 state->itoi.pipeline, &state->alloc);
696 if (device->physical_device->rad_info.chip_class >= GFX9)
697 radv_DestroyPipeline(radv_device_to_handle(device),
698 state->itoi.pipeline_3d, &state->alloc);
699 }
700
701 static nir_shader *
702 build_nir_cleari_compute_shader(struct radv_device *dev, bool is_3d)
703 {
704 nir_builder b;
705 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;
706 const struct glsl_type *img_type = glsl_sampler_type(dim,
707 false,
708 false,
709 GLSL_TYPE_FLOAT);
710 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
711 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_cleari_cs_3d" : "meta_cleari_cs");
712 b.shader->info.cs.local_size[0] = 16;
713 b.shader->info.cs.local_size[1] = 16;
714 b.shader->info.cs.local_size[2] = 1;
715
716 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform,
717 img_type, "out_img");
718 output_img->data.descriptor_set = 0;
719 output_img->data.binding = 0;
720
721 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
722 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
723 nir_ssa_def *block_size = nir_imm_ivec4(&b,
724 b.shader->info.cs.local_size[0],
725 b.shader->info.cs.local_size[1],
726 b.shader->info.cs.local_size[2], 0);
727
728 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
729
730 nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
731 nir_intrinsic_set_base(clear_val, 0);
732 nir_intrinsic_set_range(clear_val, 20);
733 clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
734 clear_val->num_components = 4;
735 nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value");
736 nir_builder_instr_insert(&b, &clear_val->instr);
737
738 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
739 nir_intrinsic_set_base(layer, 0);
740 nir_intrinsic_set_range(layer, 20);
741 layer->src[0] = nir_src_for_ssa(nir_imm_int(&b, 16));
742 layer->num_components = 1;
743 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer");
744 nir_builder_instr_insert(&b, &layer->instr);
745
746 nir_ssa_def *global_z = nir_iadd(&b, nir_channel(&b, global_id, 2), &layer->dest.ssa);
747
748 nir_ssa_def *comps[4];
749 comps[0] = nir_channel(&b, global_id, 0);
750 comps[1] = nir_channel(&b, global_id, 1);
751 comps[2] = global_z;
752 comps[3] = nir_imm_int(&b, 0);
753 global_id = nir_vec(&b, comps, 4);
754
755 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
756 store->src[0] = nir_src_for_ssa(&nir_build_deref_var(&b, output_img)->dest.ssa);
757 store->src[1] = nir_src_for_ssa(global_id);
758 store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
759 store->src[3] = nir_src_for_ssa(&clear_val->dest.ssa);
760
761 nir_builder_instr_insert(&b, &store->instr);
762 return b.shader;
763 }
764
765 static VkResult
766 radv_device_init_meta_cleari_state(struct radv_device *device)
767 {
768 VkResult result;
769 struct radv_shader_module cs = { .nir = NULL };
770 struct radv_shader_module cs_3d = { .nir = NULL };
771 cs.nir = build_nir_cleari_compute_shader(device, false);
772 if (device->physical_device->rad_info.chip_class >= GFX9)
773 cs_3d.nir = build_nir_cleari_compute_shader(device, true);
774
775 /*
776 * two descriptors one for the image being sampled
777 * one for the buffer being written.
778 */
779 VkDescriptorSetLayoutCreateInfo ds_create_info = {
780 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
781 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
782 .bindingCount = 1,
783 .pBindings = (VkDescriptorSetLayoutBinding[]) {
784 {
785 .binding = 0,
786 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
787 .descriptorCount = 1,
788 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
789 .pImmutableSamplers = NULL
790 },
791 }
792 };
793
794 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
795 &ds_create_info,
796 &device->meta_state.alloc,
797 &device->meta_state.cleari.img_ds_layout);
798 if (result != VK_SUCCESS)
799 goto fail;
800
801
802 VkPipelineLayoutCreateInfo pl_create_info = {
803 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
804 .setLayoutCount = 1,
805 .pSetLayouts = &device->meta_state.cleari.img_ds_layout,
806 .pushConstantRangeCount = 1,
807 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 20},
808 };
809
810 result = radv_CreatePipelineLayout(radv_device_to_handle(device),
811 &pl_create_info,
812 &device->meta_state.alloc,
813 &device->meta_state.cleari.img_p_layout);
814 if (result != VK_SUCCESS)
815 goto fail;
816
817 /* compute shader */
818
819 VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
820 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
821 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
822 .module = radv_shader_module_to_handle(&cs),
823 .pName = "main",
824 .pSpecializationInfo = NULL,
825 };
826
827 VkComputePipelineCreateInfo vk_pipeline_info = {
828 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
829 .stage = pipeline_shader_stage,
830 .flags = 0,
831 .layout = device->meta_state.cleari.img_p_layout,
832 };
833
834 result = radv_CreateComputePipelines(radv_device_to_handle(device),
835 radv_pipeline_cache_to_handle(&device->meta_state.cache),
836 1, &vk_pipeline_info, NULL,
837 &device->meta_state.cleari.pipeline);
838 if (result != VK_SUCCESS)
839 goto fail;
840
841
842 if (device->physical_device->rad_info.chip_class >= GFX9) {
843 /* compute shader */
844 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = {
845 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
846 .stage = VK_SHADER_STAGE_COMPUTE_BIT,
847 .module = radv_shader_module_to_handle(&cs_3d),
848 .pName = "main",
849 .pSpecializationInfo = NULL,
850 };
851
852 VkComputePipelineCreateInfo vk_pipeline_info_3d = {
853 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
854 .stage = pipeline_shader_stage_3d,
855 .flags = 0,
856 .layout = device->meta_state.cleari.img_p_layout,
857 };
858
859 result = radv_CreateComputePipelines(radv_device_to_handle(device),
860 radv_pipeline_cache_to_handle(&device->meta_state.cache),
861 1, &vk_pipeline_info_3d, NULL,
862 &device->meta_state.cleari.pipeline_3d);
863 if (result != VK_SUCCESS)
864 goto fail;
865
866 ralloc_free(cs_3d.nir);
867 }
868 ralloc_free(cs.nir);
869 return VK_SUCCESS;
870 fail:
871 ralloc_free(cs.nir);
872 ralloc_free(cs_3d.nir);
873 return result;
874 }
875
876 static void
877 radv_device_finish_meta_cleari_state(struct radv_device *device)
878 {
879 struct radv_meta_state *state = &device->meta_state;
880
881 radv_DestroyPipelineLayout(radv_device_to_handle(device),
882 state->cleari.img_p_layout, &state->alloc);
883 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
884 state->cleari.img_ds_layout,
885 &state->alloc);
886 radv_DestroyPipeline(radv_device_to_handle(device),
887 state->cleari.pipeline, &state->alloc);
888 radv_DestroyPipeline(radv_device_to_handle(device),
889 state->cleari.pipeline_3d, &state->alloc);
890 }
891
892 void
893 radv_device_finish_meta_bufimage_state(struct radv_device *device)
894 {
895 radv_device_finish_meta_itob_state(device);
896 radv_device_finish_meta_btoi_state(device);
897 radv_device_finish_meta_itoi_state(device);
898 radv_device_finish_meta_cleari_state(device);
899 }
900
901 VkResult
902 radv_device_init_meta_bufimage_state(struct radv_device *device)
903 {
904 VkResult result;
905
906 result = radv_device_init_meta_itob_state(device);
907 if (result != VK_SUCCESS)
908 goto fail_itob;
909
910 result = radv_device_init_meta_btoi_state(device);
911 if (result != VK_SUCCESS)
912 goto fail_btoi;
913
914 result = radv_device_init_meta_itoi_state(device);
915 if (result != VK_SUCCESS)
916 goto fail_itoi;
917
918 result = radv_device_init_meta_cleari_state(device);
919 if (result != VK_SUCCESS)
920 goto fail_cleari;
921
922 return VK_SUCCESS;
923 fail_cleari:
924 radv_device_finish_meta_cleari_state(device);
925 fail_itoi:
926 radv_device_finish_meta_itoi_state(device);
927 fail_btoi:
928 radv_device_finish_meta_btoi_state(device);
929 fail_itob:
930 radv_device_finish_meta_itob_state(device);
931 return result;
932 }
933
934 static void
935 create_iview(struct radv_cmd_buffer *cmd_buffer,
936 struct radv_meta_blit2d_surf *surf,
937 struct radv_image_view *iview)
938 {
939 VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D :
940 radv_meta_get_view_type(surf->image);
941 radv_image_view_init(iview, cmd_buffer->device,
942 &(VkImageViewCreateInfo) {
943 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
944 .image = radv_image_to_handle(surf->image),
945 .viewType = view_type,
946 .format = surf->format,
947 .subresourceRange = {
948 .aspectMask = surf->aspect_mask,
949 .baseMipLevel = surf->level,
950 .levelCount = 1,
951 .baseArrayLayer = surf->layer,
952 .layerCount = 1
953 },
954 });
955 }
956
957 static void
958 create_bview(struct radv_cmd_buffer *cmd_buffer,
959 struct radv_buffer *buffer,
960 unsigned offset,
961 VkFormat format,
962 struct radv_buffer_view *bview)
963 {
964 radv_buffer_view_init(bview, cmd_buffer->device,
965 &(VkBufferViewCreateInfo) {
966 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
967 .flags = 0,
968 .buffer = radv_buffer_to_handle(buffer),
969 .format = format,
970 .offset = offset,
971 .range = VK_WHOLE_SIZE,
972 });
973
974 }
975
976 static void
977 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
978 struct radv_image_view *src,
979 struct radv_buffer_view *dst)
980 {
981 struct radv_device *device = cmd_buffer->device;
982
983 radv_meta_push_descriptor_set(cmd_buffer,
984 VK_PIPELINE_BIND_POINT_COMPUTE,
985 device->meta_state.itob.img_p_layout,
986 0, /* set */
987 2, /* descriptorWriteCount */
988 (VkWriteDescriptorSet[]) {
989 {
990 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
991 .dstBinding = 0,
992 .dstArrayElement = 0,
993 .descriptorCount = 1,
994 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
995 .pImageInfo = (VkDescriptorImageInfo[]) {
996 {
997 .sampler = VK_NULL_HANDLE,
998 .imageView = radv_image_view_to_handle(src),
999 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1000 },
1001 }
1002 },
1003 {
1004 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1005 .dstBinding = 1,
1006 .dstArrayElement = 0,
1007 .descriptorCount = 1,
1008 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1009 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(dst) },
1010 }
1011 });
1012 }
1013
1014 void
1015 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
1016 struct radv_meta_blit2d_surf *src,
1017 struct radv_meta_blit2d_buffer *dst,
1018 unsigned num_rects,
1019 struct radv_meta_blit2d_rect *rects)
1020 {
1021 VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline;
1022 struct radv_device *device = cmd_buffer->device;
1023 struct radv_image_view src_view;
1024 struct radv_buffer_view dst_view;
1025
1026 create_iview(cmd_buffer, src, &src_view);
1027 create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &dst_view);
1028 itob_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1029
1030 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1031 src->image->type == VK_IMAGE_TYPE_3D)
1032 pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d;
1033
1034 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1035 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1036
1037 for (unsigned r = 0; r < num_rects; ++r) {
1038 unsigned push_constants[4] = {
1039 rects[r].src_x,
1040 rects[r].src_y,
1041 src->layer,
1042 dst->pitch
1043 };
1044 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1045 device->meta_state.itob.img_p_layout,
1046 VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1047 push_constants);
1048
1049 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1050 }
1051 }
1052
1053 static void
1054 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1055 struct radv_buffer_view *src,
1056 struct radv_image_view *dst)
1057 {
1058 struct radv_device *device = cmd_buffer->device;
1059
1060 radv_meta_push_descriptor_set(cmd_buffer,
1061 VK_PIPELINE_BIND_POINT_COMPUTE,
1062 device->meta_state.btoi.img_p_layout,
1063 0, /* set */
1064 2, /* descriptorWriteCount */
1065 (VkWriteDescriptorSet[]) {
1066 {
1067 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1068 .dstBinding = 0,
1069 .dstArrayElement = 0,
1070 .descriptorCount = 1,
1071 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
1072 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(src) },
1073 },
1074 {
1075 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1076 .dstBinding = 1,
1077 .dstArrayElement = 0,
1078 .descriptorCount = 1,
1079 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1080 .pImageInfo = (VkDescriptorImageInfo[]) {
1081 {
1082 .sampler = VK_NULL_HANDLE,
1083 .imageView = radv_image_view_to_handle(dst),
1084 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1085 },
1086 }
1087 }
1088 });
1089 }
1090
1091 void
1092 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1093 struct radv_meta_blit2d_buffer *src,
1094 struct radv_meta_blit2d_surf *dst,
1095 unsigned num_rects,
1096 struct radv_meta_blit2d_rect *rects)
1097 {
1098 VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline;
1099 struct radv_device *device = cmd_buffer->device;
1100 struct radv_buffer_view src_view;
1101 struct radv_image_view dst_view;
1102
1103 create_bview(cmd_buffer, src->buffer, src->offset, src->format, &src_view);
1104 create_iview(cmd_buffer, dst, &dst_view);
1105 btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1106
1107 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1108 dst->image->type == VK_IMAGE_TYPE_3D)
1109 pipeline = cmd_buffer->device->meta_state.btoi.pipeline_3d;
1110 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1111 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1112
1113 for (unsigned r = 0; r < num_rects; ++r) {
1114 unsigned push_constants[4] = {
1115 rects[r].dst_x,
1116 rects[r].dst_y,
1117 dst->layer,
1118 src->pitch,
1119 };
1120 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1121 device->meta_state.btoi.img_p_layout,
1122 VK_SHADER_STAGE_COMPUTE_BIT, 0, 16,
1123 push_constants);
1124
1125 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1126 }
1127 }
1128
1129 static void
1130 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1131 struct radv_image_view *src,
1132 struct radv_image_view *dst)
1133 {
1134 struct radv_device *device = cmd_buffer->device;
1135
1136 radv_meta_push_descriptor_set(cmd_buffer,
1137 VK_PIPELINE_BIND_POINT_COMPUTE,
1138 device->meta_state.itoi.img_p_layout,
1139 0, /* set */
1140 2, /* descriptorWriteCount */
1141 (VkWriteDescriptorSet[]) {
1142 {
1143 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1144 .dstBinding = 0,
1145 .dstArrayElement = 0,
1146 .descriptorCount = 1,
1147 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
1148 .pImageInfo = (VkDescriptorImageInfo[]) {
1149 {
1150 .sampler = VK_NULL_HANDLE,
1151 .imageView = radv_image_view_to_handle(src),
1152 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1153 },
1154 }
1155 },
1156 {
1157 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1158 .dstBinding = 1,
1159 .dstArrayElement = 0,
1160 .descriptorCount = 1,
1161 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1162 .pImageInfo = (VkDescriptorImageInfo[]) {
1163 {
1164 .sampler = VK_NULL_HANDLE,
1165 .imageView = radv_image_view_to_handle(dst),
1166 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1167 },
1168 }
1169 }
1170 });
1171 }
1172
1173 void
1174 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
1175 struct radv_meta_blit2d_surf *src,
1176 struct radv_meta_blit2d_surf *dst,
1177 unsigned num_rects,
1178 struct radv_meta_blit2d_rect *rects)
1179 {
1180 VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline;
1181 struct radv_device *device = cmd_buffer->device;
1182 struct radv_image_view src_view, dst_view;
1183
1184 create_iview(cmd_buffer, src, &src_view);
1185 create_iview(cmd_buffer, dst, &dst_view);
1186
1187 itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view);
1188
1189 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1190 src->image->type == VK_IMAGE_TYPE_3D)
1191 pipeline = cmd_buffer->device->meta_state.itoi.pipeline_3d;
1192 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1193 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1194
1195 for (unsigned r = 0; r < num_rects; ++r) {
1196 unsigned push_constants[6] = {
1197 rects[r].src_x,
1198 rects[r].src_y,
1199 src->layer,
1200 rects[r].dst_x,
1201 rects[r].dst_y,
1202 dst->layer,
1203 };
1204 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1205 device->meta_state.itoi.img_p_layout,
1206 VK_SHADER_STAGE_COMPUTE_BIT, 0, 24,
1207 push_constants);
1208
1209 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
1210 }
1211 }
1212
1213 static void
1214 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer,
1215 struct radv_image_view *dst_iview)
1216 {
1217 struct radv_device *device = cmd_buffer->device;
1218
1219 radv_meta_push_descriptor_set(cmd_buffer,
1220 VK_PIPELINE_BIND_POINT_COMPUTE,
1221 device->meta_state.cleari.img_p_layout,
1222 0, /* set */
1223 1, /* descriptorWriteCount */
1224 (VkWriteDescriptorSet[]) {
1225 {
1226 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
1227 .dstBinding = 0,
1228 .dstArrayElement = 0,
1229 .descriptorCount = 1,
1230 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
1231 .pImageInfo = (VkDescriptorImageInfo[]) {
1232 {
1233 .sampler = VK_NULL_HANDLE,
1234 .imageView = radv_image_view_to_handle(dst_iview),
1235 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
1236 },
1237 }
1238 },
1239 });
1240 }
1241
1242 void
1243 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer,
1244 struct radv_meta_blit2d_surf *dst,
1245 const VkClearColorValue *clear_color)
1246 {
1247 VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline;
1248 struct radv_device *device = cmd_buffer->device;
1249 struct radv_image_view dst_iview;
1250
1251 create_iview(cmd_buffer, dst, &dst_iview);
1252 cleari_bind_descriptors(cmd_buffer, &dst_iview);
1253
1254 if (device->physical_device->rad_info.chip_class >= GFX9 &&
1255 dst->image->type == VK_IMAGE_TYPE_3D)
1256 pipeline = cmd_buffer->device->meta_state.cleari.pipeline_3d;
1257
1258 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
1259 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1260
1261 unsigned push_constants[5] = {
1262 clear_color->uint32[0],
1263 clear_color->uint32[1],
1264 clear_color->uint32[2],
1265 clear_color->uint32[3],
1266 dst->layer,
1267 };
1268
1269 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
1270 device->meta_state.cleari.img_p_layout,
1271 VK_SHADER_STAGE_COMPUTE_BIT, 0, 20,
1272 push_constants);
1273
1274 radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1);
1275 }