2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "radv_meta.h"
25 #include "nir/nir_builder.h"
28 * GFX queue: Compute shader implementation of image->buffer copy
29 * Compute queue: implementation also of buffer->image, image->image, and image clear.
32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options
36 build_nir_itob_compute_shader(struct radv_device
*dev
, bool is_3d
)
39 enum glsl_sampler_dim dim
= is_3d
? GLSL_SAMPLER_DIM_3D
: GLSL_SAMPLER_DIM_2D
;
40 const struct glsl_type
*sampler_type
= glsl_sampler_type(dim
,
44 const struct glsl_type
*img_type
= glsl_sampler_type(GLSL_SAMPLER_DIM_BUF
,
48 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
49 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, is_3d
? "meta_itob_cs_3d" : "meta_itob_cs");
50 b
.shader
->info
.cs
.local_size
[0] = 16;
51 b
.shader
->info
.cs
.local_size
[1] = 16;
52 b
.shader
->info
.cs
.local_size
[2] = 1;
53 nir_variable
*input_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
54 sampler_type
, "s_tex");
55 input_img
->data
.descriptor_set
= 0;
56 input_img
->data
.binding
= 0;
58 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
60 output_img
->data
.descriptor_set
= 0;
61 output_img
->data
.binding
= 1;
63 nir_ssa_def
*invoc_id
= nir_load_system_value(&b
, nir_intrinsic_load_local_invocation_id
, 0);
64 nir_ssa_def
*wg_id
= nir_load_system_value(&b
, nir_intrinsic_load_work_group_id
, 0);
65 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
66 b
.shader
->info
.cs
.local_size
[0],
67 b
.shader
->info
.cs
.local_size
[1],
68 b
.shader
->info
.cs
.local_size
[2], 0);
70 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
74 nir_intrinsic_instr
*offset
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
75 nir_intrinsic_set_base(offset
, 0);
76 nir_intrinsic_set_range(offset
, 16);
77 offset
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 0));
78 offset
->num_components
= is_3d
? 3 : 2;
79 nir_ssa_dest_init(&offset
->instr
, &offset
->dest
, is_3d
? 3 : 2, 32, "offset");
80 nir_builder_instr_insert(&b
, &offset
->instr
);
82 nir_intrinsic_instr
*stride
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
83 nir_intrinsic_set_base(stride
, 0);
84 nir_intrinsic_set_range(stride
, 16);
85 stride
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 12));
86 stride
->num_components
= 1;
87 nir_ssa_dest_init(&stride
->instr
, &stride
->dest
, 1, 32, "stride");
88 nir_builder_instr_insert(&b
, &stride
->instr
);
90 nir_ssa_def
*img_coord
= nir_iadd(&b
, global_id
, &offset
->dest
.ssa
);
91 nir_ssa_def
*input_img_deref
= &nir_build_deref_var(&b
, input_img
)->dest
.ssa
;
93 nir_tex_instr
*tex
= nir_tex_instr_create(b
.shader
, 3);
94 tex
->sampler_dim
= dim
;
95 tex
->op
= nir_texop_txf
;
96 tex
->src
[0].src_type
= nir_tex_src_coord
;
97 tex
->src
[0].src
= nir_src_for_ssa(nir_channels(&b
, img_coord
, is_3d
? 0x7 : 0x3));
98 tex
->src
[1].src_type
= nir_tex_src_lod
;
99 tex
->src
[1].src
= nir_src_for_ssa(nir_imm_int(&b
, 0));
100 tex
->src
[2].src_type
= nir_tex_src_texture_deref
;
101 tex
->src
[2].src
= nir_src_for_ssa(input_img_deref
);
102 tex
->dest_type
= nir_type_float
;
103 tex
->is_array
= false;
104 tex
->coord_components
= is_3d
? 3 : 2;
106 nir_ssa_dest_init(&tex
->instr
, &tex
->dest
, 4, 32, "tex");
107 nir_builder_instr_insert(&b
, &tex
->instr
);
109 nir_ssa_def
*pos_x
= nir_channel(&b
, global_id
, 0);
110 nir_ssa_def
*pos_y
= nir_channel(&b
, global_id
, 1);
112 nir_ssa_def
*tmp
= nir_imul(&b
, pos_y
, &stride
->dest
.ssa
);
113 tmp
= nir_iadd(&b
, tmp
, pos_x
);
115 nir_ssa_def
*coord
= nir_vec4(&b
, tmp
, tmp
, tmp
, tmp
);
117 nir_ssa_def
*outval
= &tex
->dest
.ssa
;
118 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
119 store
->num_components
= 4;
120 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
121 store
->src
[1] = nir_src_for_ssa(coord
);
122 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
123 store
->src
[3] = nir_src_for_ssa(outval
);
125 nir_builder_instr_insert(&b
, &store
->instr
);
129 /* Image to buffer - don't write use image accessors */
131 radv_device_init_meta_itob_state(struct radv_device
*device
)
134 struct radv_shader_module cs
= { .nir
= NULL
};
135 struct radv_shader_module cs_3d
= { .nir
= NULL
};
137 cs
.nir
= build_nir_itob_compute_shader(device
, false);
138 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
)
139 cs_3d
.nir
= build_nir_itob_compute_shader(device
, true);
142 * two descriptors one for the image being sampled
143 * one for the buffer being written.
145 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
146 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
147 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
149 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
152 .descriptorType
= VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE
,
153 .descriptorCount
= 1,
154 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
155 .pImmutableSamplers
= NULL
159 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
160 .descriptorCount
= 1,
161 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
162 .pImmutableSamplers
= NULL
167 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
169 &device
->meta_state
.alloc
,
170 &device
->meta_state
.itob
.img_ds_layout
);
171 if (result
!= VK_SUCCESS
)
175 VkPipelineLayoutCreateInfo pl_create_info
= {
176 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
178 .pSetLayouts
= &device
->meta_state
.itob
.img_ds_layout
,
179 .pushConstantRangeCount
= 1,
180 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16},
183 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
185 &device
->meta_state
.alloc
,
186 &device
->meta_state
.itob
.img_p_layout
);
187 if (result
!= VK_SUCCESS
)
192 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
193 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
194 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
195 .module
= radv_shader_module_to_handle(&cs
),
197 .pSpecializationInfo
= NULL
,
200 VkComputePipelineCreateInfo vk_pipeline_info
= {
201 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
202 .stage
= pipeline_shader_stage
,
204 .layout
= device
->meta_state
.itob
.img_p_layout
,
207 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
208 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
209 1, &vk_pipeline_info
, NULL
,
210 &device
->meta_state
.itob
.pipeline
);
211 if (result
!= VK_SUCCESS
)
214 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
215 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d
= {
216 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
217 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
218 .module
= radv_shader_module_to_handle(&cs_3d
),
220 .pSpecializationInfo
= NULL
,
223 VkComputePipelineCreateInfo vk_pipeline_info_3d
= {
224 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
225 .stage
= pipeline_shader_stage_3d
,
227 .layout
= device
->meta_state
.itob
.img_p_layout
,
230 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
231 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
232 1, &vk_pipeline_info_3d
, NULL
,
233 &device
->meta_state
.itob
.pipeline_3d
);
234 if (result
!= VK_SUCCESS
)
236 ralloc_free(cs_3d
.nir
);
243 ralloc_free(cs_3d
.nir
);
248 radv_device_finish_meta_itob_state(struct radv_device
*device
)
250 struct radv_meta_state
*state
= &device
->meta_state
;
252 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
253 state
->itob
.img_p_layout
, &state
->alloc
);
254 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
255 state
->itob
.img_ds_layout
,
257 radv_DestroyPipeline(radv_device_to_handle(device
),
258 state
->itob
.pipeline
, &state
->alloc
);
259 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
)
260 radv_DestroyPipeline(radv_device_to_handle(device
),
261 state
->itob
.pipeline_3d
, &state
->alloc
);
265 build_nir_btoi_compute_shader(struct radv_device
*dev
, bool is_3d
)
268 enum glsl_sampler_dim dim
= is_3d
? GLSL_SAMPLER_DIM_3D
: GLSL_SAMPLER_DIM_2D
;
269 const struct glsl_type
*buf_type
= glsl_sampler_type(GLSL_SAMPLER_DIM_BUF
,
273 const struct glsl_type
*img_type
= glsl_sampler_type(dim
,
277 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
278 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, is_3d
? "meta_btoi_cs_3d" : "meta_btoi_cs");
279 b
.shader
->info
.cs
.local_size
[0] = 16;
280 b
.shader
->info
.cs
.local_size
[1] = 16;
281 b
.shader
->info
.cs
.local_size
[2] = 1;
282 nir_variable
*input_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
284 input_img
->data
.descriptor_set
= 0;
285 input_img
->data
.binding
= 0;
287 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
288 img_type
, "out_img");
289 output_img
->data
.descriptor_set
= 0;
290 output_img
->data
.binding
= 1;
292 nir_ssa_def
*invoc_id
= nir_load_system_value(&b
, nir_intrinsic_load_local_invocation_id
, 0);
293 nir_ssa_def
*wg_id
= nir_load_system_value(&b
, nir_intrinsic_load_work_group_id
, 0);
294 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
295 b
.shader
->info
.cs
.local_size
[0],
296 b
.shader
->info
.cs
.local_size
[1],
297 b
.shader
->info
.cs
.local_size
[2], 0);
299 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
301 nir_intrinsic_instr
*offset
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
302 nir_intrinsic_set_base(offset
, 0);
303 nir_intrinsic_set_range(offset
, 16);
304 offset
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 0));
305 offset
->num_components
= is_3d
? 3 : 2;
306 nir_ssa_dest_init(&offset
->instr
, &offset
->dest
, is_3d
? 3 : 2, 32, "offset");
307 nir_builder_instr_insert(&b
, &offset
->instr
);
309 nir_intrinsic_instr
*stride
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
310 nir_intrinsic_set_base(stride
, 0);
311 nir_intrinsic_set_range(stride
, 16);
312 stride
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 12));
313 stride
->num_components
= 1;
314 nir_ssa_dest_init(&stride
->instr
, &stride
->dest
, 1, 32, "stride");
315 nir_builder_instr_insert(&b
, &stride
->instr
);
317 nir_ssa_def
*pos_x
= nir_channel(&b
, global_id
, 0);
318 nir_ssa_def
*pos_y
= nir_channel(&b
, global_id
, 1);
320 nir_ssa_def
*tmp
= nir_imul(&b
, pos_y
, &stride
->dest
.ssa
);
321 tmp
= nir_iadd(&b
, tmp
, pos_x
);
323 nir_ssa_def
*buf_coord
= nir_vec4(&b
, tmp
, tmp
, tmp
, tmp
);
325 nir_ssa_def
*img_coord
= nir_iadd(&b
, global_id
, &offset
->dest
.ssa
);
326 nir_ssa_def
*input_img_deref
= &nir_build_deref_var(&b
, input_img
)->dest
.ssa
;
328 nir_tex_instr
*tex
= nir_tex_instr_create(b
.shader
, 3);
329 tex
->sampler_dim
= GLSL_SAMPLER_DIM_BUF
;
330 tex
->op
= nir_texop_txf
;
331 tex
->src
[0].src_type
= nir_tex_src_coord
;
332 tex
->src
[0].src
= nir_src_for_ssa(nir_channels(&b
, buf_coord
, 1));
333 tex
->src
[1].src_type
= nir_tex_src_lod
;
334 tex
->src
[1].src
= nir_src_for_ssa(nir_imm_int(&b
, 0));
335 tex
->src
[2].src_type
= nir_tex_src_texture_deref
;
336 tex
->src
[2].src
= nir_src_for_ssa(input_img_deref
);
337 tex
->dest_type
= nir_type_float
;
338 tex
->is_array
= false;
339 tex
->coord_components
= 1;
341 nir_ssa_dest_init(&tex
->instr
, &tex
->dest
, 4, 32, "tex");
342 nir_builder_instr_insert(&b
, &tex
->instr
);
344 nir_ssa_def
*outval
= &tex
->dest
.ssa
;
345 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
346 store
->num_components
= 4;
347 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
348 store
->src
[1] = nir_src_for_ssa(img_coord
);
349 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
350 store
->src
[3] = nir_src_for_ssa(outval
);
352 nir_builder_instr_insert(&b
, &store
->instr
);
356 /* Buffer to image - don't write use image accessors */
358 radv_device_init_meta_btoi_state(struct radv_device
*device
)
361 struct radv_shader_module cs
= { .nir
= NULL
};
362 struct radv_shader_module cs_3d
= { .nir
= NULL
};
363 cs
.nir
= build_nir_btoi_compute_shader(device
, false);
364 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
)
365 cs_3d
.nir
= build_nir_btoi_compute_shader(device
, true);
367 * two descriptors one for the image being sampled
368 * one for the buffer being written.
370 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
371 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
372 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
374 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
377 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
378 .descriptorCount
= 1,
379 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
380 .pImmutableSamplers
= NULL
384 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
385 .descriptorCount
= 1,
386 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
387 .pImmutableSamplers
= NULL
392 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
394 &device
->meta_state
.alloc
,
395 &device
->meta_state
.btoi
.img_ds_layout
);
396 if (result
!= VK_SUCCESS
)
400 VkPipelineLayoutCreateInfo pl_create_info
= {
401 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
403 .pSetLayouts
= &device
->meta_state
.btoi
.img_ds_layout
,
404 .pushConstantRangeCount
= 1,
405 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16},
408 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
410 &device
->meta_state
.alloc
,
411 &device
->meta_state
.btoi
.img_p_layout
);
412 if (result
!= VK_SUCCESS
)
417 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
418 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
419 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
420 .module
= radv_shader_module_to_handle(&cs
),
422 .pSpecializationInfo
= NULL
,
425 VkComputePipelineCreateInfo vk_pipeline_info
= {
426 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
427 .stage
= pipeline_shader_stage
,
429 .layout
= device
->meta_state
.btoi
.img_p_layout
,
432 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
433 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
434 1, &vk_pipeline_info
, NULL
,
435 &device
->meta_state
.btoi
.pipeline
);
436 if (result
!= VK_SUCCESS
)
439 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
440 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d
= {
441 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
442 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
443 .module
= radv_shader_module_to_handle(&cs_3d
),
445 .pSpecializationInfo
= NULL
,
448 VkComputePipelineCreateInfo vk_pipeline_info_3d
= {
449 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
450 .stage
= pipeline_shader_stage_3d
,
452 .layout
= device
->meta_state
.btoi
.img_p_layout
,
455 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
456 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
457 1, &vk_pipeline_info_3d
, NULL
,
458 &device
->meta_state
.btoi
.pipeline_3d
);
459 ralloc_free(cs_3d
.nir
);
465 ralloc_free(cs_3d
.nir
);
471 radv_device_finish_meta_btoi_state(struct radv_device
*device
)
473 struct radv_meta_state
*state
= &device
->meta_state
;
475 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
476 state
->btoi
.img_p_layout
, &state
->alloc
);
477 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
478 state
->btoi
.img_ds_layout
,
480 radv_DestroyPipeline(radv_device_to_handle(device
),
481 state
->btoi
.pipeline
, &state
->alloc
);
482 radv_DestroyPipeline(radv_device_to_handle(device
),
483 state
->btoi
.pipeline_3d
, &state
->alloc
);
486 /* Buffer to image - special path for R32G32B32 */
488 build_nir_btoi_r32g32b32_compute_shader(struct radv_device
*dev
)
491 const struct glsl_type
*buf_type
= glsl_sampler_type(GLSL_SAMPLER_DIM_BUF
,
495 const struct glsl_type
*img_type
= glsl_sampler_type(GLSL_SAMPLER_DIM_BUF
,
499 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
500 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, "meta_btoi_r32g32b32_cs");
501 b
.shader
->info
.cs
.local_size
[0] = 16;
502 b
.shader
->info
.cs
.local_size
[1] = 16;
503 b
.shader
->info
.cs
.local_size
[2] = 1;
504 nir_variable
*input_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
506 input_img
->data
.descriptor_set
= 0;
507 input_img
->data
.binding
= 0;
509 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
510 img_type
, "out_img");
511 output_img
->data
.descriptor_set
= 0;
512 output_img
->data
.binding
= 1;
514 nir_ssa_def
*invoc_id
= nir_load_system_value(&b
, nir_intrinsic_load_local_invocation_id
, 0);
515 nir_ssa_def
*wg_id
= nir_load_system_value(&b
, nir_intrinsic_load_work_group_id
, 0);
516 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
517 b
.shader
->info
.cs
.local_size
[0],
518 b
.shader
->info
.cs
.local_size
[1],
519 b
.shader
->info
.cs
.local_size
[2], 0);
521 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
523 nir_intrinsic_instr
*offset
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
524 nir_intrinsic_set_base(offset
, 0);
525 nir_intrinsic_set_range(offset
, 16);
526 offset
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 0));
527 offset
->num_components
= 2;
528 nir_ssa_dest_init(&offset
->instr
, &offset
->dest
, 2, 32, "offset");
529 nir_builder_instr_insert(&b
, &offset
->instr
);
531 nir_intrinsic_instr
*pitch
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
532 nir_intrinsic_set_base(pitch
, 0);
533 nir_intrinsic_set_range(pitch
, 16);
534 pitch
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 8));
535 pitch
->num_components
= 1;
536 nir_ssa_dest_init(&pitch
->instr
, &pitch
->dest
, 1, 32, "pitch");
537 nir_builder_instr_insert(&b
, &pitch
->instr
);
539 nir_intrinsic_instr
*stride
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
540 nir_intrinsic_set_base(stride
, 0);
541 nir_intrinsic_set_range(stride
, 16);
542 stride
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 12));
543 stride
->num_components
= 1;
544 nir_ssa_dest_init(&stride
->instr
, &stride
->dest
, 1, 32, "stride");
545 nir_builder_instr_insert(&b
, &stride
->instr
);
547 nir_ssa_def
*pos_x
= nir_channel(&b
, global_id
, 0);
548 nir_ssa_def
*pos_y
= nir_channel(&b
, global_id
, 1);
550 nir_ssa_def
*tmp
= nir_imul(&b
, pos_y
, &stride
->dest
.ssa
);
551 tmp
= nir_iadd(&b
, tmp
, pos_x
);
553 nir_ssa_def
*buf_coord
= nir_vec4(&b
, tmp
, tmp
, tmp
, tmp
);
555 nir_ssa_def
*img_coord
= nir_iadd(&b
, global_id
, &offset
->dest
.ssa
);
557 nir_ssa_def
*global_pos
=
559 nir_imul(&b
, nir_channel(&b
, img_coord
, 1), &pitch
->dest
.ssa
),
560 nir_imul(&b
, nir_channel(&b
, img_coord
, 0), nir_imm_int(&b
, 3)));
562 nir_ssa_def
*input_img_deref
= &nir_build_deref_var(&b
, input_img
)->dest
.ssa
;
564 nir_tex_instr
*tex
= nir_tex_instr_create(b
.shader
, 3);
565 tex
->sampler_dim
= GLSL_SAMPLER_DIM_BUF
;
566 tex
->op
= nir_texop_txf
;
567 tex
->src
[0].src_type
= nir_tex_src_coord
;
568 tex
->src
[0].src
= nir_src_for_ssa(nir_channels(&b
, buf_coord
, 1));
569 tex
->src
[1].src_type
= nir_tex_src_lod
;
570 tex
->src
[1].src
= nir_src_for_ssa(nir_imm_int(&b
, 0));
571 tex
->src
[2].src_type
= nir_tex_src_texture_deref
;
572 tex
->src
[2].src
= nir_src_for_ssa(input_img_deref
);
573 tex
->dest_type
= nir_type_float
;
574 tex
->is_array
= false;
575 tex
->coord_components
= 1;
576 nir_ssa_dest_init(&tex
->instr
, &tex
->dest
, 4, 32, "tex");
577 nir_builder_instr_insert(&b
, &tex
->instr
);
579 nir_ssa_def
*outval
= &tex
->dest
.ssa
;
581 for (int chan
= 0; chan
< 3; chan
++) {
582 nir_ssa_def
*local_pos
=
583 nir_iadd(&b
, global_pos
, nir_imm_int(&b
, chan
));
586 nir_vec4(&b
, local_pos
, local_pos
, local_pos
, local_pos
);
588 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
589 store
->num_components
= 1;
590 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
591 store
->src
[1] = nir_src_for_ssa(coord
);
592 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
593 store
->src
[3] = nir_src_for_ssa(nir_channel(&b
, outval
, chan
));
594 nir_builder_instr_insert(&b
, &store
->instr
);
601 radv_device_init_meta_btoi_r32g32b32_state(struct radv_device
*device
)
604 struct radv_shader_module cs
= { .nir
= NULL
};
606 cs
.nir
= build_nir_btoi_r32g32b32_compute_shader(device
);
608 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
609 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
610 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
612 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
615 .descriptorType
= VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER
,
616 .descriptorCount
= 1,
617 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
618 .pImmutableSamplers
= NULL
622 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
623 .descriptorCount
= 1,
624 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
625 .pImmutableSamplers
= NULL
630 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
632 &device
->meta_state
.alloc
,
633 &device
->meta_state
.btoi_r32g32b32
.img_ds_layout
);
634 if (result
!= VK_SUCCESS
)
638 VkPipelineLayoutCreateInfo pl_create_info
= {
639 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
641 .pSetLayouts
= &device
->meta_state
.btoi_r32g32b32
.img_ds_layout
,
642 .pushConstantRangeCount
= 1,
643 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16},
646 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
648 &device
->meta_state
.alloc
,
649 &device
->meta_state
.btoi_r32g32b32
.img_p_layout
);
650 if (result
!= VK_SUCCESS
)
655 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
656 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
657 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
658 .module
= radv_shader_module_to_handle(&cs
),
660 .pSpecializationInfo
= NULL
,
663 VkComputePipelineCreateInfo vk_pipeline_info
= {
664 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
665 .stage
= pipeline_shader_stage
,
667 .layout
= device
->meta_state
.btoi_r32g32b32
.img_p_layout
,
670 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
671 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
672 1, &vk_pipeline_info
, NULL
,
673 &device
->meta_state
.btoi_r32g32b32
.pipeline
);
681 radv_device_finish_meta_btoi_r32g32b32_state(struct radv_device
*device
)
683 struct radv_meta_state
*state
= &device
->meta_state
;
685 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
686 state
->btoi_r32g32b32
.img_p_layout
, &state
->alloc
);
687 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
688 state
->btoi_r32g32b32
.img_ds_layout
,
690 radv_DestroyPipeline(radv_device_to_handle(device
),
691 state
->btoi_r32g32b32
.pipeline
, &state
->alloc
);
695 build_nir_itoi_compute_shader(struct radv_device
*dev
, bool is_3d
)
698 enum glsl_sampler_dim dim
= is_3d
? GLSL_SAMPLER_DIM_3D
: GLSL_SAMPLER_DIM_2D
;
699 const struct glsl_type
*buf_type
= glsl_sampler_type(dim
,
703 const struct glsl_type
*img_type
= glsl_sampler_type(dim
,
707 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
708 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, is_3d
? "meta_itoi_cs_3d" : "meta_itoi_cs");
709 b
.shader
->info
.cs
.local_size
[0] = 16;
710 b
.shader
->info
.cs
.local_size
[1] = 16;
711 b
.shader
->info
.cs
.local_size
[2] = 1;
712 nir_variable
*input_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
714 input_img
->data
.descriptor_set
= 0;
715 input_img
->data
.binding
= 0;
717 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
718 img_type
, "out_img");
719 output_img
->data
.descriptor_set
= 0;
720 output_img
->data
.binding
= 1;
722 nir_ssa_def
*invoc_id
= nir_load_system_value(&b
, nir_intrinsic_load_local_invocation_id
, 0);
723 nir_ssa_def
*wg_id
= nir_load_system_value(&b
, nir_intrinsic_load_work_group_id
, 0);
724 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
725 b
.shader
->info
.cs
.local_size
[0],
726 b
.shader
->info
.cs
.local_size
[1],
727 b
.shader
->info
.cs
.local_size
[2], 0);
729 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
731 nir_intrinsic_instr
*src_offset
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
732 nir_intrinsic_set_base(src_offset
, 0);
733 nir_intrinsic_set_range(src_offset
, 24);
734 src_offset
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 0));
735 src_offset
->num_components
= is_3d
? 3 : 2;
736 nir_ssa_dest_init(&src_offset
->instr
, &src_offset
->dest
, is_3d
? 3 : 2, 32, "src_offset");
737 nir_builder_instr_insert(&b
, &src_offset
->instr
);
739 nir_intrinsic_instr
*dst_offset
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
740 nir_intrinsic_set_base(dst_offset
, 0);
741 nir_intrinsic_set_range(dst_offset
, 24);
742 dst_offset
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 12));
743 dst_offset
->num_components
= is_3d
? 3 : 2;
744 nir_ssa_dest_init(&dst_offset
->instr
, &dst_offset
->dest
, is_3d
? 3 : 2, 32, "dst_offset");
745 nir_builder_instr_insert(&b
, &dst_offset
->instr
);
747 nir_ssa_def
*src_coord
= nir_iadd(&b
, global_id
, &src_offset
->dest
.ssa
);
748 nir_ssa_def
*input_img_deref
= &nir_build_deref_var(&b
, input_img
)->dest
.ssa
;
750 nir_ssa_def
*dst_coord
= nir_iadd(&b
, global_id
, &dst_offset
->dest
.ssa
);
752 nir_tex_instr
*tex
= nir_tex_instr_create(b
.shader
, 3);
753 tex
->sampler_dim
= dim
;
754 tex
->op
= nir_texop_txf
;
755 tex
->src
[0].src_type
= nir_tex_src_coord
;
756 tex
->src
[0].src
= nir_src_for_ssa(nir_channels(&b
, src_coord
, is_3d
? 0x7 : 0x3));
757 tex
->src
[1].src_type
= nir_tex_src_lod
;
758 tex
->src
[1].src
= nir_src_for_ssa(nir_imm_int(&b
, 0));
759 tex
->src
[2].src_type
= nir_tex_src_texture_deref
;
760 tex
->src
[2].src
= nir_src_for_ssa(input_img_deref
);
761 tex
->dest_type
= nir_type_float
;
762 tex
->is_array
= false;
763 tex
->coord_components
= is_3d
? 3 : 2;
765 nir_ssa_dest_init(&tex
->instr
, &tex
->dest
, 4, 32, "tex");
766 nir_builder_instr_insert(&b
, &tex
->instr
);
768 nir_ssa_def
*outval
= &tex
->dest
.ssa
;
769 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
770 store
->num_components
= 4;
771 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
772 store
->src
[1] = nir_src_for_ssa(dst_coord
);
773 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
774 store
->src
[3] = nir_src_for_ssa(outval
);
776 nir_builder_instr_insert(&b
, &store
->instr
);
780 /* image to image - don't write use image accessors */
782 radv_device_init_meta_itoi_state(struct radv_device
*device
)
785 struct radv_shader_module cs
= { .nir
= NULL
};
786 struct radv_shader_module cs_3d
= { .nir
= NULL
};
787 cs
.nir
= build_nir_itoi_compute_shader(device
, false);
788 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
)
789 cs_3d
.nir
= build_nir_itoi_compute_shader(device
, true);
791 * two descriptors one for the image being sampled
792 * one for the buffer being written.
794 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
795 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
796 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
798 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
801 .descriptorType
= VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE
,
802 .descriptorCount
= 1,
803 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
804 .pImmutableSamplers
= NULL
808 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
809 .descriptorCount
= 1,
810 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
811 .pImmutableSamplers
= NULL
816 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
818 &device
->meta_state
.alloc
,
819 &device
->meta_state
.itoi
.img_ds_layout
);
820 if (result
!= VK_SUCCESS
)
824 VkPipelineLayoutCreateInfo pl_create_info
= {
825 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
827 .pSetLayouts
= &device
->meta_state
.itoi
.img_ds_layout
,
828 .pushConstantRangeCount
= 1,
829 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 24},
832 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
834 &device
->meta_state
.alloc
,
835 &device
->meta_state
.itoi
.img_p_layout
);
836 if (result
!= VK_SUCCESS
)
841 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
842 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
843 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
844 .module
= radv_shader_module_to_handle(&cs
),
846 .pSpecializationInfo
= NULL
,
849 VkComputePipelineCreateInfo vk_pipeline_info
= {
850 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
851 .stage
= pipeline_shader_stage
,
853 .layout
= device
->meta_state
.itoi
.img_p_layout
,
856 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
857 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
858 1, &vk_pipeline_info
, NULL
,
859 &device
->meta_state
.itoi
.pipeline
);
860 if (result
!= VK_SUCCESS
)
863 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
864 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d
= {
865 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
866 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
867 .module
= radv_shader_module_to_handle(&cs_3d
),
869 .pSpecializationInfo
= NULL
,
872 VkComputePipelineCreateInfo vk_pipeline_info_3d
= {
873 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
874 .stage
= pipeline_shader_stage_3d
,
876 .layout
= device
->meta_state
.itoi
.img_p_layout
,
879 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
880 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
881 1, &vk_pipeline_info_3d
, NULL
,
882 &device
->meta_state
.itoi
.pipeline_3d
);
884 ralloc_free(cs_3d
.nir
);
891 ralloc_free(cs_3d
.nir
);
896 radv_device_finish_meta_itoi_state(struct radv_device
*device
)
898 struct radv_meta_state
*state
= &device
->meta_state
;
900 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
901 state
->itoi
.img_p_layout
, &state
->alloc
);
902 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
903 state
->itoi
.img_ds_layout
,
905 radv_DestroyPipeline(radv_device_to_handle(device
),
906 state
->itoi
.pipeline
, &state
->alloc
);
907 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
)
908 radv_DestroyPipeline(radv_device_to_handle(device
),
909 state
->itoi
.pipeline_3d
, &state
->alloc
);
913 build_nir_cleari_compute_shader(struct radv_device
*dev
, bool is_3d
)
916 enum glsl_sampler_dim dim
= is_3d
? GLSL_SAMPLER_DIM_3D
: GLSL_SAMPLER_DIM_2D
;
917 const struct glsl_type
*img_type
= glsl_sampler_type(dim
,
921 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
922 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, is_3d
? "meta_cleari_cs_3d" : "meta_cleari_cs");
923 b
.shader
->info
.cs
.local_size
[0] = 16;
924 b
.shader
->info
.cs
.local_size
[1] = 16;
925 b
.shader
->info
.cs
.local_size
[2] = 1;
927 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
928 img_type
, "out_img");
929 output_img
->data
.descriptor_set
= 0;
930 output_img
->data
.binding
= 0;
932 nir_ssa_def
*invoc_id
= nir_load_system_value(&b
, nir_intrinsic_load_local_invocation_id
, 0);
933 nir_ssa_def
*wg_id
= nir_load_system_value(&b
, nir_intrinsic_load_work_group_id
, 0);
934 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
935 b
.shader
->info
.cs
.local_size
[0],
936 b
.shader
->info
.cs
.local_size
[1],
937 b
.shader
->info
.cs
.local_size
[2], 0);
939 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
941 nir_intrinsic_instr
*clear_val
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
942 nir_intrinsic_set_base(clear_val
, 0);
943 nir_intrinsic_set_range(clear_val
, 20);
944 clear_val
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 0));
945 clear_val
->num_components
= 4;
946 nir_ssa_dest_init(&clear_val
->instr
, &clear_val
->dest
, 4, 32, "clear_value");
947 nir_builder_instr_insert(&b
, &clear_val
->instr
);
949 nir_intrinsic_instr
*layer
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
950 nir_intrinsic_set_base(layer
, 0);
951 nir_intrinsic_set_range(layer
, 20);
952 layer
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 16));
953 layer
->num_components
= 1;
954 nir_ssa_dest_init(&layer
->instr
, &layer
->dest
, 1, 32, "layer");
955 nir_builder_instr_insert(&b
, &layer
->instr
);
957 nir_ssa_def
*global_z
= nir_iadd(&b
, nir_channel(&b
, global_id
, 2), &layer
->dest
.ssa
);
959 nir_ssa_def
*comps
[4];
960 comps
[0] = nir_channel(&b
, global_id
, 0);
961 comps
[1] = nir_channel(&b
, global_id
, 1);
963 comps
[3] = nir_imm_int(&b
, 0);
964 global_id
= nir_vec(&b
, comps
, 4);
966 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
967 store
->num_components
= 4;
968 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
969 store
->src
[1] = nir_src_for_ssa(global_id
);
970 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
971 store
->src
[3] = nir_src_for_ssa(&clear_val
->dest
.ssa
);
973 nir_builder_instr_insert(&b
, &store
->instr
);
978 radv_device_init_meta_cleari_state(struct radv_device
*device
)
981 struct radv_shader_module cs
= { .nir
= NULL
};
982 struct radv_shader_module cs_3d
= { .nir
= NULL
};
983 cs
.nir
= build_nir_cleari_compute_shader(device
, false);
984 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
)
985 cs_3d
.nir
= build_nir_cleari_compute_shader(device
, true);
988 * two descriptors one for the image being sampled
989 * one for the buffer being written.
991 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
992 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
993 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
995 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
998 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
999 .descriptorCount
= 1,
1000 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
1001 .pImmutableSamplers
= NULL
1006 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
1008 &device
->meta_state
.alloc
,
1009 &device
->meta_state
.cleari
.img_ds_layout
);
1010 if (result
!= VK_SUCCESS
)
1014 VkPipelineLayoutCreateInfo pl_create_info
= {
1015 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
1016 .setLayoutCount
= 1,
1017 .pSetLayouts
= &device
->meta_state
.cleari
.img_ds_layout
,
1018 .pushConstantRangeCount
= 1,
1019 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 20},
1022 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
1024 &device
->meta_state
.alloc
,
1025 &device
->meta_state
.cleari
.img_p_layout
);
1026 if (result
!= VK_SUCCESS
)
1029 /* compute shader */
1031 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
1032 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
1033 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
1034 .module
= radv_shader_module_to_handle(&cs
),
1036 .pSpecializationInfo
= NULL
,
1039 VkComputePipelineCreateInfo vk_pipeline_info
= {
1040 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
1041 .stage
= pipeline_shader_stage
,
1043 .layout
= device
->meta_state
.cleari
.img_p_layout
,
1046 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
1047 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
1048 1, &vk_pipeline_info
, NULL
,
1049 &device
->meta_state
.cleari
.pipeline
);
1050 if (result
!= VK_SUCCESS
)
1054 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
1055 /* compute shader */
1056 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d
= {
1057 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
1058 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
1059 .module
= radv_shader_module_to_handle(&cs_3d
),
1061 .pSpecializationInfo
= NULL
,
1064 VkComputePipelineCreateInfo vk_pipeline_info_3d
= {
1065 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
1066 .stage
= pipeline_shader_stage_3d
,
1068 .layout
= device
->meta_state
.cleari
.img_p_layout
,
1071 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
1072 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
1073 1, &vk_pipeline_info_3d
, NULL
,
1074 &device
->meta_state
.cleari
.pipeline_3d
);
1075 if (result
!= VK_SUCCESS
)
1078 ralloc_free(cs_3d
.nir
);
1080 ralloc_free(cs
.nir
);
1083 ralloc_free(cs
.nir
);
1084 ralloc_free(cs_3d
.nir
);
1089 radv_device_finish_meta_cleari_state(struct radv_device
*device
)
1091 struct radv_meta_state
*state
= &device
->meta_state
;
1093 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
1094 state
->cleari
.img_p_layout
, &state
->alloc
);
1095 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
1096 state
->cleari
.img_ds_layout
,
1098 radv_DestroyPipeline(radv_device_to_handle(device
),
1099 state
->cleari
.pipeline
, &state
->alloc
);
1100 radv_DestroyPipeline(radv_device_to_handle(device
),
1101 state
->cleari
.pipeline_3d
, &state
->alloc
);
1104 /* Special path for clearing R32G32B32 images using a compute shader. */
1106 build_nir_cleari_r32g32b32_compute_shader(struct radv_device
*dev
)
1109 const struct glsl_type
*img_type
= glsl_sampler_type(GLSL_SAMPLER_DIM_BUF
,
1113 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
1114 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, "meta_cleari_r32g32b32_cs");
1115 b
.shader
->info
.cs
.local_size
[0] = 16;
1116 b
.shader
->info
.cs
.local_size
[1] = 16;
1117 b
.shader
->info
.cs
.local_size
[2] = 1;
1119 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
1120 img_type
, "out_img");
1121 output_img
->data
.descriptor_set
= 0;
1122 output_img
->data
.binding
= 0;
1124 nir_ssa_def
*invoc_id
= nir_load_system_value(&b
, nir_intrinsic_load_local_invocation_id
, 0);
1125 nir_ssa_def
*wg_id
= nir_load_system_value(&b
, nir_intrinsic_load_work_group_id
, 0);
1126 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
1127 b
.shader
->info
.cs
.local_size
[0],
1128 b
.shader
->info
.cs
.local_size
[1],
1129 b
.shader
->info
.cs
.local_size
[2], 0);
1131 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
1133 nir_intrinsic_instr
*clear_val
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
1134 nir_intrinsic_set_base(clear_val
, 0);
1135 nir_intrinsic_set_range(clear_val
, 16);
1136 clear_val
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 0));
1137 clear_val
->num_components
= 3;
1138 nir_ssa_dest_init(&clear_val
->instr
, &clear_val
->dest
, 3, 32, "clear_value");
1139 nir_builder_instr_insert(&b
, &clear_val
->instr
);
1141 nir_intrinsic_instr
*stride
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_load_push_constant
);
1142 nir_intrinsic_set_base(stride
, 0);
1143 nir_intrinsic_set_range(stride
, 16);
1144 stride
->src
[0] = nir_src_for_ssa(nir_imm_int(&b
, 12));
1145 stride
->num_components
= 1;
1146 nir_ssa_dest_init(&stride
->instr
, &stride
->dest
, 1, 32, "stride");
1147 nir_builder_instr_insert(&b
, &stride
->instr
);
1149 nir_ssa_def
*global_x
= nir_channel(&b
, global_id
, 0);
1150 nir_ssa_def
*global_y
= nir_channel(&b
, global_id
, 1);
1152 nir_ssa_def
*global_pos
=
1154 nir_imul(&b
, global_y
, &stride
->dest
.ssa
),
1155 nir_imul(&b
, global_x
, nir_imm_int(&b
, 3)));
1157 for (unsigned chan
= 0; chan
< 3; chan
++) {
1158 nir_ssa_def
*local_pos
=
1159 nir_iadd(&b
, global_pos
, nir_imm_int(&b
, chan
));
1161 nir_ssa_def
*coord
=
1162 nir_vec4(&b
, local_pos
, local_pos
, local_pos
, local_pos
);
1164 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
1165 store
->num_components
= 1;
1166 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
1167 store
->src
[1] = nir_src_for_ssa(coord
);
1168 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
1169 store
->src
[3] = nir_src_for_ssa(nir_channel(&b
, &clear_val
->dest
.ssa
, chan
));
1170 nir_builder_instr_insert(&b
, &store
->instr
);
1177 radv_device_init_meta_cleari_r32g32b32_state(struct radv_device
*device
)
1180 struct radv_shader_module cs
= { .nir
= NULL
};
1182 cs
.nir
= build_nir_cleari_r32g32b32_compute_shader(device
);
1184 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
1185 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
1186 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
1188 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
1191 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
1192 .descriptorCount
= 1,
1193 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
1194 .pImmutableSamplers
= NULL
1199 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
1201 &device
->meta_state
.alloc
,
1202 &device
->meta_state
.cleari_r32g32b32
.img_ds_layout
);
1203 if (result
!= VK_SUCCESS
)
1206 VkPipelineLayoutCreateInfo pl_create_info
= {
1207 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
1208 .setLayoutCount
= 1,
1209 .pSetLayouts
= &device
->meta_state
.cleari_r32g32b32
.img_ds_layout
,
1210 .pushConstantRangeCount
= 1,
1211 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16},
1214 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
1216 &device
->meta_state
.alloc
,
1217 &device
->meta_state
.cleari_r32g32b32
.img_p_layout
);
1218 if (result
!= VK_SUCCESS
)
1221 /* compute shader */
1222 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
1223 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
1224 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
1225 .module
= radv_shader_module_to_handle(&cs
),
1227 .pSpecializationInfo
= NULL
,
1230 VkComputePipelineCreateInfo vk_pipeline_info
= {
1231 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
1232 .stage
= pipeline_shader_stage
,
1234 .layout
= device
->meta_state
.cleari_r32g32b32
.img_p_layout
,
1237 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
1238 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
1239 1, &vk_pipeline_info
, NULL
,
1240 &device
->meta_state
.cleari_r32g32b32
.pipeline
);
1243 ralloc_free(cs
.nir
);
1248 radv_device_finish_meta_cleari_r32g32b32_state(struct radv_device
*device
)
1250 struct radv_meta_state
*state
= &device
->meta_state
;
1252 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
1253 state
->cleari_r32g32b32
.img_p_layout
,
1255 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
1256 state
->cleari_r32g32b32
.img_ds_layout
,
1258 radv_DestroyPipeline(radv_device_to_handle(device
),
1259 state
->cleari_r32g32b32
.pipeline
, &state
->alloc
);
1263 radv_device_finish_meta_bufimage_state(struct radv_device
*device
)
1265 radv_device_finish_meta_itob_state(device
);
1266 radv_device_finish_meta_btoi_state(device
);
1267 radv_device_finish_meta_btoi_r32g32b32_state(device
);
1268 radv_device_finish_meta_itoi_state(device
);
1269 radv_device_finish_meta_cleari_state(device
);
1270 radv_device_finish_meta_cleari_r32g32b32_state(device
);
1274 radv_device_init_meta_bufimage_state(struct radv_device
*device
)
1278 result
= radv_device_init_meta_itob_state(device
);
1279 if (result
!= VK_SUCCESS
)
1282 result
= radv_device_init_meta_btoi_state(device
);
1283 if (result
!= VK_SUCCESS
)
1286 result
= radv_device_init_meta_btoi_r32g32b32_state(device
);
1287 if (result
!= VK_SUCCESS
)
1288 goto fail_btoi_r32g32b32
;
1290 result
= radv_device_init_meta_itoi_state(device
);
1291 if (result
!= VK_SUCCESS
)
1294 result
= radv_device_init_meta_cleari_state(device
);
1295 if (result
!= VK_SUCCESS
)
1298 result
= radv_device_init_meta_cleari_r32g32b32_state(device
);
1299 if (result
!= VK_SUCCESS
)
1300 goto fail_cleari_r32g32b32
;
1303 fail_cleari_r32g32b32
:
1304 radv_device_finish_meta_cleari_r32g32b32_state(device
);
1306 radv_device_finish_meta_cleari_state(device
);
1308 radv_device_finish_meta_itoi_state(device
);
1309 fail_btoi_r32g32b32
:
1310 radv_device_finish_meta_btoi_r32g32b32_state(device
);
1312 radv_device_finish_meta_btoi_state(device
);
1314 radv_device_finish_meta_itob_state(device
);
1319 create_iview(struct radv_cmd_buffer
*cmd_buffer
,
1320 struct radv_meta_blit2d_surf
*surf
,
1321 struct radv_image_view
*iview
)
1323 VkImageViewType view_type
= cmd_buffer
->device
->physical_device
->rad_info
.chip_class
< GFX9
? VK_IMAGE_VIEW_TYPE_2D
:
1324 radv_meta_get_view_type(surf
->image
);
1325 radv_image_view_init(iview
, cmd_buffer
->device
,
1326 &(VkImageViewCreateInfo
) {
1327 .sType
= VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO
,
1328 .image
= radv_image_to_handle(surf
->image
),
1329 .viewType
= view_type
,
1330 .format
= surf
->format
,
1331 .subresourceRange
= {
1332 .aspectMask
= surf
->aspect_mask
,
1333 .baseMipLevel
= surf
->level
,
1335 .baseArrayLayer
= surf
->layer
,
1342 create_bview(struct radv_cmd_buffer
*cmd_buffer
,
1343 struct radv_buffer
*buffer
,
1346 struct radv_buffer_view
*bview
)
1348 radv_buffer_view_init(bview
, cmd_buffer
->device
,
1349 &(VkBufferViewCreateInfo
) {
1350 .sType
= VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO
,
1352 .buffer
= radv_buffer_to_handle(buffer
),
1355 .range
= VK_WHOLE_SIZE
,
1361 create_buffer_from_image(struct radv_cmd_buffer
*cmd_buffer
,
1362 struct radv_meta_blit2d_surf
*surf
,
1363 VkBufferUsageFlagBits usage
,
1366 struct radv_device
*device
= cmd_buffer
->device
;
1367 struct radv_device_memory mem
= { .bo
= surf
->image
->bo
};
1369 radv_CreateBuffer(radv_device_to_handle(device
),
1370 &(VkBufferCreateInfo
) {
1371 .sType
= VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO
,
1373 .size
= surf
->image
->size
,
1375 .sharingMode
= VK_SHARING_MODE_EXCLUSIVE
,
1378 radv_BindBufferMemory2(radv_device_to_handle(device
), 1,
1379 (VkBindBufferMemoryInfoKHR
[]) {
1381 .sType
= VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO
,
1383 .memory
= radv_device_memory_to_handle(&mem
),
1384 .memoryOffset
= surf
->image
->offset
,
1390 create_bview_for_r32g32b32(struct radv_cmd_buffer
*cmd_buffer
,
1391 struct radv_buffer
*buffer
,
1393 VkFormat src_format
,
1394 struct radv_buffer_view
*bview
)
1398 switch (src_format
) {
1399 case VK_FORMAT_R32G32B32_UINT
:
1400 format
= VK_FORMAT_R32_UINT
;
1402 case VK_FORMAT_R32G32B32_SINT
:
1403 format
= VK_FORMAT_R32_SINT
;
1405 case VK_FORMAT_R32G32B32_SFLOAT
:
1406 format
= VK_FORMAT_R32_SFLOAT
;
1409 unreachable("invalid R32G32B32 format");
1412 radv_buffer_view_init(bview
, cmd_buffer
->device
,
1413 &(VkBufferViewCreateInfo
) {
1414 .sType
= VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO
,
1416 .buffer
= radv_buffer_to_handle(buffer
),
1419 .range
= VK_WHOLE_SIZE
,
1424 itob_bind_descriptors(struct radv_cmd_buffer
*cmd_buffer
,
1425 struct radv_image_view
*src
,
1426 struct radv_buffer_view
*dst
)
1428 struct radv_device
*device
= cmd_buffer
->device
;
1430 radv_meta_push_descriptor_set(cmd_buffer
,
1431 VK_PIPELINE_BIND_POINT_COMPUTE
,
1432 device
->meta_state
.itob
.img_p_layout
,
1434 2, /* descriptorWriteCount */
1435 (VkWriteDescriptorSet
[]) {
1437 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1439 .dstArrayElement
= 0,
1440 .descriptorCount
= 1,
1441 .descriptorType
= VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE
,
1442 .pImageInfo
= (VkDescriptorImageInfo
[]) {
1444 .sampler
= VK_NULL_HANDLE
,
1445 .imageView
= radv_image_view_to_handle(src
),
1446 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
1451 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1453 .dstArrayElement
= 0,
1454 .descriptorCount
= 1,
1455 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
1456 .pTexelBufferView
= (VkBufferView
[]) { radv_buffer_view_to_handle(dst
) },
1462 radv_meta_image_to_buffer(struct radv_cmd_buffer
*cmd_buffer
,
1463 struct radv_meta_blit2d_surf
*src
,
1464 struct radv_meta_blit2d_buffer
*dst
,
1466 struct radv_meta_blit2d_rect
*rects
)
1468 VkPipeline pipeline
= cmd_buffer
->device
->meta_state
.itob
.pipeline
;
1469 struct radv_device
*device
= cmd_buffer
->device
;
1470 struct radv_image_view src_view
;
1471 struct radv_buffer_view dst_view
;
1473 create_iview(cmd_buffer
, src
, &src_view
);
1474 create_bview(cmd_buffer
, dst
->buffer
, dst
->offset
, dst
->format
, &dst_view
);
1475 itob_bind_descriptors(cmd_buffer
, &src_view
, &dst_view
);
1477 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
&&
1478 src
->image
->type
== VK_IMAGE_TYPE_3D
)
1479 pipeline
= cmd_buffer
->device
->meta_state
.itob
.pipeline_3d
;
1481 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
1482 VK_PIPELINE_BIND_POINT_COMPUTE
, pipeline
);
1484 for (unsigned r
= 0; r
< num_rects
; ++r
) {
1485 unsigned push_constants
[4] = {
1491 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer
),
1492 device
->meta_state
.itob
.img_p_layout
,
1493 VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16,
1496 radv_unaligned_dispatch(cmd_buffer
, rects
[r
].width
, rects
[r
].height
, 1);
1501 btoi_r32g32b32_bind_descriptors(struct radv_cmd_buffer
*cmd_buffer
,
1502 struct radv_buffer_view
*src
,
1503 struct radv_buffer_view
*dst
)
1505 struct radv_device
*device
= cmd_buffer
->device
;
1507 radv_meta_push_descriptor_set(cmd_buffer
,
1508 VK_PIPELINE_BIND_POINT_COMPUTE
,
1509 device
->meta_state
.btoi_r32g32b32
.img_p_layout
,
1511 2, /* descriptorWriteCount */
1512 (VkWriteDescriptorSet
[]) {
1514 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1516 .dstArrayElement
= 0,
1517 .descriptorCount
= 1,
1518 .descriptorType
= VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER
,
1519 .pTexelBufferView
= (VkBufferView
[]) { radv_buffer_view_to_handle(src
) },
1522 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1524 .dstArrayElement
= 0,
1525 .descriptorCount
= 1,
1526 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
1527 .pTexelBufferView
= (VkBufferView
[]) { radv_buffer_view_to_handle(dst
) },
1533 radv_meta_buffer_to_image_cs_r32g32b32(struct radv_cmd_buffer
*cmd_buffer
,
1534 struct radv_meta_blit2d_buffer
*src
,
1535 struct radv_meta_blit2d_surf
*dst
,
1537 struct radv_meta_blit2d_rect
*rects
)
1539 VkPipeline pipeline
= cmd_buffer
->device
->meta_state
.btoi_r32g32b32
.pipeline
;
1540 struct radv_device
*device
= cmd_buffer
->device
;
1541 struct radv_buffer_view src_view
, dst_view
;
1542 unsigned dst_offset
= 0;
1546 /* This special btoi path for R32G32B32 formats will write the linear
1547 * image as a buffer with the same underlying memory. The compute
1548 * shader will clear all components separately using a R32 format.
1550 create_buffer_from_image(cmd_buffer
, dst
,
1551 VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT
,
1554 create_bview(cmd_buffer
, src
->buffer
, src
->offset
,
1555 src
->format
, &src_view
);
1556 create_bview_for_r32g32b32(cmd_buffer
, radv_buffer_from_handle(buffer
),
1557 dst_offset
, dst
->format
, &dst_view
);
1558 btoi_r32g32b32_bind_descriptors(cmd_buffer
, &src_view
, &dst_view
);
1560 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
1561 VK_PIPELINE_BIND_POINT_COMPUTE
, pipeline
);
1563 if (cmd_buffer
->device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
1564 stride
= dst
->image
->surface
.u
.gfx9
.surf_pitch
;
1566 stride
= dst
->image
->surface
.u
.legacy
.level
[0].nblk_x
* 3;
1569 for (unsigned r
= 0; r
< num_rects
; ++r
) {
1570 unsigned push_constants
[4] = {
1577 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer
),
1578 device
->meta_state
.btoi_r32g32b32
.img_p_layout
,
1579 VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16,
1582 radv_unaligned_dispatch(cmd_buffer
, rects
[r
].width
, rects
[r
].height
, 1);
1585 radv_DestroyBuffer(radv_device_to_handle(device
), buffer
, NULL
);
1589 btoi_bind_descriptors(struct radv_cmd_buffer
*cmd_buffer
,
1590 struct radv_buffer_view
*src
,
1591 struct radv_image_view
*dst
)
1593 struct radv_device
*device
= cmd_buffer
->device
;
1595 radv_meta_push_descriptor_set(cmd_buffer
,
1596 VK_PIPELINE_BIND_POINT_COMPUTE
,
1597 device
->meta_state
.btoi
.img_p_layout
,
1599 2, /* descriptorWriteCount */
1600 (VkWriteDescriptorSet
[]) {
1602 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1604 .dstArrayElement
= 0,
1605 .descriptorCount
= 1,
1606 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
1607 .pTexelBufferView
= (VkBufferView
[]) { radv_buffer_view_to_handle(src
) },
1610 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1612 .dstArrayElement
= 0,
1613 .descriptorCount
= 1,
1614 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
1615 .pImageInfo
= (VkDescriptorImageInfo
[]) {
1617 .sampler
= VK_NULL_HANDLE
,
1618 .imageView
= radv_image_view_to_handle(dst
),
1619 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
1627 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer
*cmd_buffer
,
1628 struct radv_meta_blit2d_buffer
*src
,
1629 struct radv_meta_blit2d_surf
*dst
,
1631 struct radv_meta_blit2d_rect
*rects
)
1633 VkPipeline pipeline
= cmd_buffer
->device
->meta_state
.btoi
.pipeline
;
1634 struct radv_device
*device
= cmd_buffer
->device
;
1635 struct radv_buffer_view src_view
;
1636 struct radv_image_view dst_view
;
1638 if (dst
->image
->vk_format
== VK_FORMAT_R32G32B32_UINT
||
1639 dst
->image
->vk_format
== VK_FORMAT_R32G32B32_SINT
||
1640 dst
->image
->vk_format
== VK_FORMAT_R32G32B32_SFLOAT
) {
1641 radv_meta_buffer_to_image_cs_r32g32b32(cmd_buffer
, src
, dst
,
1646 create_bview(cmd_buffer
, src
->buffer
, src
->offset
, src
->format
, &src_view
);
1647 create_iview(cmd_buffer
, dst
, &dst_view
);
1648 btoi_bind_descriptors(cmd_buffer
, &src_view
, &dst_view
);
1650 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
&&
1651 dst
->image
->type
== VK_IMAGE_TYPE_3D
)
1652 pipeline
= cmd_buffer
->device
->meta_state
.btoi
.pipeline_3d
;
1653 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
1654 VK_PIPELINE_BIND_POINT_COMPUTE
, pipeline
);
1656 for (unsigned r
= 0; r
< num_rects
; ++r
) {
1657 unsigned push_constants
[4] = {
1663 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer
),
1664 device
->meta_state
.btoi
.img_p_layout
,
1665 VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16,
1668 radv_unaligned_dispatch(cmd_buffer
, rects
[r
].width
, rects
[r
].height
, 1);
1673 itoi_bind_descriptors(struct radv_cmd_buffer
*cmd_buffer
,
1674 struct radv_image_view
*src
,
1675 struct radv_image_view
*dst
)
1677 struct radv_device
*device
= cmd_buffer
->device
;
1679 radv_meta_push_descriptor_set(cmd_buffer
,
1680 VK_PIPELINE_BIND_POINT_COMPUTE
,
1681 device
->meta_state
.itoi
.img_p_layout
,
1683 2, /* descriptorWriteCount */
1684 (VkWriteDescriptorSet
[]) {
1686 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1688 .dstArrayElement
= 0,
1689 .descriptorCount
= 1,
1690 .descriptorType
= VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE
,
1691 .pImageInfo
= (VkDescriptorImageInfo
[]) {
1693 .sampler
= VK_NULL_HANDLE
,
1694 .imageView
= radv_image_view_to_handle(src
),
1695 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
1700 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1702 .dstArrayElement
= 0,
1703 .descriptorCount
= 1,
1704 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
1705 .pImageInfo
= (VkDescriptorImageInfo
[]) {
1707 .sampler
= VK_NULL_HANDLE
,
1708 .imageView
= radv_image_view_to_handle(dst
),
1709 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
1717 radv_meta_image_to_image_cs(struct radv_cmd_buffer
*cmd_buffer
,
1718 struct radv_meta_blit2d_surf
*src
,
1719 struct radv_meta_blit2d_surf
*dst
,
1721 struct radv_meta_blit2d_rect
*rects
)
1723 VkPipeline pipeline
= cmd_buffer
->device
->meta_state
.itoi
.pipeline
;
1724 struct radv_device
*device
= cmd_buffer
->device
;
1725 struct radv_image_view src_view
, dst_view
;
1727 create_iview(cmd_buffer
, src
, &src_view
);
1728 create_iview(cmd_buffer
, dst
, &dst_view
);
1730 itoi_bind_descriptors(cmd_buffer
, &src_view
, &dst_view
);
1732 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
&&
1733 src
->image
->type
== VK_IMAGE_TYPE_3D
)
1734 pipeline
= cmd_buffer
->device
->meta_state
.itoi
.pipeline_3d
;
1735 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
1736 VK_PIPELINE_BIND_POINT_COMPUTE
, pipeline
);
1738 for (unsigned r
= 0; r
< num_rects
; ++r
) {
1739 unsigned push_constants
[6] = {
1747 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer
),
1748 device
->meta_state
.itoi
.img_p_layout
,
1749 VK_SHADER_STAGE_COMPUTE_BIT
, 0, 24,
1752 radv_unaligned_dispatch(cmd_buffer
, rects
[r
].width
, rects
[r
].height
, 1);
1757 cleari_r32g32b32_bind_descriptors(struct radv_cmd_buffer
*cmd_buffer
,
1758 struct radv_buffer_view
*view
)
1760 struct radv_device
*device
= cmd_buffer
->device
;
1762 radv_meta_push_descriptor_set(cmd_buffer
,
1763 VK_PIPELINE_BIND_POINT_COMPUTE
,
1764 device
->meta_state
.cleari_r32g32b32
.img_p_layout
,
1766 1, /* descriptorWriteCount */
1767 (VkWriteDescriptorSet
[]) {
1769 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1771 .dstArrayElement
= 0,
1772 .descriptorCount
= 1,
1773 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
,
1774 .pTexelBufferView
= (VkBufferView
[]) { radv_buffer_view_to_handle(view
) },
1780 radv_meta_clear_image_cs_r32g32b32(struct radv_cmd_buffer
*cmd_buffer
,
1781 struct radv_meta_blit2d_surf
*dst
,
1782 const VkClearColorValue
*clear_color
)
1784 VkPipeline pipeline
= cmd_buffer
->device
->meta_state
.cleari_r32g32b32
.pipeline
;
1785 struct radv_device
*device
= cmd_buffer
->device
;
1786 struct radv_buffer_view dst_view
;
1790 /* This special clear path for R32G32B32 formats will write the linear
1791 * image as a buffer with the same underlying memory. The compute
1792 * shader will clear all components separately using a R32 format.
1794 create_buffer_from_image(cmd_buffer
, dst
,
1795 VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT
,
1798 create_bview_for_r32g32b32(cmd_buffer
, radv_buffer_from_handle(buffer
),
1799 0, dst
->format
, &dst_view
);
1800 cleari_r32g32b32_bind_descriptors(cmd_buffer
, &dst_view
);
1802 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
1803 VK_PIPELINE_BIND_POINT_COMPUTE
, pipeline
);
1805 if (cmd_buffer
->device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
1806 stride
= dst
->image
->surface
.u
.gfx9
.surf_pitch
;
1808 stride
= dst
->image
->surface
.u
.legacy
.level
[0].nblk_x
* 3;
1811 unsigned push_constants
[4] = {
1812 clear_color
->uint32
[0],
1813 clear_color
->uint32
[1],
1814 clear_color
->uint32
[2],
1818 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer
),
1819 device
->meta_state
.cleari_r32g32b32
.img_p_layout
,
1820 VK_SHADER_STAGE_COMPUTE_BIT
, 0, 16,
1823 radv_unaligned_dispatch(cmd_buffer
, dst
->image
->info
.width
,
1824 dst
->image
->info
.height
, 1);
1826 radv_DestroyBuffer(radv_device_to_handle(device
), buffer
, NULL
);
1830 cleari_bind_descriptors(struct radv_cmd_buffer
*cmd_buffer
,
1831 struct radv_image_view
*dst_iview
)
1833 struct radv_device
*device
= cmd_buffer
->device
;
1835 radv_meta_push_descriptor_set(cmd_buffer
,
1836 VK_PIPELINE_BIND_POINT_COMPUTE
,
1837 device
->meta_state
.cleari
.img_p_layout
,
1839 1, /* descriptorWriteCount */
1840 (VkWriteDescriptorSet
[]) {
1842 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
1844 .dstArrayElement
= 0,
1845 .descriptorCount
= 1,
1846 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
1847 .pImageInfo
= (VkDescriptorImageInfo
[]) {
1849 .sampler
= VK_NULL_HANDLE
,
1850 .imageView
= radv_image_view_to_handle(dst_iview
),
1851 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
1859 radv_meta_clear_image_cs(struct radv_cmd_buffer
*cmd_buffer
,
1860 struct radv_meta_blit2d_surf
*dst
,
1861 const VkClearColorValue
*clear_color
)
1863 VkPipeline pipeline
= cmd_buffer
->device
->meta_state
.cleari
.pipeline
;
1864 struct radv_device
*device
= cmd_buffer
->device
;
1865 struct radv_image_view dst_iview
;
1867 if (dst
->format
== VK_FORMAT_R32G32B32_UINT
||
1868 dst
->format
== VK_FORMAT_R32G32B32_SINT
||
1869 dst
->format
== VK_FORMAT_R32G32B32_SFLOAT
) {
1870 radv_meta_clear_image_cs_r32g32b32(cmd_buffer
, dst
, clear_color
);
1874 create_iview(cmd_buffer
, dst
, &dst_iview
);
1875 cleari_bind_descriptors(cmd_buffer
, &dst_iview
);
1877 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
&&
1878 dst
->image
->type
== VK_IMAGE_TYPE_3D
)
1879 pipeline
= cmd_buffer
->device
->meta_state
.cleari
.pipeline_3d
;
1881 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
1882 VK_PIPELINE_BIND_POINT_COMPUTE
, pipeline
);
1884 unsigned push_constants
[5] = {
1885 clear_color
->uint32
[0],
1886 clear_color
->uint32
[1],
1887 clear_color
->uint32
[2],
1888 clear_color
->uint32
[3],
1892 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer
),
1893 device
->meta_state
.cleari
.img_p_layout
,
1894 VK_SHADER_STAGE_COMPUTE_BIT
, 0, 20,
1897 radv_unaligned_dispatch(cmd_buffer
, dst
->image
->info
.width
, dst
->image
->info
.height
, 1);