2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 #include "radv_meta.h"
28 #include "radv_private.h"
33 build_dcc_decompress_compute_shader(struct radv_device
*dev
)
36 const struct glsl_type
*buf_type
= glsl_sampler_type(GLSL_SAMPLER_DIM_2D
,
40 const struct glsl_type
*img_type
= glsl_image_type(GLSL_SAMPLER_DIM_2D
,
43 nir_builder_init_simple_shader(&b
, NULL
, MESA_SHADER_COMPUTE
, NULL
);
44 b
.shader
->info
.name
= ralloc_strdup(b
.shader
, "dcc_decompress_compute");
46 /* We need at least 16/16/1 to cover an entire DCC block in a single workgroup. */
47 b
.shader
->info
.cs
.local_size
[0] = 16;
48 b
.shader
->info
.cs
.local_size
[1] = 16;
49 b
.shader
->info
.cs
.local_size
[2] = 1;
50 nir_variable
*input_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
52 input_img
->data
.descriptor_set
= 0;
53 input_img
->data
.binding
= 0;
55 nir_variable
*output_img
= nir_variable_create(b
.shader
, nir_var_uniform
,
57 output_img
->data
.descriptor_set
= 0;
58 output_img
->data
.binding
= 1;
60 nir_ssa_def
*invoc_id
= nir_load_local_invocation_id(&b
);
61 nir_ssa_def
*wg_id
= nir_load_work_group_id(&b
);
62 nir_ssa_def
*block_size
= nir_imm_ivec4(&b
,
63 b
.shader
->info
.cs
.local_size
[0],
64 b
.shader
->info
.cs
.local_size
[1],
65 b
.shader
->info
.cs
.local_size
[2], 0);
67 nir_ssa_def
*global_id
= nir_iadd(&b
, nir_imul(&b
, wg_id
, block_size
), invoc_id
);
68 nir_ssa_def
*input_img_deref
= &nir_build_deref_var(&b
, input_img
)->dest
.ssa
;
70 nir_tex_instr
*tex
= nir_tex_instr_create(b
.shader
, 3);
71 tex
->sampler_dim
= GLSL_SAMPLER_DIM_2D
;
72 tex
->op
= nir_texop_txf
;
73 tex
->src
[0].src_type
= nir_tex_src_coord
;
74 tex
->src
[0].src
= nir_src_for_ssa(nir_channels(&b
, global_id
, 3));
75 tex
->src
[1].src_type
= nir_tex_src_lod
;
76 tex
->src
[1].src
= nir_src_for_ssa(nir_imm_int(&b
, 0));
77 tex
->src
[2].src_type
= nir_tex_src_texture_deref
;
78 tex
->src
[2].src
= nir_src_for_ssa(input_img_deref
);
79 tex
->dest_type
= nir_type_float
;
80 tex
->is_array
= false;
81 tex
->coord_components
= 2;
83 nir_ssa_dest_init(&tex
->instr
, &tex
->dest
, 4, 32, "tex");
84 nir_builder_instr_insert(&b
, &tex
->instr
);
86 nir_scoped_barrier(&b
, NIR_SCOPE_WORKGROUP
, NIR_SCOPE_WORKGROUP
,
87 NIR_MEMORY_ACQ_REL
, nir_var_mem_ssbo
);
89 nir_ssa_def
*outval
= &tex
->dest
.ssa
;
90 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(b
.shader
, nir_intrinsic_image_deref_store
);
91 store
->num_components
= 4;
92 store
->src
[0] = nir_src_for_ssa(&nir_build_deref_var(&b
, output_img
)->dest
.ssa
);
93 store
->src
[1] = nir_src_for_ssa(global_id
);
94 store
->src
[2] = nir_src_for_ssa(nir_ssa_undef(&b
, 1, 32));
95 store
->src
[3] = nir_src_for_ssa(outval
);
96 store
->src
[4] = nir_src_for_ssa(nir_imm_int(&b
, 0));
98 nir_builder_instr_insert(&b
, &store
->instr
);
103 create_dcc_compress_compute(struct radv_device
*device
)
105 VkResult result
= VK_SUCCESS
;
106 struct radv_shader_module cs
= { .nir
= NULL
};
108 cs
.nir
= build_dcc_decompress_compute_shader(device
);
110 VkDescriptorSetLayoutCreateInfo ds_create_info
= {
111 .sType
= VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
,
112 .flags
= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR
,
114 .pBindings
= (VkDescriptorSetLayoutBinding
[]) {
117 .descriptorType
= VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE
,
118 .descriptorCount
= 1,
119 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
120 .pImmutableSamplers
= NULL
124 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
125 .descriptorCount
= 1,
126 .stageFlags
= VK_SHADER_STAGE_COMPUTE_BIT
,
127 .pImmutableSamplers
= NULL
132 result
= radv_CreateDescriptorSetLayout(radv_device_to_handle(device
),
134 &device
->meta_state
.alloc
,
135 &device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_ds_layout
);
136 if (result
!= VK_SUCCESS
)
140 VkPipelineLayoutCreateInfo pl_create_info
= {
141 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
143 .pSetLayouts
= &device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_ds_layout
,
144 .pushConstantRangeCount
= 1,
145 .pPushConstantRanges
= &(VkPushConstantRange
){VK_SHADER_STAGE_COMPUTE_BIT
, 0, 8},
148 result
= radv_CreatePipelineLayout(radv_device_to_handle(device
),
150 &device
->meta_state
.alloc
,
151 &device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_p_layout
);
152 if (result
!= VK_SUCCESS
)
157 VkPipelineShaderStageCreateInfo pipeline_shader_stage
= {
158 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
159 .stage
= VK_SHADER_STAGE_COMPUTE_BIT
,
160 .module
= radv_shader_module_to_handle(&cs
),
162 .pSpecializationInfo
= NULL
,
165 VkComputePipelineCreateInfo vk_pipeline_info
= {
166 .sType
= VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
,
167 .stage
= pipeline_shader_stage
,
169 .layout
= device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_p_layout
,
172 result
= radv_CreateComputePipelines(radv_device_to_handle(device
),
173 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
174 1, &vk_pipeline_info
, NULL
,
175 &device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_pipeline
);
176 if (result
!= VK_SUCCESS
)
185 create_pass(struct radv_device
*device
)
188 VkDevice device_h
= radv_device_to_handle(device
);
189 const VkAllocationCallbacks
*alloc
= &device
->meta_state
.alloc
;
190 VkAttachmentDescription attachment
;
192 attachment
.format
= VK_FORMAT_UNDEFINED
;
193 attachment
.samples
= 1;
194 attachment
.loadOp
= VK_ATTACHMENT_LOAD_OP_LOAD
;
195 attachment
.storeOp
= VK_ATTACHMENT_STORE_OP_STORE
;
196 attachment
.initialLayout
= VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
;
197 attachment
.finalLayout
= VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
;
199 result
= radv_CreateRenderPass(device_h
,
200 &(VkRenderPassCreateInfo
) {
201 .sType
= VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO
,
202 .attachmentCount
= 1,
203 .pAttachments
= &attachment
,
205 .pSubpasses
= &(VkSubpassDescription
) {
206 .pipelineBindPoint
= VK_PIPELINE_BIND_POINT_GRAPHICS
,
207 .inputAttachmentCount
= 0,
208 .colorAttachmentCount
= 1,
209 .pColorAttachments
= (VkAttachmentReference
[]) {
212 .layout
= VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
,
215 .pResolveAttachments
= NULL
,
216 .pDepthStencilAttachment
= &(VkAttachmentReference
) {
217 .attachment
= VK_ATTACHMENT_UNUSED
,
219 .preserveAttachmentCount
= 0,
220 .pPreserveAttachments
= NULL
,
222 .dependencyCount
= 2,
223 .pDependencies
= (VkSubpassDependency
[]) {
225 .srcSubpass
= VK_SUBPASS_EXTERNAL
,
227 .srcStageMask
= VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT
,
228 .dstStageMask
= VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT
,
235 .dstSubpass
= VK_SUBPASS_EXTERNAL
,
236 .srcStageMask
= VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT
,
237 .dstStageMask
= VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT
,
245 &device
->meta_state
.fast_clear_flush
.pass
);
251 create_pipeline_layout(struct radv_device
*device
, VkPipelineLayout
*layout
)
253 VkPipelineLayoutCreateInfo pl_create_info
= {
254 .sType
= VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
,
257 .pushConstantRangeCount
= 0,
258 .pPushConstantRanges
= NULL
,
261 return radv_CreatePipelineLayout(radv_device_to_handle(device
),
263 &device
->meta_state
.alloc
,
268 create_pipeline(struct radv_device
*device
,
269 VkShaderModule vs_module_h
,
270 VkPipelineLayout layout
)
273 VkDevice device_h
= radv_device_to_handle(device
);
275 struct radv_shader_module fs_module
= {
276 .nir
= radv_meta_build_nir_fs_noop(),
279 if (!fs_module
.nir
) {
280 /* XXX: Need more accurate error */
281 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
285 const VkPipelineShaderStageCreateInfo stages
[2] = {
287 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
288 .stage
= VK_SHADER_STAGE_VERTEX_BIT
,
289 .module
= vs_module_h
,
293 .sType
= VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
,
294 .stage
= VK_SHADER_STAGE_FRAGMENT_BIT
,
295 .module
= radv_shader_module_to_handle(&fs_module
),
300 const VkPipelineVertexInputStateCreateInfo vi_state
= {
301 .sType
= VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO
,
302 .vertexBindingDescriptionCount
= 0,
303 .vertexAttributeDescriptionCount
= 0,
306 const VkPipelineInputAssemblyStateCreateInfo ia_state
= {
307 .sType
= VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO
,
308 .topology
= VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP
,
309 .primitiveRestartEnable
= false,
312 const VkPipelineColorBlendStateCreateInfo blend_state
= {
313 .sType
= VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO
,
314 .logicOpEnable
= false,
315 .attachmentCount
= 1,
316 .pAttachments
= (VkPipelineColorBlendAttachmentState
[]) {
318 .colorWriteMask
= VK_COLOR_COMPONENT_R_BIT
|
319 VK_COLOR_COMPONENT_G_BIT
|
320 VK_COLOR_COMPONENT_B_BIT
|
321 VK_COLOR_COMPONENT_A_BIT
,
325 const VkPipelineRasterizationStateCreateInfo rs_state
= {
326 .sType
= VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO
,
327 .depthClampEnable
= false,
328 .rasterizerDiscardEnable
= false,
329 .polygonMode
= VK_POLYGON_MODE_FILL
,
330 .cullMode
= VK_CULL_MODE_NONE
,
331 .frontFace
= VK_FRONT_FACE_COUNTER_CLOCKWISE
,
334 result
= radv_graphics_pipeline_create(device_h
,
335 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
336 &(VkGraphicsPipelineCreateInfo
) {
337 .sType
= VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO
,
341 .pVertexInputState
= &vi_state
,
342 .pInputAssemblyState
= &ia_state
,
344 .pViewportState
= &(VkPipelineViewportStateCreateInfo
) {
345 .sType
= VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO
,
349 .pRasterizationState
= &rs_state
,
350 .pMultisampleState
= &(VkPipelineMultisampleStateCreateInfo
) {
351 .sType
= VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO
,
352 .rasterizationSamples
= 1,
353 .sampleShadingEnable
= false,
355 .alphaToCoverageEnable
= false,
356 .alphaToOneEnable
= false,
358 .pColorBlendState
= &blend_state
,
359 .pDynamicState
= &(VkPipelineDynamicStateCreateInfo
) {
360 .sType
= VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO
,
361 .dynamicStateCount
= 2,
362 .pDynamicStates
= (VkDynamicState
[]) {
363 VK_DYNAMIC_STATE_VIEWPORT
,
364 VK_DYNAMIC_STATE_SCISSOR
,
368 .renderPass
= device
->meta_state
.fast_clear_flush
.pass
,
371 &(struct radv_graphics_pipeline_create_info
) {
372 .use_rectlist
= true,
373 .custom_blend_mode
= V_028808_CB_ELIMINATE_FAST_CLEAR
,
375 &device
->meta_state
.alloc
,
376 &device
->meta_state
.fast_clear_flush
.cmask_eliminate_pipeline
);
377 if (result
!= VK_SUCCESS
)
380 result
= radv_graphics_pipeline_create(device_h
,
381 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
382 &(VkGraphicsPipelineCreateInfo
) {
383 .sType
= VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO
,
387 .pVertexInputState
= &vi_state
,
388 .pInputAssemblyState
= &ia_state
,
390 .pViewportState
= &(VkPipelineViewportStateCreateInfo
) {
391 .sType
= VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO
,
395 .pRasterizationState
= &rs_state
,
396 .pMultisampleState
= &(VkPipelineMultisampleStateCreateInfo
) {
397 .sType
= VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO
,
398 .rasterizationSamples
= 1,
399 .sampleShadingEnable
= false,
401 .alphaToCoverageEnable
= false,
402 .alphaToOneEnable
= false,
404 .pColorBlendState
= &blend_state
,
405 .pDynamicState
= &(VkPipelineDynamicStateCreateInfo
) {
406 .sType
= VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO
,
407 .dynamicStateCount
= 2,
408 .pDynamicStates
= (VkDynamicState
[]) {
409 VK_DYNAMIC_STATE_VIEWPORT
,
410 VK_DYNAMIC_STATE_SCISSOR
,
414 .renderPass
= device
->meta_state
.fast_clear_flush
.pass
,
417 &(struct radv_graphics_pipeline_create_info
) {
418 .use_rectlist
= true,
419 .custom_blend_mode
= V_028808_CB_FMASK_DECOMPRESS
,
421 &device
->meta_state
.alloc
,
422 &device
->meta_state
.fast_clear_flush
.fmask_decompress_pipeline
);
423 if (result
!= VK_SUCCESS
)
426 result
= radv_graphics_pipeline_create(device_h
,
427 radv_pipeline_cache_to_handle(&device
->meta_state
.cache
),
428 &(VkGraphicsPipelineCreateInfo
) {
429 .sType
= VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO
,
433 .pVertexInputState
= &vi_state
,
434 .pInputAssemblyState
= &ia_state
,
436 .pViewportState
= &(VkPipelineViewportStateCreateInfo
) {
437 .sType
= VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO
,
441 .pRasterizationState
= &rs_state
,
442 .pMultisampleState
= &(VkPipelineMultisampleStateCreateInfo
) {
443 .sType
= VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO
,
444 .rasterizationSamples
= 1,
445 .sampleShadingEnable
= false,
447 .alphaToCoverageEnable
= false,
448 .alphaToOneEnable
= false,
450 .pColorBlendState
= &blend_state
,
451 .pDynamicState
= &(VkPipelineDynamicStateCreateInfo
) {
452 .sType
= VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO
,
453 .dynamicStateCount
= 2,
454 .pDynamicStates
= (VkDynamicState
[]) {
455 VK_DYNAMIC_STATE_VIEWPORT
,
456 VK_DYNAMIC_STATE_SCISSOR
,
460 .renderPass
= device
->meta_state
.fast_clear_flush
.pass
,
463 &(struct radv_graphics_pipeline_create_info
) {
464 .use_rectlist
= true,
465 .custom_blend_mode
= V_028808_CB_DCC_DECOMPRESS
,
467 &device
->meta_state
.alloc
,
468 &device
->meta_state
.fast_clear_flush
.dcc_decompress_pipeline
);
469 if (result
!= VK_SUCCESS
)
475 ralloc_free(fs_module
.nir
);
480 radv_device_finish_meta_fast_clear_flush_state(struct radv_device
*device
)
482 struct radv_meta_state
*state
= &device
->meta_state
;
484 radv_DestroyPipeline(radv_device_to_handle(device
),
485 state
->fast_clear_flush
.dcc_decompress_pipeline
,
487 radv_DestroyPipeline(radv_device_to_handle(device
),
488 state
->fast_clear_flush
.fmask_decompress_pipeline
,
490 radv_DestroyPipeline(radv_device_to_handle(device
),
491 state
->fast_clear_flush
.cmask_eliminate_pipeline
,
493 radv_DestroyRenderPass(radv_device_to_handle(device
),
494 state
->fast_clear_flush
.pass
, &state
->alloc
);
495 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
496 state
->fast_clear_flush
.p_layout
,
499 radv_DestroyPipeline(radv_device_to_handle(device
),
500 state
->fast_clear_flush
.dcc_decompress_compute_pipeline
,
502 radv_DestroyPipelineLayout(radv_device_to_handle(device
),
503 state
->fast_clear_flush
.dcc_decompress_compute_p_layout
,
505 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device
),
506 state
->fast_clear_flush
.dcc_decompress_compute_ds_layout
,
511 radv_device_init_meta_fast_clear_flush_state_internal(struct radv_device
*device
)
513 VkResult res
= VK_SUCCESS
;
515 mtx_lock(&device
->meta_state
.mtx
);
516 if (device
->meta_state
.fast_clear_flush
.cmask_eliminate_pipeline
) {
517 mtx_unlock(&device
->meta_state
.mtx
);
521 struct radv_shader_module vs_module
= { .nir
= radv_meta_build_nir_vs_generate_vertices() };
522 if (!vs_module
.nir
) {
523 /* XXX: Need more accurate error */
524 res
= VK_ERROR_OUT_OF_HOST_MEMORY
;
528 res
= create_pass(device
);
529 if (res
!= VK_SUCCESS
)
532 res
= create_pipeline_layout(device
,
533 &device
->meta_state
.fast_clear_flush
.p_layout
);
534 if (res
!= VK_SUCCESS
)
537 VkShaderModule vs_module_h
= radv_shader_module_to_handle(&vs_module
);
538 res
= create_pipeline(device
, vs_module_h
,
539 device
->meta_state
.fast_clear_flush
.p_layout
);
540 if (res
!= VK_SUCCESS
)
543 res
= create_dcc_compress_compute(device
);
544 if (res
!= VK_SUCCESS
)
550 radv_device_finish_meta_fast_clear_flush_state(device
);
553 ralloc_free(vs_module
.nir
);
554 mtx_unlock(&device
->meta_state
.mtx
);
561 radv_device_init_meta_fast_clear_flush_state(struct radv_device
*device
, bool on_demand
)
566 return radv_device_init_meta_fast_clear_flush_state_internal(device
);
570 radv_emit_set_predication_state_from_image(struct radv_cmd_buffer
*cmd_buffer
,
571 struct radv_image
*image
,
572 uint64_t pred_offset
, bool value
)
577 va
= radv_buffer_get_va(image
->bo
) + image
->offset
;
581 si_emit_set_predication_state(cmd_buffer
, true, va
);
585 radv_process_color_image_layer(struct radv_cmd_buffer
*cmd_buffer
,
586 struct radv_image
*image
,
587 const VkImageSubresourceRange
*range
,
588 int level
, int layer
)
590 struct radv_device
*device
= cmd_buffer
->device
;
591 struct radv_image_view iview
;
592 uint32_t width
, height
;
594 width
= radv_minify(image
->info
.width
, range
->baseMipLevel
+ level
);
595 height
= radv_minify(image
->info
.height
, range
->baseMipLevel
+ level
);
597 radv_image_view_init(&iview
, device
,
598 &(VkImageViewCreateInfo
) {
599 .sType
= VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO
,
600 .image
= radv_image_to_handle(image
),
601 .viewType
= radv_meta_get_view_type(image
),
602 .format
= image
->vk_format
,
603 .subresourceRange
= {
604 .aspectMask
= VK_IMAGE_ASPECT_COLOR_BIT
,
605 .baseMipLevel
= range
->baseMipLevel
+ level
,
607 .baseArrayLayer
= range
->baseArrayLayer
+ layer
,
613 radv_CreateFramebuffer(radv_device_to_handle(device
),
614 &(VkFramebufferCreateInfo
) {
615 .sType
= VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO
,
616 .attachmentCount
= 1,
617 .pAttachments
= (VkImageView
[]) {
618 radv_image_view_to_handle(&iview
)
623 }, &cmd_buffer
->pool
->alloc
, &fb_h
);
625 radv_cmd_buffer_begin_render_pass(cmd_buffer
,
626 &(VkRenderPassBeginInfo
) {
627 .sType
= VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO
,
628 .renderPass
= device
->meta_state
.fast_clear_flush
.pass
,
632 .extent
= { width
, height
, }
634 .clearValueCount
= 0,
635 .pClearValues
= NULL
,
638 radv_cmd_buffer_set_subpass(cmd_buffer
,
639 &cmd_buffer
->state
.pass
->subpasses
[0]);
641 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer
), 3, 1, 0, 0);
643 cmd_buffer
->state
.flush_bits
|= RADV_CMD_FLAG_FLUSH_AND_INV_CB
|
644 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META
;
646 radv_cmd_buffer_end_render_pass(cmd_buffer
);
648 radv_DestroyFramebuffer(radv_device_to_handle(device
), fb_h
,
649 &cmd_buffer
->pool
->alloc
);
653 radv_process_color_image(struct radv_cmd_buffer
*cmd_buffer
,
654 struct radv_image
*image
,
655 const VkImageSubresourceRange
*subresourceRange
,
658 struct radv_meta_saved_state saved_state
;
659 VkPipeline
*pipeline
;
661 if (decompress_dcc
&& radv_dcc_enabled(image
, subresourceRange
->baseMipLevel
)) {
662 pipeline
= &cmd_buffer
->device
->meta_state
.fast_clear_flush
.dcc_decompress_pipeline
;
663 } else if (radv_image_has_fmask(image
) && !image
->tc_compatible_cmask
) {
664 pipeline
= &cmd_buffer
->device
->meta_state
.fast_clear_flush
.fmask_decompress_pipeline
;
666 pipeline
= &cmd_buffer
->device
->meta_state
.fast_clear_flush
.cmask_eliminate_pipeline
;
672 ret
= radv_device_init_meta_fast_clear_flush_state_internal(cmd_buffer
->device
);
673 if (ret
!= VK_SUCCESS
) {
674 cmd_buffer
->record_result
= ret
;
679 radv_meta_save(&saved_state
, cmd_buffer
,
680 RADV_META_SAVE_GRAPHICS_PIPELINE
|
681 RADV_META_SAVE_PASS
);
683 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
684 VK_PIPELINE_BIND_POINT_GRAPHICS
, *pipeline
);
686 for (uint32_t l
= 0; l
< radv_get_levelCount(image
, subresourceRange
); ++l
) {
687 uint32_t width
, height
;
689 /* Do not decompress levels without DCC. */
690 if (decompress_dcc
&&
691 !radv_dcc_enabled(image
, subresourceRange
->baseMipLevel
+ l
))
694 width
= radv_minify(image
->info
.width
,
695 subresourceRange
->baseMipLevel
+ l
);
696 height
= radv_minify(image
->info
.height
,
697 subresourceRange
->baseMipLevel
+ l
);
699 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer
), 0, 1,
709 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer
), 0, 1,
712 .extent
= { width
, height
},
715 for (uint32_t s
= 0; s
< radv_get_layerCount(image
, subresourceRange
); s
++) {
716 radv_process_color_image_layer(cmd_buffer
, image
,
717 subresourceRange
, l
, s
);
721 radv_meta_restore(&saved_state
, cmd_buffer
);
725 radv_emit_color_decompress(struct radv_cmd_buffer
*cmd_buffer
,
726 struct radv_image
*image
,
727 const VkImageSubresourceRange
*subresourceRange
,
730 bool old_predicating
= false;
732 assert(cmd_buffer
->queue_family_index
== RADV_QUEUE_GENERAL
);
734 if (radv_dcc_enabled(image
, subresourceRange
->baseMipLevel
)) {
735 uint64_t pred_offset
= decompress_dcc
? image
->dcc_pred_offset
:
736 image
->fce_pred_offset
;
737 pred_offset
+= 8 * subresourceRange
->baseMipLevel
;
739 old_predicating
= cmd_buffer
->state
.predicating
;
741 radv_emit_set_predication_state_from_image(cmd_buffer
, image
, pred_offset
, true);
742 cmd_buffer
->state
.predicating
= true;
745 radv_process_color_image(cmd_buffer
, image
, subresourceRange
,
748 if (radv_dcc_enabled(image
, subresourceRange
->baseMipLevel
)) {
749 uint64_t pred_offset
= decompress_dcc
? image
->dcc_pred_offset
:
750 image
->fce_pred_offset
;
751 pred_offset
+= 8 * subresourceRange
->baseMipLevel
;
753 cmd_buffer
->state
.predicating
= old_predicating
;
755 radv_emit_set_predication_state_from_image(cmd_buffer
, image
, pred_offset
, false);
757 if (cmd_buffer
->state
.predication_type
!= -1) {
758 /* Restore previous conditional rendering user state. */
759 si_emit_set_predication_state(cmd_buffer
,
760 cmd_buffer
->state
.predication_type
,
761 cmd_buffer
->state
.predication_va
);
765 if (radv_dcc_enabled(image
, subresourceRange
->baseMipLevel
)) {
766 /* Clear the image's fast-clear eliminate predicate because
767 * FMASK and DCC also imply a fast-clear eliminate.
769 radv_update_fce_metadata(cmd_buffer
, image
, subresourceRange
, false);
771 /* Mark the image as being decompressed. */
773 radv_update_dcc_metadata(cmd_buffer
, image
, subresourceRange
, false);
778 radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer
*cmd_buffer
,
779 struct radv_image
*image
,
780 const VkImageSubresourceRange
*subresourceRange
)
782 struct radv_barrier_data barrier
= {};
784 if (radv_image_has_fmask(image
)) {
785 barrier
.layout_transitions
.fmask_decompress
= 1;
787 barrier
.layout_transitions
.fast_clear_eliminate
= 1;
789 radv_describe_layout_transition(cmd_buffer
, &barrier
);
791 radv_emit_color_decompress(cmd_buffer
, image
, subresourceRange
, false);
795 radv_decompress_dcc_gfx(struct radv_cmd_buffer
*cmd_buffer
,
796 struct radv_image
*image
,
797 const VkImageSubresourceRange
*subresourceRange
)
799 radv_emit_color_decompress(cmd_buffer
, image
, subresourceRange
, true);
803 radv_decompress_dcc_compute(struct radv_cmd_buffer
*cmd_buffer
,
804 struct radv_image
*image
,
805 const VkImageSubresourceRange
*subresourceRange
)
807 struct radv_meta_saved_state saved_state
;
808 struct radv_image_view load_iview
= {0};
809 struct radv_image_view store_iview
= {0};
810 struct radv_device
*device
= cmd_buffer
->device
;
812 /* This assumes the image is 2d with 1 layer */
813 struct radv_cmd_state
*state
= &cmd_buffer
->state
;
815 state
->flush_bits
|= RADV_CMD_FLAG_FLUSH_AND_INV_CB
|
816 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META
;
818 if (!cmd_buffer
->device
->meta_state
.fast_clear_flush
.cmask_eliminate_pipeline
) {
819 VkResult ret
= radv_device_init_meta_fast_clear_flush_state_internal(cmd_buffer
->device
);
820 if (ret
!= VK_SUCCESS
) {
821 cmd_buffer
->record_result
= ret
;
826 radv_meta_save(&saved_state
, cmd_buffer
, RADV_META_SAVE_DESCRIPTORS
|
827 RADV_META_SAVE_COMPUTE_PIPELINE
);
829 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer
),
830 VK_PIPELINE_BIND_POINT_COMPUTE
,
831 device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_pipeline
);
833 for (uint32_t l
= 0; l
< radv_get_levelCount(image
, subresourceRange
); l
++) {
834 uint32_t width
, height
;
836 /* Do not decompress levels without DCC. */
837 if (!radv_dcc_enabled(image
, subresourceRange
->baseMipLevel
+ l
))
840 width
= radv_minify(image
->info
.width
,
841 subresourceRange
->baseMipLevel
+ l
);
842 height
= radv_minify(image
->info
.height
,
843 subresourceRange
->baseMipLevel
+ l
);
845 for (uint32_t s
= 0; s
< radv_get_layerCount(image
, subresourceRange
); s
++) {
846 radv_image_view_init(&load_iview
, cmd_buffer
->device
,
847 &(VkImageViewCreateInfo
) {
848 .sType
= VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO
,
849 .image
= radv_image_to_handle(image
),
850 .viewType
= VK_IMAGE_VIEW_TYPE_2D
,
851 .format
= image
->vk_format
,
852 .subresourceRange
= {
853 .aspectMask
= VK_IMAGE_ASPECT_COLOR_BIT
,
854 .baseMipLevel
= subresourceRange
->baseMipLevel
+ l
,
856 .baseArrayLayer
= subresourceRange
->baseArrayLayer
+ s
,
860 radv_image_view_init(&store_iview
, cmd_buffer
->device
,
861 &(VkImageViewCreateInfo
) {
862 .sType
= VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO
,
863 .image
= radv_image_to_handle(image
),
864 .viewType
= VK_IMAGE_VIEW_TYPE_2D
,
865 .format
= image
->vk_format
,
866 .subresourceRange
= {
867 .aspectMask
= VK_IMAGE_ASPECT_COLOR_BIT
,
868 .baseMipLevel
= subresourceRange
->baseMipLevel
+ l
,
870 .baseArrayLayer
= subresourceRange
->baseArrayLayer
+ s
,
873 }, &(struct radv_image_view_extra_create_info
) {
874 .disable_compression
= true
877 radv_meta_push_descriptor_set(cmd_buffer
,
878 VK_PIPELINE_BIND_POINT_COMPUTE
,
879 device
->meta_state
.fast_clear_flush
.dcc_decompress_compute_p_layout
,
881 2, /* descriptorWriteCount */
882 (VkWriteDescriptorSet
[]) {
884 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
886 .dstArrayElement
= 0,
887 .descriptorCount
= 1,
888 .descriptorType
= VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE
,
889 .pImageInfo
= (VkDescriptorImageInfo
[]) {
891 .sampler
= VK_NULL_HANDLE
,
892 .imageView
= radv_image_view_to_handle(&load_iview
),
893 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
898 .sType
= VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
,
900 .dstArrayElement
= 0,
901 .descriptorCount
= 1,
902 .descriptorType
= VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
,
903 .pImageInfo
= (VkDescriptorImageInfo
[]) {
905 .sampler
= VK_NULL_HANDLE
,
906 .imageView
= radv_image_view_to_handle(&store_iview
),
907 .imageLayout
= VK_IMAGE_LAYOUT_GENERAL
,
913 radv_unaligned_dispatch(cmd_buffer
, width
, height
, 1);
917 /* Mark this image as actually being decompressed. */
918 radv_update_dcc_metadata(cmd_buffer
, image
, subresourceRange
, false);
920 /* The fill buffer below does its own saving */
921 radv_meta_restore(&saved_state
, cmd_buffer
);
923 state
->flush_bits
|= RADV_CMD_FLAG_CS_PARTIAL_FLUSH
|
924 RADV_CMD_FLAG_INV_VCACHE
;
927 /* Initialize the DCC metadata as "fully expanded". */
928 radv_initialize_dcc(cmd_buffer
, image
, subresourceRange
, 0xffffffff);
932 radv_decompress_dcc(struct radv_cmd_buffer
*cmd_buffer
,
933 struct radv_image
*image
,
934 const VkImageSubresourceRange
*subresourceRange
)
936 struct radv_barrier_data barrier
= {};
938 barrier
.layout_transitions
.dcc_decompress
= 1;
939 radv_describe_layout_transition(cmd_buffer
, &barrier
);
941 if (cmd_buffer
->queue_family_index
== RADV_QUEUE_GENERAL
)
942 radv_decompress_dcc_gfx(cmd_buffer
, image
, subresourceRange
);
944 radv_decompress_dcc_compute(cmd_buffer
, image
, subresourceRange
);