amd/common: cleanup DATA_FORMAT/NUM_FORMAT field names
[mesa.git] / src / amd / vulkan / radv_meta_resolve_cs.c
index ca8f826f5388445901e976ee244bf5549b48208e..b6c9e1c2b1faa9076115be2939a8c626c0570706 100644 (file)
 static nir_ssa_def *radv_meta_build_resolve_srgb_conversion(nir_builder *b,
                                                            nir_ssa_def *input)
 {
-       nir_const_value v;
        unsigned i;
-       v.u32[0] = 0x3b4d2e1c; // 0.00313080009
 
        nir_ssa_def *cmp[3];
        for (i = 0; i < 3; i++)
                cmp[i] = nir_flt(b, nir_channel(b, input, i),
-                                nir_build_imm(b, 1, 32, v));
+                                nir_imm_int(b, 0x3b4d2e1c));
 
        nir_ssa_def *ltvals[3];
-       v.f32[0] = 12.92;
        for (i = 0; i < 3; i++)
                ltvals[i] = nir_fmul(b, nir_channel(b, input, i),
-                                    nir_build_imm(b, 1, 32, v));
+                                    nir_imm_float(b, 12.92));
 
        nir_ssa_def *gtvals[3];
 
        for (i = 0; i < 3; i++) {
-               v.f32[0] = 1.0/2.4;
                gtvals[i] = nir_fpow(b, nir_channel(b, input, i),
-                                    nir_build_imm(b, 1, 32, v));
-               v.f32[0] = 1.055;
+                                    nir_imm_float(b, 1.0/2.4));
                gtvals[i] = nir_fmul(b, gtvals[i],
-                                    nir_build_imm(b, 1, 32, v));
-               v.f32[0] = 0.055;
+                                    nir_imm_float(b, 1.055));
                gtvals[i] = nir_fsub(b, gtvals[i],
-                                    nir_build_imm(b, 1, 32, v));
+                                    nir_imm_float(b, 0.055));
        }
 
        nir_ssa_def *comp[4];
@@ -99,8 +93,8 @@ build_resolve_compute_shader(struct radv_device *dev, bool is_integer, bool is_s
                                                       img_type, "out_img");
        output_img->data.descriptor_set = 0;
        output_img->data.binding = 1;
-       nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0);
-       nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0);
+       nir_ssa_def *invoc_id = nir_load_local_invocation_id(&b);
+       nir_ssa_def *wg_id = nir_load_work_group_id(&b);
        nir_ssa_def *block_size = nir_imm_ivec4(&b,
                                                b.shader->info.cs.local_size[0],
                                                b.shader->info.cs.local_size[1],
@@ -135,11 +129,12 @@ build_resolve_compute_shader(struct radv_device *dev, bool is_integer, bool is_s
                outval = radv_meta_build_resolve_srgb_conversion(&b, outval);
 
        nir_ssa_def *coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa);
-       nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_var_store);
-       store->src[0] = nir_src_for_ssa(coord);
-       store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
-       store->src[2] = nir_src_for_ssa(outval);
-       store->variables[0] = nir_deref_var_create(store, output_img);
+       nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
+       store->num_components = 4;
+       store->src[0] = nir_src_for_ssa(&nir_build_deref_var(&b, output_img)->dest.ssa);
+       store->src[1] = nir_src_for_ssa(coord);
+       store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
+       store->src[3] = nir_src_for_ssa(outval);
        nir_builder_instr_insert(&b, &store->instr);
        return b.shader;
 }
@@ -212,6 +207,12 @@ create_resolve_pipeline(struct radv_device *device,
        VkResult result;
        struct radv_shader_module cs = { .nir = NULL };
 
+       mtx_lock(&device->meta_state.mtx);
+       if (*pipeline) {
+               mtx_unlock(&device->meta_state.mtx);
+               return VK_SUCCESS;
+       }
+
        cs.nir = build_resolve_compute_shader(device, is_integer, is_srgb, samples);
 
        /* compute shader */
@@ -239,14 +240,16 @@ create_resolve_pipeline(struct radv_device *device,
                goto fail;
 
        ralloc_free(cs.nir);
+       mtx_unlock(&device->meta_state.mtx);
        return VK_SUCCESS;
 fail:
        ralloc_free(cs.nir);
+       mtx_unlock(&device->meta_state.mtx);
        return result;
 }
 
 VkResult
-radv_device_init_meta_resolve_compute_state(struct radv_device *device)
+radv_device_init_meta_resolve_compute_state(struct radv_device *device, bool on_demand)
 {
        struct radv_meta_state *state = &device->meta_state;
        VkResult res;
@@ -255,6 +258,9 @@ radv_device_init_meta_resolve_compute_state(struct radv_device *device)
        if (res != VK_SUCCESS)
                goto fail;
 
+       if (on_demand)
+               return VK_SUCCESS;
+
        for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
                uint32_t samples = 1 << i;
 
@@ -307,6 +313,39 @@ radv_device_finish_meta_resolve_compute_state(struct radv_device *device)
                                   &state->alloc);
 }
 
+static VkPipeline *
+radv_get_resolve_pipeline(struct radv_cmd_buffer *cmd_buffer,
+                         struct radv_image_view *src_iview)
+{
+       struct radv_device *device = cmd_buffer->device;
+       struct radv_meta_state *state = &device->meta_state;
+       uint32_t samples = src_iview->image->info.samples;
+       uint32_t samples_log2 = ffs(samples) - 1;
+       VkPipeline *pipeline;
+
+       if (vk_format_is_int(src_iview->vk_format))
+               pipeline = &state->resolve_compute.rc[samples_log2].i_pipeline;
+       else if (vk_format_is_srgb(src_iview->vk_format))
+               pipeline = &state->resolve_compute.rc[samples_log2].srgb_pipeline;
+       else
+               pipeline = &state->resolve_compute.rc[samples_log2].pipeline;
+
+       if (!*pipeline) {
+               VkResult ret;
+
+               ret = create_resolve_pipeline(device, samples,
+                                             vk_format_is_int(src_iview->vk_format),
+                                             vk_format_is_srgb(src_iview->vk_format),
+                                             pipeline);
+               if (ret != VK_SUCCESS) {
+                       cmd_buffer->record_result = ret;
+                       return NULL;
+               }
+       }
+
+       return pipeline;
+}
+
 static void
 emit_resolve(struct radv_cmd_buffer *cmd_buffer,
             struct radv_image_view *src_iview,
@@ -316,8 +355,8 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer,
              const VkExtent2D *resolve_extent)
 {
        struct radv_device *device = cmd_buffer->device;
-       const uint32_t samples = src_iview->image->info.samples;
-       const uint32_t samples_log2 = ffs(samples) - 1;
+       VkPipeline *pipeline;
+
        radv_meta_push_descriptor_set(cmd_buffer,
                                      VK_PIPELINE_BIND_POINT_COMPUTE,
                                      device->meta_state.resolve_compute.p_layout,
@@ -353,16 +392,10 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer,
                              }
                                      });
 
-       VkPipeline pipeline;
-       if (vk_format_is_int(src_iview->image->vk_format))
-               pipeline = device->meta_state.resolve_compute.rc[samples_log2].i_pipeline;
-       else if (vk_format_is_srgb(src_iview->image->vk_format))
-               pipeline = device->meta_state.resolve_compute.rc[samples_log2].srgb_pipeline;
-       else
-               pipeline = device->meta_state.resolve_compute.rc[samples_log2].pipeline;
+       pipeline = radv_get_resolve_pipeline(cmd_buffer, src_iview);
 
        radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
-                            VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
+                            VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
 
        unsigned push_constants[4] = {
                src_offset->x,
@@ -380,27 +413,18 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer,
 
 void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer,
                                     struct radv_image *src_image,
+                                    VkFormat src_format,
                                     VkImageLayout src_image_layout,
                                     struct radv_image *dest_image,
+                                    VkFormat dest_format,
                                     VkImageLayout dest_image_layout,
                                     uint32_t region_count,
                                     const VkImageResolve *regions)
 {
        struct radv_meta_saved_state saved_state;
 
-       for (uint32_t r = 0; r < region_count; ++r) {
-               const VkImageResolve *region = &regions[r];
-               const uint32_t src_base_layer =
-                       radv_meta_get_iview_layer(src_image, &region->srcSubresource,
-                                                 &region->srcOffset);
-               VkImageSubresourceRange range;
-               range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
-               range.baseMipLevel = region->srcSubresource.mipLevel;
-               range.levelCount = 1;
-               range.baseArrayLayer = src_base_layer;
-               range.layerCount = region->srcSubresource.layerCount;
-               radv_fast_clear_flush_image_inplace(cmd_buffer, src_image, &range);
-       }
+       radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout,
+                                   region_count, regions);
 
        radv_meta_save(&saved_state, cmd_buffer,
                       RADV_META_SAVE_COMPUTE_PIPELINE |
@@ -438,7 +462,7 @@ void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer,
                                                     .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
                                                             .image = radv_image_to_handle(src_image),
                                                             .viewType = radv_meta_get_view_type(src_image),
-                                                            .format = src_image->vk_format,
+                                                            .format = src_format,
                                                             .subresourceRange = {
                                                             .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
                                                             .baseMipLevel = region->srcSubresource.mipLevel,
@@ -454,7 +478,7 @@ void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer,
                                                     .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
                                                             .image = radv_image_to_handle(dest_image),
                                                             .viewType = radv_meta_get_view_type(dest_image),
-                                                            .format = vk_to_non_srgb_format(dest_image->vk_format),
+                                                            .format = vk_to_non_srgb_format(dest_format),
                                                             .subresourceRange = {
                                                             .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
                                                             .baseMipLevel = region->dstSubresource.mipLevel,
@@ -483,80 +507,53 @@ radv_cmd_buffer_resolve_subpass_cs(struct radv_cmd_buffer *cmd_buffer)
 {
        struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
        const struct radv_subpass *subpass = cmd_buffer->state.subpass;
-       struct radv_meta_saved_state saved_state;
-       /* FINISHME(perf): Skip clears for resolve attachments.
-        *
-        * From the Vulkan 1.0 spec:
-        *
-        *    If the first use of an attachment in a render pass is as a resolve
-        *    attachment, then the loadOp is effectively ignored as the resolve is
-        *    guaranteed to overwrite all pixels in the render area.
-        */
+       struct radv_subpass_barrier barrier;
 
-       if (!subpass->has_resolve)
-               return;
-
-       /* Resolves happen before the end-of-subpass barriers get executed,
-        * so we have to make the attachment shader-readable */
-       cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
-                                       RADV_CMD_FLAG_FLUSH_AND_INV_CB |
-                                       RADV_CMD_FLAG_FLUSH_AND_INV_CB_META |
-                                       RADV_CMD_FLAG_INV_GLOBAL_L2 |
-                                       RADV_CMD_FLAG_INV_VMEM_L1;
+       /* Resolves happen before the end-of-subpass barriers get executed, so
+        * we have to make the attachment shader-readable.
+        */
+       barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
+       barrier.src_access_mask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+       barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
+       radv_subpass_barrier(cmd_buffer, &barrier);
 
        for (uint32_t i = 0; i < subpass->color_count; ++i) {
-               VkAttachmentReference src_att = subpass->color_attachments[i];
-               VkAttachmentReference dest_att = subpass->resolve_attachments[i];
-
-               if (src_att.attachment == VK_ATTACHMENT_UNUSED ||
-                   dest_att.attachment == VK_ATTACHMENT_UNUSED)
-                       continue;
-
-               struct radv_image_view *src_iview = cmd_buffer->state.framebuffer->attachments[src_att.attachment].attachment;
-
-               VkImageSubresourceRange range;
-               range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
-               range.baseMipLevel = 0;
-               range.levelCount = 1;
-               range.baseArrayLayer = 0;
-               range.layerCount = 1;
-               radv_fast_clear_flush_image_inplace(cmd_buffer, src_iview->image, &range);
-       }
-
-       radv_meta_save(&saved_state, cmd_buffer,
-                      RADV_META_SAVE_COMPUTE_PIPELINE |
-                      RADV_META_SAVE_CONSTANTS |
-                      RADV_META_SAVE_DESCRIPTORS);
+               struct radv_subpass_attachment src_att = subpass->color_attachments[i];
+               struct radv_subpass_attachment dst_att = subpass->resolve_attachments[i];
+               struct radv_image_view *src_iview = fb->attachments[src_att.attachment].attachment;
+               struct radv_image_view *dst_iview = fb->attachments[dst_att.attachment].attachment;
 
-       for (uint32_t i = 0; i < subpass->color_count; ++i) {
-               VkAttachmentReference src_att = subpass->color_attachments[i];
-               VkAttachmentReference dest_att = subpass->resolve_attachments[i];
-               struct radv_image_view *src_iview = cmd_buffer->state.framebuffer->attachments[src_att.attachment].attachment;
-               struct radv_image_view *dst_iview = cmd_buffer->state.framebuffer->attachments[dest_att.attachment].attachment;
-               if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
+               if (dst_att.attachment == VK_ATTACHMENT_UNUSED)
                        continue;
 
-               emit_resolve(cmd_buffer,
-                            src_iview,
-                            dst_iview,
-                            &(VkOffset2D) { 0, 0 },
-                            &(VkOffset2D) { 0, 0 },
-                            &(VkExtent2D) { fb->width, fb->height });
+               VkImageResolve region = {
+                       .extent = (VkExtent3D){ fb->width, fb->height, 0 },
+                       .srcSubresource = (VkImageSubresourceLayers) {
+                               .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+                               .mipLevel = src_iview->base_mip,
+                               .baseArrayLayer = 0,
+                               .layerCount = src_iview->image->info.array_size
+                       },
+                       .dstSubresource = (VkImageSubresourceLayers) {
+                               .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+                               .mipLevel = dst_iview->base_mip,
+                               .baseArrayLayer = 0,
+                               .layerCount = dst_iview->image->info.array_size
+                       },
+                       .srcOffset = (VkOffset3D){ 0, 0, 0 },
+                       .dstOffset = (VkOffset3D){ 0, 0, 0 },
+               };
+
+               radv_meta_resolve_compute_image(cmd_buffer,
+                                               src_iview->image,
+                                               src_iview->vk_format,
+                                               src_att.layout,
+                                               dst_iview->image,
+                                               dst_iview->vk_format,
+                                               dst_att.layout,
+                                               1, &region);
        }
 
-       radv_meta_restore(&saved_state, cmd_buffer);
-
-       for (uint32_t i = 0; i < subpass->color_count; ++i) {
-               VkAttachmentReference dest_att = subpass->resolve_attachments[i];
-               struct radv_image *dst_img = cmd_buffer->state.framebuffer->attachments[dest_att.attachment].attachment->image;
-               if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
-                       continue;
-               VkImageSubresourceRange range;
-               range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
-               range.baseMipLevel = 0;
-               range.levelCount = 1;
-               range.baseArrayLayer = 0;
-               range.layerCount = 1;
-               radv_fast_clear_flush_image_inplace(cmd_buffer, dst_img, &range);
-       }
+       cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH |
+                                       RADV_CMD_FLAG_INV_VMEM_L1;
 }