X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fvulkan%2Fradv_cmd_buffer.c;h=3646c1ae89e3bff9676a494d5650ed0c0cafea78;hb=4cf8f329edb3ad3482819de8dc091061ae19c5af;hp=4b5556bcc62e2460b4e30f4d9aba6ff054e406e8;hpb=5db0bf99944ff2d3e83ef27d2aebe8f074d93d59;p=mesa.git diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 4b5556bcc62..3646c1ae89e 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -37,6 +37,20 @@ #include "ac_debug.h" +enum { + RADV_PREFETCH_VBO_DESCRIPTORS = (1 << 0), + RADV_PREFETCH_VS = (1 << 1), + RADV_PREFETCH_TCS = (1 << 2), + RADV_PREFETCH_TES = (1 << 3), + RADV_PREFETCH_GS = (1 << 4), + RADV_PREFETCH_PS = (1 << 5), + RADV_PREFETCH_SHADERS = (RADV_PREFETCH_VS | + RADV_PREFETCH_TCS | + RADV_PREFETCH_TES | + RADV_PREFETCH_GS | + RADV_PREFETCH_PS) +}; + static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout src_layout, @@ -212,7 +226,7 @@ static VkResult radv_create_cmd_buffer( cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (cmd_buffer == NULL) - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC; cmd_buffer->device = device; @@ -224,7 +238,7 @@ static VkResult radv_create_cmd_buffer( cmd_buffer->queue_family_index = pool->queue_family_index; } else { - /* Init the pool_link so we can safefly call list_del when we destroy + /* Init the pool_link so we can safely call list_del when we destroy * the command buffer */ list_inithead(&cmd_buffer->pool_link); @@ -236,7 +250,7 @@ static VkResult radv_create_cmd_buffer( cmd_buffer->cs = device->ws->cs_create(device->ws, ring); if (!cmd_buffer->cs) { vk_free(&cmd_buffer->pool->alloc, cmd_buffer); - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); } *pCommandBuffer = radv_cmd_buffer_to_handle(cmd_buffer); @@ -261,7 +275,10 @@ radv_cmd_buffer_destroy(struct radv_cmd_buffer *cmd_buffer) if (cmd_buffer->upload.upload_bo) cmd_buffer->device->ws->buffer_destroy(cmd_buffer->upload.upload_bo); cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs); - free(cmd_buffer->push_descriptors.set.mapped_ptr); + + for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) + free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr); + vk_free(&cmd_buffer->pool->alloc, cmd_buffer); } @@ -288,19 +305,33 @@ radv_reset_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) if (cmd_buffer->upload.upload_bo) radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, - cmd_buffer->upload.upload_bo, 8); + cmd_buffer->upload.upload_bo); cmd_buffer->upload.offset = 0; cmd_buffer->record_result = VK_SUCCESS; - cmd_buffer->ring_offsets_idx = -1; + for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) { + cmd_buffer->descriptors[i].dirty = 0; + cmd_buffer->descriptors[i].valid = 0; + cmd_buffer->descriptors[i].push_dirty = false; + } if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { + unsigned num_db = cmd_buffer->device->physical_device->rad_info.num_render_backends; + unsigned eop_bug_offset; void *fence_ptr; + radv_cmd_buffer_upload_alloc(cmd_buffer, 8, 0, &cmd_buffer->gfx9_fence_offset, &fence_ptr); cmd_buffer->gfx9_fence_bo = cmd_buffer->upload.upload_bo; + + /* Allocate a buffer for the EOP bug on GFX9. */ + radv_cmd_buffer_upload_alloc(cmd_buffer, 16 * num_db, 0, + &eop_bug_offset, &fence_ptr); + cmd_buffer->gfx9_eop_bug_va = + radv_buffer_get_va(cmd_buffer->upload.upload_bo); + cmd_buffer->gfx9_eop_bug_va += eop_bug_offset; } cmd_buffer->status = RADV_CMD_BUFFER_STATUS_INITIAL; @@ -324,14 +355,15 @@ radv_cmd_buffer_resize_upload_buf(struct radv_cmd_buffer *cmd_buffer, new_size, 4096, RADEON_DOMAIN_GTT, RADEON_FLAG_CPU_ACCESS| - RADEON_FLAG_NO_INTERPROCESS_SHARING); + RADEON_FLAG_NO_INTERPROCESS_SHARING | + RADEON_FLAG_32BIT); if (!bo) { cmd_buffer->record_result = VK_ERROR_OUT_OF_DEVICE_MEMORY; return false; } - radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo, 8); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo); if (cmd_buffer->upload.upload_bo) { upload = malloc(sizeof(*upload)); @@ -397,7 +429,7 @@ radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, } static void -radv_emit_write_data_packet(struct radeon_winsys_cs *cs, uint64_t va, +radv_emit_write_data_packet(struct radeon_cmdbuf *cs, uint64_t va, unsigned count, const uint32_t *data) { radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0)); @@ -412,7 +444,7 @@ radv_emit_write_data_packet(struct radeon_winsys_cs *cs, uint64_t va, void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) { struct radv_device *device = cmd_buffer->device; - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va; va = radv_buffer_get_va(device->trace_bo); @@ -422,27 +454,34 @@ void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 7); ++cmd_buffer->state.trace_id; - radv_cs_add_buffer(device->ws, cs, device->trace_bo, 8); radv_emit_write_data_packet(cs, va, 1, &cmd_buffer->state.trace_id); radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); } static void -radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer) +radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, + enum radv_cmd_flush_bits flags) { if (cmd_buffer->device->instance->debug_flags & RADV_DEBUG_SYNC_SHADERS) { - enum radv_cmd_flush_bits flags; + uint32_t *ptr = NULL; + uint64_t va = 0; - /* Force wait for graphics/compute engines to be idle. */ - flags = RADV_CMD_FLAG_PS_PARTIAL_FLUSH | - RADV_CMD_FLAG_CS_PARTIAL_FLUSH; + assert(flags & (RADV_CMD_FLAG_PS_PARTIAL_FLUSH | + RADV_CMD_FLAG_CS_PARTIAL_FLUSH)); - si_cs_emit_cache_flush(cmd_buffer->cs, false, + if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) { + va = radv_buffer_get_va(cmd_buffer->gfx9_fence_bo) + + cmd_buffer->gfx9_fence_offset; + ptr = &cmd_buffer->gfx9_fence_idx; + } + + /* Force wait for graphics or compute engines to be idle. */ + si_cs_emit_cache_flush(cmd_buffer->cs, cmd_buffer->device->physical_device->rad_info.chip_class, - NULL, 0, + ptr, va, radv_cmd_buffer_uses_mec(cmd_buffer), - flags); + flags, cmd_buffer->gfx9_eop_bug_va); } if (unlikely(cmd_buffer->device->trace_bo)) @@ -454,7 +493,7 @@ radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline, enum ring_type ring) { struct radv_device *device = cmd_buffer->device; - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint32_t data[2]; uint64_t va; @@ -477,28 +516,31 @@ radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer, data[0] = (uintptr_t)pipeline; data[1] = (uintptr_t)pipeline >> 32; - radv_cs_add_buffer(device->ws, cs, device->trace_bo, 8); radv_emit_write_data_packet(cs, va, 2, data); } void radv_set_descriptor_set(struct radv_cmd_buffer *cmd_buffer, + VkPipelineBindPoint bind_point, struct radv_descriptor_set *set, unsigned idx) { - cmd_buffer->descriptors[idx] = set; - if (set) - cmd_buffer->state.valid_descriptors |= (1u << idx); - else - cmd_buffer->state.valid_descriptors &= ~(1u << idx); - cmd_buffer->state.descriptors_dirty |= (1u << idx); + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); + descriptors_state->sets[idx] = set; + + descriptors_state->valid |= (1u << idx); /* active descriptors */ + descriptors_state->dirty |= (1u << idx); } static void -radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer) +radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer, + VkPipelineBindPoint bind_point) { + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); struct radv_device *device = cmd_buffer->device; - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint32_t data[MAX_SETS * 2] = {}; uint64_t va; unsigned i; @@ -507,69 +549,22 @@ radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer) MAYBE_UNUSED unsigned cdw_max = radeon_check_space(device->ws, cmd_buffer->cs, 4 + MAX_SETS * 2); - for_each_bit(i, cmd_buffer->state.valid_descriptors) { - struct radv_descriptor_set *set = cmd_buffer->descriptors[i]; + for_each_bit(i, descriptors_state->valid) { + struct radv_descriptor_set *set = descriptors_state->sets[i]; data[i * 2] = (uintptr_t)set; data[i * 2 + 1] = (uintptr_t)set >> 32; } - radv_cs_add_buffer(device->ws, cs, device->trace_bo, 8); radv_emit_write_data_packet(cs, va, MAX_SETS * 2, data); } -static void -radv_emit_graphics_blend_state(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - radeon_set_context_reg_seq(cmd_buffer->cs, R_028780_CB_BLEND0_CONTROL, 8); - radeon_emit_array(cmd_buffer->cs, pipeline->graphics.blend.cb_blend_control, - 8); - radeon_set_context_reg(cmd_buffer->cs, R_028808_CB_COLOR_CONTROL, pipeline->graphics.blend.cb_color_control); - radeon_set_context_reg(cmd_buffer->cs, R_028B70_DB_ALPHA_TO_MASK, pipeline->graphics.blend.db_alpha_to_mask); - - if (cmd_buffer->device->physical_device->has_rbplus) { - - radeon_set_context_reg_seq(cmd_buffer->cs, R_028760_SX_MRT0_BLEND_OPT, 8); - radeon_emit_array(cmd_buffer->cs, pipeline->graphics.blend.sx_mrt_blend_opt, 8); - - radeon_set_context_reg_seq(cmd_buffer->cs, R_028754_SX_PS_DOWNCONVERT, 3); - radeon_emit(cmd_buffer->cs, 0); /* R_028754_SX_PS_DOWNCONVERT */ - radeon_emit(cmd_buffer->cs, 0); /* R_028758_SX_BLEND_OPT_EPSILON */ - radeon_emit(cmd_buffer->cs, 0); /* R_02875C_SX_BLEND_OPT_CONTROL */ - } -} - -static void -radv_emit_graphics_depth_stencil_state(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radv_depth_stencil_state *ds = &pipeline->graphics.ds; - radeon_set_context_reg(cmd_buffer->cs, R_028800_DB_DEPTH_CONTROL, ds->db_depth_control); - radeon_set_context_reg(cmd_buffer->cs, R_02842C_DB_STENCIL_CONTROL, ds->db_stencil_control); - - radeon_set_context_reg(cmd_buffer->cs, R_028000_DB_RENDER_CONTROL, ds->db_render_control); - radeon_set_context_reg(cmd_buffer->cs, R_028010_DB_RENDER_OVERRIDE2, ds->db_render_override2); -} - -struct ac_userdata_info * +struct radv_userdata_info * radv_lookup_user_sgpr(struct radv_pipeline *pipeline, gl_shader_stage stage, int idx) { - if (stage == MESA_SHADER_VERTEX) { - if (pipeline->shaders[MESA_SHADER_VERTEX]) - return &pipeline->shaders[MESA_SHADER_VERTEX]->info.user_sgprs_locs.shader_data[idx]; - if (pipeline->shaders[MESA_SHADER_TESS_CTRL]) - return &pipeline->shaders[MESA_SHADER_TESS_CTRL]->info.user_sgprs_locs.shader_data[idx]; - if (pipeline->shaders[MESA_SHADER_GEOMETRY]) - return &pipeline->shaders[MESA_SHADER_GEOMETRY]->info.user_sgprs_locs.shader_data[idx]; - } else if (stage == MESA_SHADER_TESS_EVAL) { - if (pipeline->shaders[MESA_SHADER_TESS_EVAL]) - return &pipeline->shaders[MESA_SHADER_TESS_EVAL]->info.user_sgprs_locs.shader_data[idx]; - if (pipeline->shaders[MESA_SHADER_GEOMETRY]) - return &pipeline->shaders[MESA_SHADER_GEOMETRY]->info.user_sgprs_locs.shader_data[idx]; - } - return &pipeline->shaders[stage]->info.user_sgprs_locs.shader_data[idx]; + struct radv_shader_variant *shader = radv_get_shader(pipeline, stage); + return &shader->info.user_sgprs_locs.shader_data[idx]; } static void @@ -578,15 +573,51 @@ radv_emit_userdata_address(struct radv_cmd_buffer *cmd_buffer, gl_shader_stage stage, int idx, uint64_t va) { - struct ac_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx); + struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx); uint32_t base_reg = pipeline->user_data_0[stage]; if (loc->sgpr_idx == -1) return; - assert(loc->num_sgprs == 2); + + assert(loc->num_sgprs == (HAVE_32BIT_POINTERS ? 1 : 2)); assert(!loc->indirect); - radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, 2); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); + + radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, + base_reg + loc->sgpr_idx * 4, va, false); +} + +static void +radv_emit_descriptor_pointers(struct radv_cmd_buffer *cmd_buffer, + struct radv_pipeline *pipeline, + struct radv_descriptor_state *descriptors_state, + gl_shader_stage stage) +{ + struct radv_device *device = cmd_buffer->device; + struct radeon_cmdbuf *cs = cmd_buffer->cs; + uint32_t sh_base = pipeline->user_data_0[stage]; + struct radv_userdata_locations *locs = + &pipeline->shaders[stage]->info.user_sgprs_locs; + unsigned mask = locs->descriptor_sets_enabled; + + mask &= descriptors_state->dirty & descriptors_state->valid; + + while (mask) { + int start, count; + + u_bit_scan_consecutive_range(&mask, &start, &count); + + struct radv_userdata_info *loc = &locs->descriptor_sets[start]; + unsigned sh_offset = sh_base + loc->sgpr_idx * 4; + + radv_emit_shader_pointer_head(cs, sh_offset, count, + HAVE_32BIT_POINTERS); + for (int i = 0; i < count; i++) { + struct radv_descriptor_set *set = + descriptors_state->sets[start + i]; + + radv_emit_shader_pointer_body(device, cs, set->va, + HAVE_32BIT_POINTERS); + } + } } static void @@ -597,21 +628,18 @@ radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer, struct radv_multisample_state *ms = &pipeline->graphics.ms; struct radv_pipeline *old_pipeline = cmd_buffer->state.emitted_pipeline; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C38_PA_SC_AA_MASK_X0Y0_X1Y0, 2); - radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_mask[0]); - radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_mask[1]); - - radeon_set_context_reg(cmd_buffer->cs, R_028804_DB_EQAA, ms->db_eqaa); - radeon_set_context_reg(cmd_buffer->cs, R_028A4C_PA_SC_MODE_CNTL_1, ms->pa_sc_mode_cntl_1); + if (pipeline->shaders[MESA_SHADER_FRAGMENT]->info.info.ps.needs_sample_positions) + cmd_buffer->sample_positions_needed = true; - if (old_pipeline && num_samples == old_pipeline->graphics.ms.num_samples && - old_pipeline->shaders[MESA_SHADER_FRAGMENT]->info.info.ps.needs_sample_positions == pipeline->shaders[MESA_SHADER_FRAGMENT]->info.info.ps.needs_sample_positions) + if (old_pipeline && num_samples == old_pipeline->graphics.ms.num_samples) return; radeon_set_context_reg_seq(cmd_buffer->cs, R_028BDC_PA_SC_LINE_CNTL, 2); radeon_emit(cmd_buffer->cs, ms->pa_sc_line_cntl); radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_config); + radeon_set_context_reg(cmd_buffer->cs, R_028A48_PA_SC_MODE_CNTL_0, ms->pa_sc_mode_cntl_0); + radv_cayman_emit_msaa_sample_locs(cmd_buffer->cs, num_samples); /* GFX9: Flush DFSM when the AA mode changes. */ @@ -619,78 +647,12 @@ radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer, radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_FLUSH_DFSM) | EVENT_INDEX(0)); } - if (pipeline->shaders[MESA_SHADER_FRAGMENT]->info.info.ps.needs_sample_positions) { - uint32_t offset; - struct ac_userdata_info *loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_FRAGMENT, AC_UD_PS_SAMPLE_POS_OFFSET); - uint32_t base_reg = pipeline->user_data_0[MESA_SHADER_FRAGMENT]; - if (loc->sgpr_idx == -1) - return; - assert(loc->num_sgprs == 1); - assert(!loc->indirect); - switch (num_samples) { - default: - offset = 0; - break; - case 2: - offset = 1; - break; - case 4: - offset = 3; - break; - case 8: - offset = 7; - break; - case 16: - offset = 15; - break; - } - - radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, offset); - cmd_buffer->sample_positions_needed = true; - } -} - -static void -radv_emit_graphics_raster_state(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radv_raster_state *raster = &pipeline->graphics.raster; - - radeon_set_context_reg(cmd_buffer->cs, R_028810_PA_CL_CLIP_CNTL, - raster->pa_cl_clip_cntl); - radeon_set_context_reg(cmd_buffer->cs, R_0286D4_SPI_INTERP_CONTROL_0, - raster->spi_interp_control); - radeon_set_context_reg(cmd_buffer->cs, R_028BE4_PA_SU_VTX_CNTL, - raster->pa_su_vtx_cntl); - radeon_set_context_reg(cmd_buffer->cs, R_028814_PA_SU_SC_MODE_CNTL, - raster->pa_su_sc_mode_cntl); -} - -static inline void -radv_emit_prefetch_TC_L2_async(struct radv_cmd_buffer *cmd_buffer, uint64_t va, - unsigned size) -{ - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) - si_cp_dma_prefetch(cmd_buffer, va, size); -} - -static void -radv_emit_VBO_descriptors_prefetch(struct radv_cmd_buffer *cmd_buffer) -{ - if (cmd_buffer->state.vb_prefetch_dirty) { - radv_emit_prefetch_TC_L2_async(cmd_buffer, - cmd_buffer->state.vb_va, - cmd_buffer->state.vb_size); - cmd_buffer->state.vb_prefetch_dirty = false; - } } static void radv_emit_shader_prefetch(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_variant *shader) { - struct radeon_winsys *ws = cmd_buffer->device->ws; - struct radeon_winsys_cs *cs = cmd_buffer->cs; uint64_t va; if (!shader) @@ -698,374 +660,186 @@ radv_emit_shader_prefetch(struct radv_cmd_buffer *cmd_buffer, va = radv_buffer_get_va(shader->bo) + shader->bo_offset; - radv_cs_add_buffer(ws, cs, shader->bo, 8); - radv_emit_prefetch_TC_L2_async(cmd_buffer, va, shader->code_size); -} - -static void -radv_emit_prefetch(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - radv_emit_shader_prefetch(cmd_buffer, - pipeline->shaders[MESA_SHADER_VERTEX]); - radv_emit_VBO_descriptors_prefetch(cmd_buffer); - radv_emit_shader_prefetch(cmd_buffer, - pipeline->shaders[MESA_SHADER_TESS_CTRL]); - radv_emit_shader_prefetch(cmd_buffer, - pipeline->shaders[MESA_SHADER_TESS_EVAL]); - radv_emit_shader_prefetch(cmd_buffer, - pipeline->shaders[MESA_SHADER_GEOMETRY]); - radv_emit_shader_prefetch(cmd_buffer, pipeline->gs_copy_shader); - radv_emit_shader_prefetch(cmd_buffer, - pipeline->shaders[MESA_SHADER_FRAGMENT]); -} - -static void -radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - struct radv_shader_variant *shader) -{ - uint64_t va = radv_buffer_get_va(shader->bo) + shader->bo_offset; - - radeon_set_context_reg(cmd_buffer->cs, R_0286C4_SPI_VS_OUT_CONFIG, - pipeline->graphics.vs.spi_vs_out_config); - - radeon_set_context_reg(cmd_buffer->cs, R_02870C_SPI_SHADER_POS_FORMAT, - pipeline->graphics.vs.spi_shader_pos_format); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B120_SPI_SHADER_PGM_LO_VS, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2); - - radeon_set_context_reg(cmd_buffer->cs, R_028818_PA_CL_VTE_CNTL, - S_028818_VTX_W0_FMT(1) | - S_028818_VPORT_X_SCALE_ENA(1) | S_028818_VPORT_X_OFFSET_ENA(1) | - S_028818_VPORT_Y_SCALE_ENA(1) | S_028818_VPORT_Y_OFFSET_ENA(1) | - S_028818_VPORT_Z_SCALE_ENA(1) | S_028818_VPORT_Z_OFFSET_ENA(1)); - - - radeon_set_context_reg(cmd_buffer->cs, R_02881C_PA_CL_VS_OUT_CNTL, - pipeline->graphics.vs.pa_cl_vs_out_cntl); - - if (cmd_buffer->device->physical_device->rad_info.chip_class <= VI) - radeon_set_context_reg(cmd_buffer->cs, R_028AB4_VGT_REUSE_OFF, - pipeline->graphics.vs.vgt_reuse_off); -} - -static void -radv_emit_hw_es(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - struct radv_shader_variant *shader) -{ - uint64_t va = radv_buffer_get_va(shader->bo) + shader->bo_offset; - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B320_SPI_SHADER_PGM_LO_ES, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2); -} - -static void -radv_emit_hw_ls(struct radv_cmd_buffer *cmd_buffer, - struct radv_shader_variant *shader) -{ - uint64_t va = radv_buffer_get_va(shader->bo) + shader->bo_offset; - uint32_t rsrc2 = shader->rsrc2; - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B520_SPI_SHADER_PGM_LO_LS, 2); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - - rsrc2 |= S_00B52C_LDS_SIZE(cmd_buffer->state.pipeline->graphics.tess.lds_size); - if (cmd_buffer->device->physical_device->rad_info.chip_class == CIK && - cmd_buffer->device->physical_device->rad_info.family != CHIP_HAWAII) - radeon_set_sh_reg(cmd_buffer->cs, R_00B52C_SPI_SHADER_PGM_RSRC2_LS, rsrc2); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B528_SPI_SHADER_PGM_RSRC1_LS, 2); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, rsrc2); -} - -static void -radv_emit_hw_hs(struct radv_cmd_buffer *cmd_buffer, - struct radv_shader_variant *shader) -{ - uint64_t va = radv_buffer_get_va(shader->bo) + shader->bo_offset; - - if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B410_SPI_SHADER_PGM_LO_LS, 2); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B428_SPI_SHADER_PGM_RSRC1_HS, 2); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2 | - S_00B42C_LDS_SIZE(cmd_buffer->state.pipeline->graphics.tess.lds_size)); - } else { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B420_SPI_SHADER_PGM_LO_HS, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2); - } + si_cp_dma_prefetch(cmd_buffer, va, shader->code_size); } static void -radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) +radv_emit_prefetch_L2(struct radv_cmd_buffer *cmd_buffer, + struct radv_pipeline *pipeline, + bool vertex_stage_only) { - struct radv_shader_variant *vs; - - radeon_set_context_reg(cmd_buffer->cs, R_028A84_VGT_PRIMITIVEID_EN, pipeline->graphics.vgt_primitiveid_en); - - /* Skip shaders merged into HS/GS */ - vs = pipeline->shaders[MESA_SHADER_VERTEX]; - if (!vs) - return; - - if (vs->info.vs.as_ls) - radv_emit_hw_ls(cmd_buffer, vs); - else if (vs->info.vs.as_es) - radv_emit_hw_es(cmd_buffer, pipeline, vs); - else - radv_emit_hw_vs(cmd_buffer, pipeline, vs); -} - - -static void -radv_emit_tess_shaders(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - if (!radv_pipeline_has_tess(pipeline)) - return; - - struct radv_shader_variant *tes, *tcs; - - tcs = pipeline->shaders[MESA_SHADER_TESS_CTRL]; - tes = pipeline->shaders[MESA_SHADER_TESS_EVAL]; + struct radv_cmd_state *state = &cmd_buffer->state; + uint32_t mask = state->prefetch_L2_mask; - if (tes) { - if (tes->info.tes.as_es) - radv_emit_hw_es(cmd_buffer, pipeline, tes); - else - radv_emit_hw_vs(cmd_buffer, pipeline, tes); + if (vertex_stage_only) { + /* Fast prefetch path for starting draws as soon as possible. + */ + mask = state->prefetch_L2_mask & (RADV_PREFETCH_VS | + RADV_PREFETCH_VBO_DESCRIPTORS); } - radv_emit_hw_hs(cmd_buffer, tcs); + if (mask & RADV_PREFETCH_VS) + radv_emit_shader_prefetch(cmd_buffer, + pipeline->shaders[MESA_SHADER_VERTEX]); - radeon_set_context_reg(cmd_buffer->cs, R_028B6C_VGT_TF_PARAM, - pipeline->graphics.tess.tf_param); + if (mask & RADV_PREFETCH_VBO_DESCRIPTORS) + si_cp_dma_prefetch(cmd_buffer, state->vb_va, state->vb_size); - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) - radeon_set_context_reg_idx(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, 2, - pipeline->graphics.tess.ls_hs_config); - else - radeon_set_context_reg(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, - pipeline->graphics.tess.ls_hs_config); - - struct ac_userdata_info *loc; - - loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_TESS_CTRL, AC_UD_TCS_OFFCHIP_LAYOUT); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = pipeline->user_data_0[MESA_SHADER_TESS_CTRL]; - assert(loc->num_sgprs == 4); - assert(!loc->indirect); - radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, 4); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.offchip_layout); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.tcs_out_offsets); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.tcs_out_layout | - pipeline->graphics.tess.num_tcs_input_cp << 26); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.tcs_in_layout); - } + if (mask & RADV_PREFETCH_TCS) + radv_emit_shader_prefetch(cmd_buffer, + pipeline->shaders[MESA_SHADER_TESS_CTRL]); - loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_TESS_EVAL, AC_UD_TES_OFFCHIP_LAYOUT); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = pipeline->user_data_0[MESA_SHADER_TESS_EVAL]; - assert(loc->num_sgprs == 1); - assert(!loc->indirect); + if (mask & RADV_PREFETCH_TES) + radv_emit_shader_prefetch(cmd_buffer, + pipeline->shaders[MESA_SHADER_TESS_EVAL]); - radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, - pipeline->graphics.tess.offchip_layout); + if (mask & RADV_PREFETCH_GS) { + radv_emit_shader_prefetch(cmd_buffer, + pipeline->shaders[MESA_SHADER_GEOMETRY]); + radv_emit_shader_prefetch(cmd_buffer, pipeline->gs_copy_shader); } - loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_VERTEX, AC_UD_VS_LS_TCS_IN_LAYOUT); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = pipeline->user_data_0[MESA_SHADER_VERTEX]; - assert(loc->num_sgprs == 1); - assert(!loc->indirect); + if (mask & RADV_PREFETCH_PS) + radv_emit_shader_prefetch(cmd_buffer, + pipeline->shaders[MESA_SHADER_FRAGMENT]); - radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, - pipeline->graphics.tess.tcs_in_layout); - } + state->prefetch_L2_mask &= ~mask; } static void -radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) +radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer) { - struct radv_shader_variant *gs; - uint64_t va; - - radeon_set_context_reg(cmd_buffer->cs, R_028A40_VGT_GS_MODE, pipeline->graphics.vgt_gs_mode); - - gs = pipeline->shaders[MESA_SHADER_GEOMETRY]; - if (!gs) + if (!cmd_buffer->device->physical_device->rbplus_allowed) return; - uint32_t gsvs_itemsize = gs->info.gs.max_gsvs_emit_size >> 2; - - radeon_set_context_reg_seq(cmd_buffer->cs, R_028A60_VGT_GSVS_RING_OFFSET_1, 3); - radeon_emit(cmd_buffer->cs, gsvs_itemsize); - radeon_emit(cmd_buffer->cs, gsvs_itemsize); - radeon_emit(cmd_buffer->cs, gsvs_itemsize); - - radeon_set_context_reg(cmd_buffer->cs, R_028AB0_VGT_GSVS_RING_ITEMSIZE, gsvs_itemsize); - - radeon_set_context_reg(cmd_buffer->cs, R_028B38_VGT_GS_MAX_VERT_OUT, gs->info.gs.vertices_out); - - uint32_t gs_vert_itemsize = gs->info.gs.gsvs_vertex_size; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028B5C_VGT_GS_VERT_ITEMSIZE, 4); - radeon_emit(cmd_buffer->cs, gs_vert_itemsize >> 2); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 0); - - uint32_t gs_num_invocations = gs->info.gs.invocations; - radeon_set_context_reg(cmd_buffer->cs, R_028B90_VGT_GS_INSTANCE_CNT, - S_028B90_CNT(MIN2(gs_num_invocations, 127)) | - S_028B90_ENABLE(gs_num_invocations > 0)); - - radeon_set_context_reg(cmd_buffer->cs, R_028AAC_VGT_ESGS_RING_ITEMSIZE, - pipeline->graphics.gs.vgt_esgs_ring_itemsize); - - va = radv_buffer_get_va(gs->bo) + gs->bo_offset; - - if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B210_SPI_SHADER_PGM_LO_ES, 2); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B228_SPI_SHADER_PGM_RSRC1_GS, 2); - radeon_emit(cmd_buffer->cs, gs->rsrc1); - radeon_emit(cmd_buffer->cs, gs->rsrc2 | - S_00B22C_LDS_SIZE(pipeline->graphics.gs.lds_size)); + struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; + struct radv_framebuffer *framebuffer = cmd_buffer->state.framebuffer; + const struct radv_subpass *subpass = cmd_buffer->state.subpass; - radeon_set_context_reg(cmd_buffer->cs, R_028A44_VGT_GS_ONCHIP_CNTL, pipeline->graphics.gs.vgt_gs_onchip_cntl); - radeon_set_context_reg(cmd_buffer->cs, R_028A94_VGT_GS_MAX_PRIMS_PER_SUBGROUP, pipeline->graphics.gs.vgt_gs_max_prims_per_subgroup); - } else { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B220_SPI_SHADER_PGM_LO_GS, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, gs->rsrc1); - radeon_emit(cmd_buffer->cs, gs->rsrc2); - } + unsigned sx_ps_downconvert = 0; + unsigned sx_blend_opt_epsilon = 0; + unsigned sx_blend_opt_control = 0; - radv_emit_hw_vs(cmd_buffer, pipeline, pipeline->gs_copy_shader); + for (unsigned i = 0; i < subpass->color_count; ++i) { + if (subpass->color_attachments[i].attachment == VK_ATTACHMENT_UNUSED) + continue; - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_GEOMETRY, - AC_UD_GS_VS_RING_STRIDE_ENTRIES); - if (loc->sgpr_idx != -1) { - uint32_t stride = gs->info.gs.max_gsvs_emit_size; - uint32_t num_entries = 64; - bool is_vi = cmd_buffer->device->physical_device->rad_info.chip_class >= VI; + int idx = subpass->color_attachments[i].attachment; + struct radv_color_buffer_info *cb = &framebuffer->attachments[idx].cb; - if (is_vi) - num_entries *= stride; + unsigned format = G_028C70_FORMAT(cb->cb_color_info); + unsigned swap = G_028C70_COMP_SWAP(cb->cb_color_info); + uint32_t spi_format = (pipeline->graphics.col_format >> (i * 4)) & 0xf; + uint32_t colormask = (pipeline->graphics.cb_target_mask >> (i * 4)) & 0xf; - stride = S_008F04_STRIDE(stride); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B230_SPI_SHADER_USER_DATA_GS_0 + loc->sgpr_idx * 4, 2); - radeon_emit(cmd_buffer->cs, stride); - radeon_emit(cmd_buffer->cs, num_entries); - } -} + bool has_alpha, has_rgb; -static void -radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radv_shader_variant *ps; - uint64_t va; - unsigned spi_baryc_cntl = S_0286E0_FRONT_FACE_ALL_BITS(1); - struct radv_blend_state *blend = &pipeline->graphics.blend; - assert (pipeline->shaders[MESA_SHADER_FRAGMENT]); + /* Set if RGB and A are present. */ + has_alpha = !G_028C74_FORCE_DST_ALPHA_1(cb->cb_color_attrib); - ps = pipeline->shaders[MESA_SHADER_FRAGMENT]; - va = radv_buffer_get_va(ps->bo) + ps->bo_offset; - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B020_SPI_SHADER_PGM_LO_PS, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, ps->rsrc1); - radeon_emit(cmd_buffer->cs, ps->rsrc2); - - radeon_set_context_reg(cmd_buffer->cs, R_02880C_DB_SHADER_CONTROL, - pipeline->graphics.db_shader_control); + if (format == V_028C70_COLOR_8 || + format == V_028C70_COLOR_16 || + format == V_028C70_COLOR_32) + has_rgb = !has_alpha; + else + has_rgb = true; - radeon_set_context_reg(cmd_buffer->cs, R_0286CC_SPI_PS_INPUT_ENA, - ps->config.spi_ps_input_ena); + /* Check the colormask and export format. */ + if (!(colormask & 0x7)) + has_rgb = false; + if (!(colormask & 0x8)) + has_alpha = false; - radeon_set_context_reg(cmd_buffer->cs, R_0286D0_SPI_PS_INPUT_ADDR, - ps->config.spi_ps_input_addr); + if (spi_format == V_028714_SPI_SHADER_ZERO) { + has_rgb = false; + has_alpha = false; + } - if (ps->info.info.ps.force_persample) - spi_baryc_cntl |= S_0286E0_POS_FLOAT_LOCATION(2); + /* Disable value checking for disabled channels. */ + if (!has_rgb) + sx_blend_opt_control |= S_02875C_MRT0_COLOR_OPT_DISABLE(1) << (i * 4); + if (!has_alpha) + sx_blend_opt_control |= S_02875C_MRT0_ALPHA_OPT_DISABLE(1) << (i * 4); + + /* Enable down-conversion for 32bpp and smaller formats. */ + switch (format) { + case V_028C70_COLOR_8: + case V_028C70_COLOR_8_8: + case V_028C70_COLOR_8_8_8_8: + /* For 1 and 2-channel formats, use the superset thereof. */ + if (spi_format == V_028714_SPI_SHADER_FP16_ABGR || + spi_format == V_028714_SPI_SHADER_UINT16_ABGR || + spi_format == V_028714_SPI_SHADER_SINT16_ABGR) { + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_8_8_8_8 << (i * 4); + sx_blend_opt_epsilon |= V_028758_8BIT_FORMAT << (i * 4); + } + break; - radeon_set_context_reg(cmd_buffer->cs, R_0286D8_SPI_PS_IN_CONTROL, - S_0286D8_NUM_INTERP(ps->info.fs.num_interp)); + case V_028C70_COLOR_5_6_5: + if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) { + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_5_6_5 << (i * 4); + sx_blend_opt_epsilon |= V_028758_6BIT_FORMAT << (i * 4); + } + break; - radeon_set_context_reg(cmd_buffer->cs, R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl); + case V_028C70_COLOR_1_5_5_5: + if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) { + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_1_5_5_5 << (i * 4); + sx_blend_opt_epsilon |= V_028758_5BIT_FORMAT << (i * 4); + } + break; - radeon_set_context_reg(cmd_buffer->cs, R_028710_SPI_SHADER_Z_FORMAT, - pipeline->graphics.shader_z_format); + case V_028C70_COLOR_4_4_4_4: + if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) { + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_4_4_4_4 << (i * 4); + sx_blend_opt_epsilon |= V_028758_4BIT_FORMAT << (i * 4); + } + break; - radeon_set_context_reg(cmd_buffer->cs, R_028714_SPI_SHADER_COL_FORMAT, blend->spi_shader_col_format); + case V_028C70_COLOR_32: + if (swap == V_028C70_SWAP_STD && + spi_format == V_028714_SPI_SHADER_32_R) + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_R << (i * 4); + else if (swap == V_028C70_SWAP_ALT_REV && + spi_format == V_028714_SPI_SHADER_32_AR) + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_A << (i * 4); + break; - radeon_set_context_reg(cmd_buffer->cs, R_028238_CB_TARGET_MASK, blend->cb_target_mask); - radeon_set_context_reg(cmd_buffer->cs, R_02823C_CB_SHADER_MASK, blend->cb_shader_mask); + case V_028C70_COLOR_16: + case V_028C70_COLOR_16_16: + /* For 1-channel formats, use the superset thereof. */ + if (spi_format == V_028714_SPI_SHADER_UNORM16_ABGR || + spi_format == V_028714_SPI_SHADER_SNORM16_ABGR || + spi_format == V_028714_SPI_SHADER_UINT16_ABGR || + spi_format == V_028714_SPI_SHADER_SINT16_ABGR) { + if (swap == V_028C70_SWAP_STD || + swap == V_028C70_SWAP_STD_REV) + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_GR << (i * 4); + else + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_AR << (i * 4); + } + break; - if (cmd_buffer->device->dfsm_allowed) { - /* optimise this? */ - radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); - radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_FLUSH_DFSM) | EVENT_INDEX(0)); - } + case V_028C70_COLOR_10_11_11: + if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) { + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_10_11_11 << (i * 4); + sx_blend_opt_epsilon |= V_028758_11BIT_FORMAT << (i * 4); + } + break; - if (pipeline->graphics.ps_input_cntl_num) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028644_SPI_PS_INPUT_CNTL_0, pipeline->graphics.ps_input_cntl_num); - for (unsigned i = 0; i < pipeline->graphics.ps_input_cntl_num; i++) { - radeon_emit(cmd_buffer->cs, pipeline->graphics.ps_input_cntl[i]); + case V_028C70_COLOR_2_10_10_10: + if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) { + sx_ps_downconvert |= V_028754_SX_RT_EXPORT_2_10_10_10 << (i * 4); + sx_blend_opt_epsilon |= V_028758_10BIT_FORMAT << (i * 4); + } + break; } } -} -static void -radv_emit_vgt_vertex_reuse(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radeon_winsys_cs *cs = cmd_buffer->cs; - - if (cmd_buffer->device->physical_device->rad_info.family < CHIP_POLARIS10) - return; - - radeon_set_context_reg(cs, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, - pipeline->graphics.vtx_reuse_depth); -} - -static void -radv_emit_binning_state(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radeon_winsys_cs *cs = cmd_buffer->cs; - - if (cmd_buffer->device->physical_device->rad_info.chip_class < GFX9) - return; - - radeon_set_context_reg(cs, R_028C44_PA_SC_BINNER_CNTL_0, - pipeline->graphics.bin.pa_sc_binner_cntl_0); - radeon_set_context_reg(cs, R_028060_DB_DFSM_CONTROL, - pipeline->graphics.bin.db_dfsm_control); + radeon_set_context_reg_seq(cmd_buffer->cs, R_028754_SX_PS_DOWNCONVERT, 3); + radeon_emit(cmd_buffer->cs, sx_ps_downconvert); + radeon_emit(cmd_buffer->cs, sx_blend_opt_epsilon); + radeon_emit(cmd_buffer->cs, sx_blend_opt_control); } static void @@ -1076,40 +850,30 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) if (!pipeline || cmd_buffer->state.emitted_pipeline == pipeline) return; - radv_emit_graphics_depth_stencil_state(cmd_buffer, pipeline); - radv_emit_graphics_blend_state(cmd_buffer, pipeline); - radv_emit_graphics_raster_state(cmd_buffer, pipeline); radv_update_multisample_state(cmd_buffer, pipeline); - radv_emit_vertex_shader(cmd_buffer, pipeline); - radv_emit_tess_shaders(cmd_buffer, pipeline); - radv_emit_geometry_shader(cmd_buffer, pipeline); - radv_emit_fragment_shader(cmd_buffer, pipeline); - radv_emit_vgt_vertex_reuse(cmd_buffer, pipeline); - radv_emit_binning_state(cmd_buffer, pipeline); cmd_buffer->scratch_size_needed = MAX2(cmd_buffer->scratch_size_needed, pipeline->max_waves * pipeline->scratch_bytes_per_wave); - radeon_set_context_reg(cmd_buffer->cs, R_0286E8_SPI_TMPRING_SIZE, - S_0286E8_WAVES(pipeline->max_waves) | - S_0286E8_WAVESIZE(pipeline->scratch_bytes_per_wave >> 10)); - if (!cmd_buffer->state.emitted_pipeline || cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband != pipeline->graphics.can_use_guardband) cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SCISSOR; - radeon_set_context_reg(cmd_buffer->cs, R_028B54_VGT_SHADER_STAGES_EN, pipeline->graphics.vgt_shader_stages_en); + radeon_emit_array(cmd_buffer->cs, pipeline->cs.buf, pipeline->cs.cdw); - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { - radeon_set_uconfig_reg_idx(cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, pipeline->graphics.prim); - } else { - radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, pipeline->graphics.prim); + for (unsigned i = 0; i < MESA_SHADER_COMPUTE; i++) { + if (!pipeline->shaders[i]) + continue; + + radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, + pipeline->shaders[i]->bo); } - radeon_set_context_reg(cmd_buffer->cs, R_028A6C_VGT_GS_OUT_PRIM_TYPE, pipeline->graphics.gs_out); - radeon_set_context_reg(cmd_buffer->cs, R_02820C_PA_SC_CLIPRECT_RULE, pipeline->graphics.pa_sc_cliprect_rule); + if (radv_pipeline_has_gs(pipeline)) + radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, + pipeline->gs_copy_shader->bo); if (unlikely(cmd_buffer->device->trace_bo)) radv_save_pipeline(cmd_buffer, pipeline, RING_GFX); @@ -1131,20 +895,10 @@ radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) { uint32_t count = cmd_buffer->state.dynamic.scissor.count; - /* Vega10/Raven scissor bug workaround. This must be done before VPORT - * scissor registers are changed. There is also a more efficient but - * more involved alternative workaround. - */ - if (cmd_buffer->device->physical_device->has_scissor_bug) { - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH; - si_emit_cache_flush(cmd_buffer); - } si_write_scissors(cmd_buffer->cs, 0, count, cmd_buffer->state.dynamic.scissor.scissors, cmd_buffer->state.dynamic.viewport.viewports, cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband); - radeon_set_context_reg(cmd_buffer->cs, R_028A48_PA_SC_MODE_CNTL_0, - cmd_buffer->state.pipeline->graphics.ms.pa_sc_mode_cntl_0 | S_028A48_VPORT_SCISSOR_ENABLE(count ? 1 : 0)); } static void @@ -1212,22 +966,20 @@ radv_emit_depth_bounds(struct radv_cmd_buffer *cmd_buffer) } static void -radv_emit_depth_biais(struct radv_cmd_buffer *cmd_buffer) +radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer) { - struct radv_raster_state *raster = &cmd_buffer->state.pipeline->graphics.raster; struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; unsigned slope = fui(d->depth_bias.slope * 16.0f); unsigned bias = fui(d->depth_bias.bias * cmd_buffer->state.offset_scale); - if (G_028814_POLY_OFFSET_FRONT_ENABLE(raster->pa_su_sc_mode_cntl)) { - radeon_set_context_reg_seq(cmd_buffer->cs, - R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); - radeon_emit(cmd_buffer->cs, fui(d->depth_bias.clamp)); /* CLAMP */ - radeon_emit(cmd_buffer->cs, slope); /* FRONT SCALE */ - radeon_emit(cmd_buffer->cs, bias); /* FRONT OFFSET */ - radeon_emit(cmd_buffer->cs, slope); /* BACK SCALE */ - radeon_emit(cmd_buffer->cs, bias); /* BACK OFFSET */ - } + + radeon_set_context_reg_seq(cmd_buffer->cs, + R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); + radeon_emit(cmd_buffer->cs, fui(d->depth_bias.clamp)); /* CLAMP */ + radeon_emit(cmd_buffer->cs, slope); /* FRONT SCALE */ + radeon_emit(cmd_buffer->cs, bias); /* FRONT OFFSET */ + radeon_emit(cmd_buffer->cs, slope); /* BACK SCALE */ + radeon_emit(cmd_buffer->cs, bias); /* BACK OFFSET */ } static void @@ -1251,20 +1003,20 @@ radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); radeon_emit(cmd_buffer->cs, cb->cb_color_base); - radeon_emit(cmd_buffer->cs, cb->cb_color_base >> 32); + radeon_emit(cmd_buffer->cs, S_028C64_BASE_256B(cb->cb_color_base >> 32)); radeon_emit(cmd_buffer->cs, cb->cb_color_attrib2); radeon_emit(cmd_buffer->cs, cb->cb_color_view); radeon_emit(cmd_buffer->cs, cb_color_info); radeon_emit(cmd_buffer->cs, cb->cb_color_attrib); radeon_emit(cmd_buffer->cs, cb->cb_dcc_control); radeon_emit(cmd_buffer->cs, cb->cb_color_cmask); - radeon_emit(cmd_buffer->cs, cb->cb_color_cmask >> 32); + radeon_emit(cmd_buffer->cs, S_028C80_BASE_256B(cb->cb_color_cmask >> 32)); radeon_emit(cmd_buffer->cs, cb->cb_color_fmask); - radeon_emit(cmd_buffer->cs, cb->cb_color_fmask >> 32); + radeon_emit(cmd_buffer->cs, S_028C88_BASE_256B(cb->cb_color_fmask >> 32)); radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2); radeon_emit(cmd_buffer->cs, cb->cb_dcc_base); - radeon_emit(cmd_buffer->cs, cb->cb_dcc_base >> 32); + radeon_emit(cmd_buffer->cs, S_028C98_BASE_256B(cb->cb_dcc_base >> 32)); radeon_set_context_reg(cmd_buffer->cs, R_0287A0_CB_MRT0_EPITCH + index * 4, S_0287A0_EPITCH(att->attachment->image->surface.u.gfx9.surf.epitch)); @@ -1288,6 +1040,68 @@ radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, } } +static void +radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, + struct radv_ds_buffer_info *ds, + struct radv_image *image, VkImageLayout layout, + bool requires_cond_write) +{ + uint32_t db_z_info = ds->db_z_info; + uint32_t db_z_info_reg; + + if (!radv_image_is_tc_compat_htile(image)) + return; + + if (!radv_layout_has_htile(image, layout, + radv_image_queue_family_mask(image, + cmd_buffer->queue_family_index, + cmd_buffer->queue_family_index))) { + db_z_info &= C_028040_TILE_SURFACE_ENABLE; + } + + db_z_info &= C_028040_ZRANGE_PRECISION; + + if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { + db_z_info_reg = R_028038_DB_Z_INFO; + } else { + db_z_info_reg = R_028040_DB_Z_INFO; + } + + /* When we don't know the last fast clear value we need to emit a + * conditional packet, otherwise we can update DB_Z_INFO directly. + */ + if (requires_cond_write) { + radeon_emit(cmd_buffer->cs, PKT3(PKT3_COND_WRITE, 7, 0)); + + const uint32_t write_space = 0 << 8; /* register */ + const uint32_t poll_space = 1 << 4; /* memory */ + const uint32_t function = 3 << 0; /* equal to the reference */ + const uint32_t options = write_space | poll_space | function; + radeon_emit(cmd_buffer->cs, options); + + /* poll address - location of the depth clear value */ + uint64_t va = radv_buffer_get_va(image->bo); + va += image->offset + image->clear_value_offset; + + /* In presence of stencil format, we have to adjust the base + * address because the first value is the stencil clear value. + */ + if (vk_format_is_stencil(image->vk_format)) + va += 4; + + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); + + radeon_emit(cmd_buffer->cs, fui(0.0f)); /* reference value */ + radeon_emit(cmd_buffer->cs, (uint32_t)-1); /* comparison mask */ + radeon_emit(cmd_buffer->cs, db_z_info_reg >> 2); /* write address low */ + radeon_emit(cmd_buffer->cs, 0u); /* write address high */ + radeon_emit(cmd_buffer->cs, db_z_info); + } else { + radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info); + } +} + static void radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds, @@ -1312,20 +1126,20 @@ radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { radeon_set_context_reg_seq(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, 3); radeon_emit(cmd_buffer->cs, ds->db_htile_data_base); - radeon_emit(cmd_buffer->cs, ds->db_htile_data_base >> 32); + radeon_emit(cmd_buffer->cs, S_028018_BASE_HI(ds->db_htile_data_base >> 32)); radeon_emit(cmd_buffer->cs, ds->db_depth_size); radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 10); radeon_emit(cmd_buffer->cs, db_z_info); /* DB_Z_INFO */ radeon_emit(cmd_buffer->cs, db_stencil_info); /* DB_STENCIL_INFO */ radeon_emit(cmd_buffer->cs, ds->db_z_read_base); /* DB_Z_READ_BASE */ - radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32); /* DB_Z_READ_BASE_HI */ + radeon_emit(cmd_buffer->cs, S_028044_BASE_HI(ds->db_z_read_base >> 32)); /* DB_Z_READ_BASE_HI */ radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); /* DB_STENCIL_READ_BASE */ - radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32); /* DB_STENCIL_READ_BASE_HI */ + radeon_emit(cmd_buffer->cs, S_02804C_BASE_HI(ds->db_stencil_read_base >> 32)); /* DB_STENCIL_READ_BASE_HI */ radeon_emit(cmd_buffer->cs, ds->db_z_write_base); /* DB_Z_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, ds->db_z_write_base >> 32); /* DB_Z_WRITE_BASE_HI */ + radeon_emit(cmd_buffer->cs, S_028054_BASE_HI(ds->db_z_write_base >> 32)); /* DB_Z_WRITE_BASE_HI */ radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base); /* DB_STENCIL_WRITE_BASE */ - radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base >> 32); /* DB_STENCIL_WRITE_BASE_HI */ + radeon_emit(cmd_buffer->cs, S_02805C_BASE_HI(ds->db_stencil_write_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */ radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_INFO2, 2); radeon_emit(cmd_buffer->cs, ds->db_z_info2); @@ -1346,21 +1160,70 @@ radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, } + /* Update the ZRANGE_PRECISION value for the TC-compat bug. */ + radv_update_zrange_precision(cmd_buffer, ds, image, layout, true); + radeon_set_context_reg(cmd_buffer->cs, R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL, ds->pa_su_poly_offset_db_fmt_cntl); } -void -radv_set_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, - VkClearDepthStencilValue ds_clear_value, - VkImageAspectFlags aspects) +/** + * Update the fast clear depth/stencil values if the image is bound as a + * depth/stencil buffer. + */ +static void +radv_update_bound_fast_clear_ds(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + VkClearDepthStencilValue ds_clear_value, + VkImageAspectFlags aspects) { + struct radv_framebuffer *framebuffer = cmd_buffer->state.framebuffer; + const struct radv_subpass *subpass = cmd_buffer->state.subpass; + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_attachment_info *att; + uint32_t att_idx; + + if (!framebuffer || !subpass) + return; + + att_idx = subpass->depth_stencil_attachment.attachment; + if (att_idx == VK_ATTACHMENT_UNUSED) + return; + + att = &framebuffer->attachments[att_idx]; + if (att->attachment->image != image) + return; + + radeon_set_context_reg_seq(cs, R_028028_DB_STENCIL_CLEAR, 2); + radeon_emit(cs, ds_clear_value.stencil); + radeon_emit(cs, fui(ds_clear_value.depth)); + + /* Update the ZRANGE_PRECISION value for the TC-compat bug. This is + * only needed when clearing Z to 0.0. + */ + if ((aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && + ds_clear_value.depth == 0.0) { + VkImageLayout layout = subpass->depth_stencil_attachment.layout; + + radv_update_zrange_precision(cmd_buffer, &att->ds, image, + layout, false); + } +} + +/** + * Set the clear depth/stencil values to the image's metadata. + */ +static void +radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + VkClearDepthStencilValue ds_clear_value, + VkImageAspectFlags aspects) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(image->bo); - va += image->offset + image->clear_value_offset; unsigned reg_offset = 0, reg_count = 0; - assert(image->surface.htile_size); + va += image->offset + image->clear_value_offset; if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { ++reg_count; @@ -1371,34 +1234,50 @@ radv_set_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer, if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) ++reg_count; - radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + reg_count, 0)); - radeon_emit(cmd_buffer->cs, S_370_DST_SEL(V_370_MEM_ASYNC) | - S_370_WR_CONFIRM(1) | - S_370_ENGINE_SEL(V_370_PFP)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); + radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + reg_count, 0)); + radeon_emit(cs, S_370_DST_SEL(V_370_MEM_ASYNC) | + S_370_WR_CONFIRM(1) | + S_370_ENGINE_SEL(V_370_PFP)); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) - radeon_emit(cmd_buffer->cs, ds_clear_value.stencil); + radeon_emit(cs, ds_clear_value.stencil); if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) - radeon_emit(cmd_buffer->cs, fui(ds_clear_value.depth)); + radeon_emit(cs, fui(ds_clear_value.depth)); +} - radeon_set_context_reg_seq(cmd_buffer->cs, R_028028_DB_STENCIL_CLEAR + 4 * reg_offset, reg_count); - if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) - radeon_emit(cmd_buffer->cs, ds_clear_value.stencil); /* R_028028_DB_STENCIL_CLEAR */ - if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) - radeon_emit(cmd_buffer->cs, fui(ds_clear_value.depth)); /* R_02802C_DB_DEPTH_CLEAR */ +/** + * Update the clear depth/stencil values for this image. + */ +void +radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + VkClearDepthStencilValue ds_clear_value, + VkImageAspectFlags aspects) +{ + assert(radv_image_has_htile(image)); + + radv_set_ds_clear_metadata(cmd_buffer, image, ds_clear_value, aspects); + + radv_update_bound_fast_clear_ds(cmd_buffer, image, ds_clear_value, + aspects); } +/** + * Load the clear depth/stencil values from the image's metadata. + */ static void -radv_load_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image) +radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image) { + struct radeon_cmdbuf *cs = cmd_buffer->cs; VkImageAspectFlags aspects = vk_format_aspects(image->vk_format); uint64_t va = radv_buffer_get_va(image->bo); - va += image->offset + image->clear_value_offset; unsigned reg_offset = 0, reg_count = 0; - if (!image->surface.htile_size) + va += image->offset + image->clear_value_offset; + + if (!radv_image_has_htile(image)) return; if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { @@ -1410,21 +1289,21 @@ radv_load_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer, if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) ++reg_count; - radeon_emit(cmd_buffer->cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cmd_buffer->cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) | - COPY_DATA_DST_SEL(COPY_DATA_REG) | - (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); - radeon_emit(cmd_buffer->cs, (R_028028_DB_STENCIL_CLEAR + 4 * reg_offset) >> 2); - radeon_emit(cmd_buffer->cs, 0); + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) | + COPY_DATA_DST_SEL(COPY_DATA_REG) | + (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0)); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + radeon_emit(cs, (R_028028_DB_STENCIL_CLEAR + 4 * reg_offset) >> 2); + radeon_emit(cs, 0); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); - radeon_emit(cmd_buffer->cs, 0); + radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0)); + radeon_emit(cs, 0); } /* - *with DCC some colors don't require CMASK elimiation before being + * With DCC some colors don't require CMASK elimination before being * used as a texture. This sets a predicate value to determine if the * cmask eliminate is required. */ @@ -1437,7 +1316,7 @@ radv_set_dcc_need_cmask_elim_pred(struct radv_cmd_buffer *cmd_buffer, uint64_t va = radv_buffer_get_va(image->bo); va += image->offset + image->dcc_pred_offset; - assert(image->surface.dcc_size); + assert(radv_image_has_dcc(image)); radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 4, 0)); radeon_emit(cmd_buffer->cs, S_370_DST_SEL(V_370_MEM_ASYNC) | @@ -1449,55 +1328,108 @@ radv_set_dcc_need_cmask_elim_pred(struct radv_cmd_buffer *cmd_buffer, radeon_emit(cmd_buffer->cs, pred_val >> 32); } -void -radv_set_color_clear_regs(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, - int idx, - uint32_t color_values[2]) +/** + * Update the fast clear color values if the image is bound as a color buffer. + */ +static void +radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + int cb_idx, + uint32_t color_values[2]) +{ + struct radv_framebuffer *framebuffer = cmd_buffer->state.framebuffer; + const struct radv_subpass *subpass = cmd_buffer->state.subpass; + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_attachment_info *att; + uint32_t att_idx; + + if (!framebuffer || !subpass) + return; + + att_idx = subpass->color_attachments[cb_idx].attachment; + if (att_idx == VK_ATTACHMENT_UNUSED) + return; + + att = &framebuffer->attachments[att_idx]; + if (att->attachment->image != image) + return; + + radeon_set_context_reg_seq(cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2); + radeon_emit(cs, color_values[0]); + radeon_emit(cs, color_values[1]); +} + +/** + * Set the clear color values to the image's metadata. + */ +static void +radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + uint32_t color_values[2]) { + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(image->bo); + va += image->offset + image->clear_value_offset; - assert(image->cmask.size || image->surface.dcc_size); + assert(radv_image_has_cmask(image) || radv_image_has_dcc(image)); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 4, 0)); - radeon_emit(cmd_buffer->cs, S_370_DST_SEL(V_370_MEM_ASYNC) | - S_370_WR_CONFIRM(1) | - S_370_ENGINE_SEL(V_370_PFP)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); - radeon_emit(cmd_buffer->cs, color_values[0]); - radeon_emit(cmd_buffer->cs, color_values[1]); + radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 4, 0)); + radeon_emit(cs, S_370_DST_SEL(V_370_MEM_ASYNC) | + S_370_WR_CONFIRM(1) | + S_370_ENGINE_SEL(V_370_PFP)); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + radeon_emit(cs, color_values[0]); + radeon_emit(cs, color_values[1]); +} - radeon_set_context_reg_seq(cmd_buffer->cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + idx * 0x3c, 2); - radeon_emit(cmd_buffer->cs, color_values[0]); - radeon_emit(cmd_buffer->cs, color_values[1]); +/** + * Update the clear color values for this image. + */ +void +radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + int cb_idx, + uint32_t color_values[2]) +{ + assert(radv_image_has_cmask(image) || radv_image_has_dcc(image)); + + radv_set_color_clear_metadata(cmd_buffer, image, color_values); + + radv_update_bound_fast_clear_color(cmd_buffer, image, cb_idx, + color_values); } +/** + * Load the clear color values from the image's metadata. + */ static void -radv_load_color_clear_regs(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, - int idx) +radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + int cb_idx) { + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(image->bo); + va += image->offset + image->clear_value_offset; - if (!image->cmask.size && !image->surface.dcc_size) + if (!radv_image_has_cmask(image) && !radv_image_has_dcc(image)) return; - uint32_t reg = R_028C8C_CB_COLOR0_CLEAR_WORD0 + idx * 0x3c; + uint32_t reg = R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c; - radeon_emit(cmd_buffer->cs, PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) | - COPY_DATA_DST_SEL(COPY_DATA_REG) | - COPY_DATA_COUNT_SEL); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); - radeon_emit(cmd_buffer->cs, reg >> 2); - radeon_emit(cmd_buffer->cs, 0); + radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); + radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) | + COPY_DATA_DST_SEL(COPY_DATA_REG) | + COPY_DATA_COUNT_SEL); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + radeon_emit(cs, reg >> 2); + radeon_emit(cs, 0); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); - radeon_emit(cmd_buffer->cs, 0); + radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); + radeon_emit(cs, 0); } static void @@ -1523,12 +1455,12 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) struct radv_image *image = att->attachment->image; VkImageLayout layout = subpass->color_attachments[i].layout; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, att->attachment->bo, 8); + radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, att->attachment->bo); assert(att->attachment->aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT); radv_emit_fb_color_state(cmd_buffer, i, att, image, layout); - radv_load_color_clear_regs(cmd_buffer, image, i); + radv_load_color_clear_metadata(cmd_buffer, image, i); } if(subpass->depth_stencil_attachment.attachment != VK_ATTACHMENT_UNUSED) { @@ -1536,7 +1468,7 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) VkImageLayout layout = subpass->depth_stencil_attachment.layout; struct radv_attachment_info *att = &framebuffer->attachments[idx]; struct radv_image *image = att->attachment->image; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, att->attachment->bo, 8); + radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, att->attachment->bo); MAYBE_UNUSED uint32_t queue_mask = radv_image_queue_family_mask(image, cmd_buffer->queue_family_index, cmd_buffer->queue_family_index); @@ -1550,7 +1482,7 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS; cmd_buffer->state.offset_scale = att->ds.offset_scale; } - radv_load_depth_clear_regs(cmd_buffer, image); + radv_load_ds_clear_metadata(cmd_buffer, image); } else { if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 2); @@ -1575,7 +1507,7 @@ radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) static void radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) { - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; struct radv_cmd_state *state = &cmd_buffer->state; if (state->index_type != state->last_index_type) { @@ -1602,24 +1534,59 @@ radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer) void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer) { + bool has_perfect_queries = cmd_buffer->state.perfect_occlusion_queries_enabled; + struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; + uint32_t pa_sc_mode_cntl_1 = + pipeline ? pipeline->graphics.ms.pa_sc_mode_cntl_1 : 0; uint32_t db_count_control; if(!cmd_buffer->state.active_occlusion_queries) { if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { + if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) && + pipeline->graphics.disable_out_of_order_rast_for_occlusion && + has_perfect_queries) { + /* Re-enable out-of-order rasterization if the + * bound pipeline supports it and if it's has + * been disabled before starting any perfect + * occlusion queries. + */ + radeon_set_context_reg(cmd_buffer->cs, + R_028A4C_PA_SC_MODE_CNTL_1, + pa_sc_mode_cntl_1); + } db_count_control = 0; } else { db_count_control = S_028004_ZPASS_INCREMENT_DISABLE(1); } } else { + const struct radv_subpass *subpass = cmd_buffer->state.subpass; + uint32_t sample_rate = subpass ? util_logbase2(subpass->max_sample_count) : 0; + if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { - db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) | - S_028004_SAMPLE_RATE(0) | /* TODO: set this to the number of samples of the current framebuffer */ + db_count_control = + S_028004_PERFECT_ZPASS_COUNTS(has_perfect_queries) | + S_028004_SAMPLE_RATE(sample_rate) | S_028004_ZPASS_ENABLE(1) | S_028004_SLICE_EVEN_ENABLE(1) | S_028004_SLICE_ODD_ENABLE(1); + + if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) && + pipeline->graphics.disable_out_of_order_rast_for_occlusion && + has_perfect_queries) { + /* If the bound pipeline has enabled + * out-of-order rasterization, we should + * disable it before starting any perfect + * occlusion queries. + */ + pa_sc_mode_cntl_1 &= C_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE; + + radeon_set_context_reg(cmd_buffer->cs, + R_028A4C_PA_SC_MODE_CNTL_1, + pa_sc_mode_cntl_1); + } } else { db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) | - S_028004_SAMPLE_RATE(0); /* TODO: set this to the number of samples of the current framebuffer */ + S_028004_SAMPLE_RATE(sample_rate); } } @@ -1629,85 +1596,45 @@ void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer) static void radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer) { - if (G_028810_DX_RASTERIZATION_KILL(cmd_buffer->state.pipeline->graphics.raster.pa_cl_clip_cntl)) - return; + uint32_t states = cmd_buffer->state.dirty & cmd_buffer->state.emitted_pipeline->graphics.needed_dynamic_state; - if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT)) + if (states & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT)) radv_emit_viewport(cmd_buffer); - if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_SCISSOR | RADV_CMD_DIRTY_DYNAMIC_VIEWPORT)) + if (states & (RADV_CMD_DIRTY_DYNAMIC_SCISSOR | RADV_CMD_DIRTY_DYNAMIC_VIEWPORT) && + !cmd_buffer->device->physical_device->has_scissor_bug) radv_emit_scissor(cmd_buffer); - if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH) + if (states & RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH) radv_emit_line_width(cmd_buffer); - if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS) + if (states & RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS) radv_emit_blend_constants(cmd_buffer); - if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE | + if (states & (RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE | RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK | RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK)) radv_emit_stencil(cmd_buffer); - if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS) + if (states & RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS) radv_emit_depth_bounds(cmd_buffer); - if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_PIPELINE | - RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)) - radv_emit_depth_biais(cmd_buffer); + if (states & RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS) + radv_emit_depth_bias(cmd_buffer); - if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE) + if (states & RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE) radv_emit_discard_rectangle(cmd_buffer); - cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_DYNAMIC_ALL; -} - -static void -emit_stage_descriptor_set_userdata(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - int idx, - uint64_t va, - gl_shader_stage stage) -{ - struct ac_userdata_info *desc_set_loc = &pipeline->shaders[stage]->info.user_sgprs_locs.descriptor_sets[idx]; - uint32_t base_reg = pipeline->user_data_0[stage]; - - if (desc_set_loc->sgpr_idx == -1 || desc_set_loc->indirect) - return; - - assert(!desc_set_loc->indirect); - assert(desc_set_loc->num_sgprs == 2); - radeon_set_sh_reg_seq(cmd_buffer->cs, - base_reg + desc_set_loc->sgpr_idx * 4, 2); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); -} - -static void -radv_emit_descriptor_set_userdata(struct radv_cmd_buffer *cmd_buffer, - VkShaderStageFlags stages, - struct radv_descriptor_set *set, - unsigned idx) -{ - if (cmd_buffer->state.pipeline) { - radv_foreach_stage(stage, stages) { - if (cmd_buffer->state.pipeline->shaders[stage]) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.pipeline, - idx, set->va, - stage); - } - } - - if (cmd_buffer->state.compute_pipeline && (stages & VK_SHADER_STAGE_COMPUTE_BIT)) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.compute_pipeline, - idx, set->va, - MESA_SHADER_COMPUTE); + cmd_buffer->state.dirty &= ~states; } static void -radv_flush_push_descriptors(struct radv_cmd_buffer *cmd_buffer) +radv_flush_push_descriptors(struct radv_cmd_buffer *cmd_buffer, + VkPipelineBindPoint bind_point) { - struct radv_descriptor_set *set = &cmd_buffer->push_descriptors.set; + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); + struct radv_descriptor_set *set = &descriptors_state->push_set.set; unsigned bo_offset; if (!radv_cmd_buffer_upload_data(cmd_buffer, set->size, 32, @@ -1720,8 +1647,11 @@ radv_flush_push_descriptors(struct radv_cmd_buffer *cmd_buffer) } static void -radv_flush_indirect_descriptor_sets(struct radv_cmd_buffer *cmd_buffer) +radv_flush_indirect_descriptor_sets(struct radv_cmd_buffer *cmd_buffer, + VkPipelineBindPoint bind_point) { + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); uint32_t size = MAX_SETS * 2 * 4; uint32_t offset; void *ptr; @@ -1733,8 +1663,8 @@ radv_flush_indirect_descriptor_sets(struct radv_cmd_buffer *cmd_buffer) for (unsigned i = 0; i < MAX_SETS; i++) { uint32_t *uptr = ((uint32_t *)ptr) + i * 2; uint64_t set_va = 0; - struct radv_descriptor_set *set = cmd_buffer->descriptors[i]; - if (cmd_buffer->state.valid_descriptors & (1u << i)) + struct radv_descriptor_set *set = descriptors_state->sets[i]; + if (descriptors_state->valid & (1u << i)) set_va = set->va; uptr[0] = set_va & 0xffffffff; uptr[1] = set_va >> 32; @@ -1774,45 +1704,69 @@ static void radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stages) { - unsigned i; + VkPipelineBindPoint bind_point = stages & VK_SHADER_STAGE_COMPUTE_BIT ? + VK_PIPELINE_BIND_POINT_COMPUTE : + VK_PIPELINE_BIND_POINT_GRAPHICS; + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); - if (!cmd_buffer->state.descriptors_dirty) + if (!descriptors_state->dirty) return; - if (cmd_buffer->state.push_descriptors_dirty) - radv_flush_push_descriptors(cmd_buffer); + if (descriptors_state->push_dirty) + radv_flush_push_descriptors(cmd_buffer, bind_point); if ((cmd_buffer->state.pipeline && cmd_buffer->state.pipeline->need_indirect_descriptor_sets) || (cmd_buffer->state.compute_pipeline && cmd_buffer->state.compute_pipeline->need_indirect_descriptor_sets)) { - radv_flush_indirect_descriptor_sets(cmd_buffer); + radv_flush_indirect_descriptor_sets(cmd_buffer, bind_point); } MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MAX_SETS * MESA_SHADER_STAGES * 4); - for_each_bit(i, cmd_buffer->state.descriptors_dirty) { - struct radv_descriptor_set *set = cmd_buffer->descriptors[i]; - if (!(cmd_buffer->state.valid_descriptors & (1u << i))) - continue; + if (cmd_buffer->state.pipeline) { + radv_foreach_stage(stage, stages) { + if (!cmd_buffer->state.pipeline->shaders[stage]) + continue; - radv_emit_descriptor_set_userdata(cmd_buffer, stages, set, i); + radv_emit_descriptor_pointers(cmd_buffer, + cmd_buffer->state.pipeline, + descriptors_state, stage); + } } - cmd_buffer->state.descriptors_dirty = 0; - cmd_buffer->state.push_descriptors_dirty = false; + + if (cmd_buffer->state.compute_pipeline && + (stages & VK_SHADER_STAGE_COMPUTE_BIT)) { + radv_emit_descriptor_pointers(cmd_buffer, + cmd_buffer->state.compute_pipeline, + descriptors_state, + MESA_SHADER_COMPUTE); + } + + descriptors_state->dirty = 0; + descriptors_state->push_dirty = false; if (unlikely(cmd_buffer->device->trace_bo)) - radv_save_descriptors(cmd_buffer); + radv_save_descriptors(cmd_buffer, bind_point); assert(cmd_buffer->cs->cdw <= cdw_max); } static void radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, VkShaderStageFlags stages) { + struct radv_pipeline *pipeline = stages & VK_SHADER_STAGE_COMPUTE_BIT + ? cmd_buffer->state.compute_pipeline + : cmd_buffer->state.pipeline; + VkPipelineBindPoint bind_point = stages & VK_SHADER_STAGE_COMPUTE_BIT ? + VK_PIPELINE_BIND_POINT_COMPUTE : + VK_PIPELINE_BIND_POINT_GRAPHICS; + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); struct radv_pipeline_layout *layout = pipeline->layout; + struct radv_shader_variant *shader, *prev_shader; unsigned offset; void *ptr; uint64_t va; @@ -1828,7 +1782,8 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, return; memcpy(ptr, cmd_buffer->push_constants, layout->push_constant_size); - memcpy((char*)ptr + layout->push_constant_size, cmd_buffer->dynamic_buffers, + memcpy((char*)ptr + layout->push_constant_size, + descriptors_state->dynamic_buffers, 16 * layout->dynamic_offset_count); va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); @@ -1837,10 +1792,16 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MESA_SHADER_STAGES * 4); + prev_shader = NULL; radv_foreach_stage(stage, stages) { - if (pipeline->shaders[stage]) { + shader = radv_get_shader(pipeline, stage); + + /* Avoid redundantly emitting the address for merged stages. */ + if (shader && shader != prev_shader) { radv_emit_userdata_address(cmd_buffer, pipeline, stage, AC_UD_PUSH_CONSTANTS, va); + + prev_shader = shader; } } @@ -1848,13 +1809,14 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, assert(cmd_buffer->cs->cdw <= cdw_max); } -static bool -radv_cmd_buffer_update_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty) +static void +radv_flush_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, + bool pipeline_is_dirty) { if ((pipeline_is_dirty || (cmd_buffer->state.dirty & RADV_CMD_DIRTY_VERTEX_BUFFER)) && cmd_buffer->state.pipeline->vertex_elements.count && - radv_get_vertex_shader(cmd_buffer->state.pipeline)->info.info.vs.has_vertex_buffers) { + radv_get_shader(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX)->info.info.vs.has_vertex_buffers) { struct radv_vertex_elements_info *velems = &cmd_buffer->state.pipeline->vertex_elements; unsigned vb_offset; void *vb_ptr; @@ -1865,7 +1827,7 @@ radv_cmd_buffer_update_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, bo /* allocate some descriptor state for vertex buffers */ if (!radv_cmd_buffer_upload_alloc(cmd_buffer, count * 16, 256, &vb_offset, &vb_ptr)) - return false; + return; for (i = 0; i < count; i++) { uint32_t *desc = &((uint32_t *)vb_ptr)[i * 4]; @@ -1895,24 +1857,17 @@ radv_cmd_buffer_update_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, bo cmd_buffer->state.vb_va = va; cmd_buffer->state.vb_size = count * 16; - cmd_buffer->state.vb_prefetch_dirty = true; + cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_VBO_DESCRIPTORS; } cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_VERTEX_BUFFER; - - return true; } -static bool +static void radv_upload_graphics_shader_descriptors(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty) { - if (!radv_cmd_buffer_update_vertex_descriptors(cmd_buffer, pipeline_is_dirty)) - return false; - + radv_flush_vertex_descriptors(cmd_buffer, pipeline_is_dirty); radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS); - radv_flush_constants(cmd_buffer, cmd_buffer->state.pipeline, - VK_SHADER_STAGE_ALL_GRAPHICS); - - return true; + radv_flush_constants(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS); } static void @@ -1922,7 +1877,7 @@ radv_emit_draw_registers(struct radv_cmd_buffer *cmd_buffer, bool indexed_draw, { struct radeon_info *info = &cmd_buffer->device->physical_device->rad_info; struct radv_cmd_state *state = &cmd_buffer->state; - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint32_t ia_multi_vgt_param; int32_t primitive_reset_en; @@ -2008,29 +1963,42 @@ static void radv_stage_flush(struct radv_cmd_buffer *cmd_buffer, static enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, - VkAccessFlags src_flags) + VkAccessFlags src_flags, + struct radv_image *image) { + bool flush_CB_meta = true, flush_DB_meta = true; enum radv_cmd_flush_bits flush_bits = 0; uint32_t b; + + if (image && !radv_image_has_CB_metadata(image)) + flush_CB_meta = false; + if (image && !radv_image_has_htile(image)) + flush_DB_meta = false; + for_each_bit(b, src_flags) { switch ((VkAccessFlagBits)(1 << b)) { case VK_ACCESS_SHADER_WRITE_BIT: flush_bits |= RADV_CMD_FLAG_WRITEBACK_GLOBAL_L2; break; case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT: - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | - RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB; + if (flush_CB_meta) + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; break; case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT: - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB; + if (flush_DB_meta) + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; break; case VK_ACCESS_TRANSFER_WRITE_BIT: flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | - RADV_CMD_FLAG_FLUSH_AND_INV_CB_META | RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META | RADV_CMD_FLAG_INV_GLOBAL_L2; + + if (flush_CB_meta) + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; + if (flush_DB_meta) + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; break; default: break; @@ -2082,14 +2050,15 @@ radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, static void radv_subpass_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_subpass_barrier *barrier) { - cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, barrier->src_access_mask); + cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, barrier->src_access_mask, + NULL); radv_stage_flush(cmd_buffer, barrier->src_stage_mask); cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask, NULL); } static void radv_handle_subpass_image_transition(struct radv_cmd_buffer *cmd_buffer, - VkAttachmentReference att) + struct radv_subpass_attachment att) { unsigned idx = att.attachment; struct radv_image_view *view = cmd_buffer->state.framebuffer->attachments[idx].attachment; @@ -2284,7 +2253,7 @@ static void emit_gfx_buffer_state(struct radv_cmd_buffer *cmd_buffer) struct radv_device *device = cmd_buffer->device; if (device->gfx_init) { uint64_t va = radv_buffer_get_va(device->gfx_init); - radv_cs_add_buffer(device->ws, cmd_buffer->cs, device->gfx_init, 8); + radv_cs_add_buffer(device->ws, cmd_buffer->cs, device->gfx_init); radeon_emit(cmd_buffer->cs, PKT3(PKT3_INDIRECT_BUFFER_CIK, 2, 0)); radeon_emit(cmd_buffer->cs, va); radeon_emit(cmd_buffer->cs, va >> 32); @@ -2312,6 +2281,9 @@ VkResult radv_BeginCommandBuffer( memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state)); cmd_buffer->state.last_primitive_reset_en = -1; cmd_buffer->state.last_index_type = -1; + cmd_buffer->state.last_num_instances = -1; + cmd_buffer->state.last_vertex_offset = -1; + cmd_buffer->state.last_first_instance = -1; cmd_buffer->usage_flags = pBeginInfo->flags; /* setup initial configuration into command buffer */ @@ -2329,7 +2301,8 @@ VkResult radv_BeginCommandBuffer( } } - if (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) { + if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY && + (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) { assert(pBeginInfo->pInheritanceInfo); cmd_buffer->state.framebuffer = radv_framebuffer_from_handle(pBeginInfo->pInheritanceInfo->framebuffer); cmd_buffer->state.pass = radv_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass); @@ -2344,8 +2317,14 @@ VkResult radv_BeginCommandBuffer( radv_cmd_buffer_set_subpass(cmd_buffer, subpass, false); } - if (unlikely(cmd_buffer->device->trace_bo)) + if (unlikely(cmd_buffer->device->trace_bo)) { + struct radv_device *device = cmd_buffer->device; + + radv_cs_add_buffer(device->ws, cmd_buffer->cs, + device->trace_bo); + radv_cmd_buffer_trace_emit(cmd_buffer); + } cmd_buffer->status = RADV_CMD_BUFFER_STATUS_RECORDING; @@ -2380,7 +2359,7 @@ void radv_CmdBindVertexBuffers( vb[idx].offset = pOffsets[i]; radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, - vb[idx].buffer->bo, 8); + vb[idx].buffer->bo); } if (!changed) { @@ -2416,28 +2395,30 @@ void radv_CmdBindIndexBuffer( int index_size_shift = cmd_buffer->state.index_type ? 2 : 1; cmd_buffer->state.max_index_count = (index_buffer->size - offset) >> index_size_shift; cmd_buffer->state.dirty |= RADV_CMD_DIRTY_INDEX_BUFFER; - radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, index_buffer->bo, 8); + radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, index_buffer->bo); } static void radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer, + VkPipelineBindPoint bind_point, struct radv_descriptor_set *set, unsigned idx) { struct radeon_winsys *ws = cmd_buffer->device->ws; - radv_set_descriptor_set(cmd_buffer, set, idx); - if (!set) - return; + radv_set_descriptor_set(cmd_buffer, bind_point, set, idx); + assert(set); assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); - for (unsigned j = 0; j < set->layout->buffer_count; ++j) - if (set->descriptors[j]) - radv_cs_add_buffer(ws, cmd_buffer->cs, set->descriptors[j], 7); + if (!cmd_buffer->device->use_global_bo_list) { + for (unsigned j = 0; j < set->layout->buffer_count; ++j) + if (set->descriptors[j]) + radv_cs_add_buffer(ws, cmd_buffer->cs, set->descriptors[j]); + } if(set->bo) - radv_cs_add_buffer(ws, cmd_buffer->cs, set->bo, 8); + radv_cs_add_buffer(ws, cmd_buffer->cs, set->bo); } void radv_CmdBindDescriptorSets( @@ -2454,21 +2435,25 @@ void radv_CmdBindDescriptorSets( RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); unsigned dyn_idx = 0; + const bool no_dynamic_bounds = cmd_buffer->device->instance->debug_flags & RADV_DEBUG_NO_DYNAMIC_BOUNDS; + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, pipelineBindPoint); + for (unsigned i = 0; i < descriptorSetCount; ++i) { unsigned idx = i + firstSet; RADV_FROM_HANDLE(radv_descriptor_set, set, pDescriptorSets[i]); - radv_bind_descriptor_set(cmd_buffer, set, idx); + radv_bind_descriptor_set(cmd_buffer, pipelineBindPoint, set, idx); for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) { unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start; - uint32_t *dst = cmd_buffer->dynamic_buffers + idx * 4; + uint32_t *dst = descriptors_state->dynamic_buffers + idx * 4; assert(dyn_idx < dynamicOffsetCount); struct radv_descriptor_range *range = set->dynamic_descriptors + j; uint64_t va = range->va + pDynamicOffsets[dyn_idx]; dst[0] = va; dst[1] = S_008F04_BASE_ADDRESS_HI(va >> 32); - dst[2] = range->size; + dst[2] = no_dynamic_bounds ? 0xffffffffu : range->size; dst[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | @@ -2483,26 +2468,29 @@ void radv_CmdBindDescriptorSets( static bool radv_init_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, struct radv_descriptor_set *set, - struct radv_descriptor_set_layout *layout) + struct radv_descriptor_set_layout *layout, + VkPipelineBindPoint bind_point) { + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); set->size = layout->size; set->layout = layout; - if (cmd_buffer->push_descriptors.capacity < set->size) { + if (descriptors_state->push_set.capacity < set->size) { size_t new_size = MAX2(set->size, 1024); - new_size = MAX2(new_size, 2 * cmd_buffer->push_descriptors.capacity); + new_size = MAX2(new_size, 2 * descriptors_state->push_set.capacity); new_size = MIN2(new_size, 96 * MAX_PUSH_DESCRIPTORS); free(set->mapped_ptr); set->mapped_ptr = malloc(new_size); if (!set->mapped_ptr) { - cmd_buffer->push_descriptors.capacity = 0; + descriptors_state->push_set.capacity = 0; cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY; return false; } - cmd_buffer->push_descriptors.capacity = new_size; + descriptors_state->push_set.capacity = new_size; } return true; @@ -2538,7 +2526,7 @@ void radv_meta_push_descriptor_set( radv_descriptor_set_to_handle(push_set), descriptorWriteCount, pDescriptorWrites, 0, NULL); - radv_set_descriptor_set(cmd_buffer, push_set, set); + radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set); } void radv_CmdPushDescriptorSetKHR( @@ -2551,19 +2539,23 @@ void radv_CmdPushDescriptorSetKHR( { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); - struct radv_descriptor_set *push_set = &cmd_buffer->push_descriptors.set; + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, pipelineBindPoint); + struct radv_descriptor_set *push_set = &descriptors_state->push_set.set; assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - if (!radv_init_push_descriptor_set(cmd_buffer, push_set, layout->set[set].layout)) + if (!radv_init_push_descriptor_set(cmd_buffer, push_set, + layout->set[set].layout, + pipelineBindPoint)) return; radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer, radv_descriptor_set_to_handle(push_set), descriptorWriteCount, pDescriptorWrites, 0, NULL); - radv_set_descriptor_set(cmd_buffer, push_set, set); - cmd_buffer->state.push_descriptors_dirty = true; + radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set); + descriptors_state->push_dirty = true; } void radv_CmdPushDescriptorSetWithTemplateKHR( @@ -2575,18 +2567,23 @@ void radv_CmdPushDescriptorSetWithTemplateKHR( { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); - struct radv_descriptor_set *push_set = &cmd_buffer->push_descriptors.set; + RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate); + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, templ->bind_point); + struct radv_descriptor_set *push_set = &descriptors_state->push_set.set; assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - if (!radv_init_push_descriptor_set(cmd_buffer, push_set, layout->set[set].layout)) + if (!radv_init_push_descriptor_set(cmd_buffer, push_set, + layout->set[set].layout, + templ->bind_point)) return; radv_update_descriptor_set_with_template(cmd_buffer->device, cmd_buffer, push_set, descriptorUpdateTemplate, pData); - radv_set_descriptor_set(cmd_buffer, push_set, set); - cmd_buffer->state.push_descriptors_dirty = true; + radv_set_descriptor_set(cmd_buffer, templ->bind_point, push_set, set); + descriptors_state->push_dirty = true; } void radv_CmdPushConstants(VkCommandBuffer commandBuffer, @@ -2612,10 +2609,15 @@ VkResult radv_EndCommandBuffer( si_emit_cache_flush(cmd_buffer); } + /* Make sure CP DMA is idle at the end of IBs because the kernel + * doesn't wait for it. + */ + si_cp_dma_wait_for_idle(cmd_buffer); + vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments); if (!cmd_buffer->device->ws->cs_finalize(cmd_buffer->cs)) - return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY); + return vk_error(cmd_buffer->device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); cmd_buffer->status = RADV_CMD_BUFFER_STATUS_EXECUTABLE; @@ -2625,83 +2627,34 @@ VkResult radv_EndCommandBuffer( static void radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer) { - struct radv_shader_variant *compute_shader; struct radv_pipeline *pipeline = cmd_buffer->state.compute_pipeline; - struct radv_device *device = cmd_buffer->device; - unsigned compute_resource_limits; - unsigned waves_per_threadgroup; - uint64_t va; if (!pipeline || pipeline == cmd_buffer->state.emitted_compute_pipeline) return; cmd_buffer->state.emitted_compute_pipeline = pipeline; - compute_shader = pipeline->shaders[MESA_SHADER_COMPUTE]; - va = radv_buffer_get_va(compute_shader->bo) + compute_shader->bo_offset; - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, 19); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B830_COMPUTE_PGM_LO, 2); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B848_COMPUTE_PGM_RSRC1, 2); - radeon_emit(cmd_buffer->cs, compute_shader->rsrc1); - radeon_emit(cmd_buffer->cs, compute_shader->rsrc2); - - - cmd_buffer->compute_scratch_size_needed = - MAX2(cmd_buffer->compute_scratch_size_needed, - pipeline->max_waves * pipeline->scratch_bytes_per_wave); - - /* change these once we have scratch support */ - radeon_set_sh_reg(cmd_buffer->cs, R_00B860_COMPUTE_TMPRING_SIZE, - S_00B860_WAVES(pipeline->max_waves) | - S_00B860_WAVESIZE(pipeline->scratch_bytes_per_wave >> 10)); - - /* Calculate best compute resource limits. */ - waves_per_threadgroup = - DIV_ROUND_UP(compute_shader->info.cs.block_size[0] * - compute_shader->info.cs.block_size[1] * - compute_shader->info.cs.block_size[2], 64); - compute_resource_limits = - S_00B854_SIMD_DEST_CNTL(waves_per_threadgroup % 4 == 0); - - if (device->physical_device->rad_info.chip_class >= CIK) { - unsigned num_cu_per_se = - device->physical_device->rad_info.num_good_compute_units / - device->physical_device->rad_info.max_se; - - /* Force even distribution on all SIMDs in CU if the workgroup - * size is 64. This has shown some good improvements if # of - * CUs per SE is not a multiple of 4. - */ - if (num_cu_per_se % 4 && waves_per_threadgroup == 1) - compute_resource_limits |= S_00B854_FORCE_SIMD_DIST(1); - } - - radeon_set_sh_reg(cmd_buffer->cs, R_00B854_COMPUTE_RESOURCE_LIMITS, - compute_resource_limits); + radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->cs.cdw); + radeon_emit_array(cmd_buffer->cs, pipeline->cs.buf, pipeline->cs.cdw); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); - radeon_emit(cmd_buffer->cs, - S_00B81C_NUM_THREAD_FULL(compute_shader->info.cs.block_size[0])); - radeon_emit(cmd_buffer->cs, - S_00B81C_NUM_THREAD_FULL(compute_shader->info.cs.block_size[1])); - radeon_emit(cmd_buffer->cs, - S_00B81C_NUM_THREAD_FULL(compute_shader->info.cs.block_size[2])); + cmd_buffer->compute_scratch_size_needed = + MAX2(cmd_buffer->compute_scratch_size_needed, + pipeline->max_waves * pipeline->scratch_bytes_per_wave); - assert(cmd_buffer->cs->cdw <= cdw_max); + radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, + pipeline->shaders[MESA_SHADER_COMPUTE]->bo); if (unlikely(cmd_buffer->device->trace_bo)) radv_save_pipeline(cmd_buffer, pipeline, RING_COMPUTE); } -static void radv_mark_descriptor_sets_dirty(struct radv_cmd_buffer *cmd_buffer) +static void radv_mark_descriptor_sets_dirty(struct radv_cmd_buffer *cmd_buffer, + VkPipelineBindPoint bind_point) { - cmd_buffer->state.descriptors_dirty |= cmd_buffer->state.valid_descriptors; + struct radv_descriptor_state *descriptors_state = + radv_get_descriptors_state(cmd_buffer, bind_point); + + descriptors_state->dirty |= descriptors_state->valid; } void radv_CmdBindPipeline( @@ -2716,7 +2669,7 @@ void radv_CmdBindPipeline( case VK_PIPELINE_BIND_POINT_COMPUTE: if (cmd_buffer->state.compute_pipeline == pipeline) return; - radv_mark_descriptor_sets_dirty(cmd_buffer); + radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint); cmd_buffer->state.compute_pipeline = pipeline; cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_COMPUTE_BIT; @@ -2724,7 +2677,7 @@ void radv_CmdBindPipeline( case VK_PIPELINE_BIND_POINT_GRAPHICS: if (cmd_buffer->state.pipeline == pipeline) return; - radv_mark_descriptor_sets_dirty(cmd_buffer); + radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint); cmd_buffer->state.pipeline = pipeline; if (!pipeline) @@ -2733,6 +2686,13 @@ void radv_CmdBindPipeline( cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE; cmd_buffer->push_constant_stages |= pipeline->active_stages; + /* the new vertex shader might not have the same user regs */ + cmd_buffer->state.last_first_instance = -1; + cmd_buffer->state.last_vertex_offset = -1; + + /* Prefetch all pipeline shaders at first draw time. */ + cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_SHADERS; + radv_bind_dynamic_state(cmd_buffer, &pipeline->dynamic_state); if (pipeline->graphics.esgs_ring_size > cmd_buffer->esgs_ring_size_needed) @@ -2742,15 +2702,6 @@ void radv_CmdBindPipeline( if (radv_pipeline_has_tess(pipeline)) cmd_buffer->tess_rings_needed = true; - - if (radv_pipeline_has_gs(pipeline)) { - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_GEOMETRY, - AC_UD_SCRATCH_RING_OFFSETS); - if (cmd_buffer->ring_offsets_idx == -1) - cmd_buffer->ring_offsets_idx = loc->sgpr_idx; - else if (loc->sgpr_idx != -1) - assert(loc->sgpr_idx == cmd_buffer->ring_offsets_idx); - } break; default: assert(!"invalid bind point"); @@ -2771,18 +2722,6 @@ void radv_CmdSetViewport( assert(firstViewport < MAX_VIEWPORTS); assert(total_count >= 1 && total_count <= MAX_VIEWPORTS); - if (cmd_buffer->device->physical_device->has_scissor_bug) { - /* Try to skip unnecessary PS partial flushes when the viewports - * don't change. - */ - if (!(state->dirty & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT | - RADV_CMD_DIRTY_DYNAMIC_SCISSOR)) && - !memcmp(state->dynamic.viewport.viewports + firstViewport, - pViewports, viewportCount * sizeof(*pViewports))) { - return; - } - } - memcpy(state->dynamic.viewport.viewports + firstViewport, pViewports, viewportCount * sizeof(*pViewports)); @@ -2802,18 +2741,6 @@ void radv_CmdSetScissor( assert(firstScissor < MAX_SCISSORS); assert(total_count >= 1 && total_count <= MAX_SCISSORS); - if (cmd_buffer->device->physical_device->has_scissor_bug) { - /* Try to skip unnecessary PS partial flushes when the scissors - * don't change. - */ - if (!(state->dirty & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT | - RADV_CMD_DIRTY_DYNAMIC_SCISSOR)) && - !memcmp(state->dynamic.scissor.scissors + firstScissor, - pScissors, scissorCount * sizeof(*pScissors))) { - return; - } - } - memcpy(state->dynamic.scissor.scissors + firstScissor, pScissors, scissorCount * sizeof(*pScissors)); @@ -2962,12 +2889,6 @@ void radv_CmdExecuteCommands( if (secondary->sample_positions_needed) primary->sample_positions_needed = true; - if (secondary->ring_offsets_idx != -1) { - if (primary->ring_offsets_idx == -1) - primary->ring_offsets_idx = secondary->ring_offsets_idx; - else - assert(secondary->ring_offsets_idx == primary->ring_offsets_idx); - } primary->device->ws->cs_execute_secondary(primary->cs, secondary->cs); @@ -3003,6 +2924,10 @@ void radv_CmdExecuteCommands( secondary->state.last_ia_multi_vgt_param; } + primary->state.last_first_instance = secondary->state.last_first_instance; + primary->state.last_num_instances = secondary->state.last_num_instances; + primary->state.last_vertex_offset = secondary->state.last_vertex_offset; + if (secondary->state.last_index_type != -1) { primary->state.last_index_type = secondary->state.last_index_type; @@ -3015,7 +2940,8 @@ void radv_CmdExecuteCommands( primary->state.dirty |= RADV_CMD_DIRTY_PIPELINE | RADV_CMD_DIRTY_INDEX_BUFFER | RADV_CMD_DIRTY_DYNAMIC_ALL; - radv_mark_descriptor_sets_dirty(primary); + radv_mark_descriptor_sets_dirty(primary, VK_PIPELINE_BIND_POINT_GRAPHICS); + radv_mark_descriptor_sets_dirty(primary, VK_PIPELINE_BIND_POINT_COMPUTE); } VkResult radv_CreateCommandPool( @@ -3030,7 +2956,7 @@ VkResult radv_CreateCommandPool( pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (pool == NULL) - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); if (pAllocator) pool->alloc = *pAllocator; @@ -3090,7 +3016,7 @@ VkResult radv_ResetCommandPool( return VK_SUCCESS; } -void radv_TrimCommandPoolKHR( +void radv_TrimCommandPool( VkDevice device, VkCommandPool commandPool, VkCommandPoolTrimFlagsKHR flags) @@ -3133,6 +3059,15 @@ void radv_CmdBeginRenderPass( radv_cmd_buffer_clear_subpass(cmd_buffer); } +void radv_CmdBeginRenderPass2KHR( + VkCommandBuffer commandBuffer, + const VkRenderPassBeginInfo* pRenderPassBeginInfo, + const VkSubpassBeginInfoKHR* pSubpassBeginInfo) +{ + radv_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo, + pSubpassBeginInfo->contents); +} + void radv_CmdNextSubpass( VkCommandBuffer commandBuffer, VkSubpassContents contents) @@ -3148,13 +3083,22 @@ void radv_CmdNextSubpass( radv_cmd_buffer_clear_subpass(cmd_buffer); } +void radv_CmdNextSubpass2KHR( + VkCommandBuffer commandBuffer, + const VkSubpassBeginInfoKHR* pSubpassBeginInfo, + const VkSubpassEndInfoKHR* pSubpassEndInfo) +{ + radv_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents); +} + static void radv_emit_view_index(struct radv_cmd_buffer *cmd_buffer, unsigned index) { struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) { - if (!pipeline->shaders[stage]) + if (!radv_get_shader(pipeline, stage)) continue; - struct ac_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, AC_UD_VIEW_INDEX); + + struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, AC_UD_VIEW_INDEX); if (loc->sgpr_idx == -1) continue; uint32_t base_reg = pipeline->user_data_0[stage]; @@ -3162,7 +3106,7 @@ static void radv_emit_view_index(struct radv_cmd_buffer *cmd_buffer, unsigned in } if (pipeline->gs_copy_shader) { - struct ac_userdata_info *loc = &pipeline->gs_copy_shader->info.user_sgprs_locs.shader_data[AC_UD_VIEW_INDEX]; + struct radv_userdata_info *loc = &pipeline->gs_copy_shader->info.user_sgprs_locs.shader_data[AC_UD_VIEW_INDEX]; if (loc->sgpr_idx != -1) { uint32_t base_reg = R_00B130_SPI_SHADER_USER_DATA_VS_0; radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, index); @@ -3200,13 +3144,18 @@ radv_cs_emit_indirect_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t count_va, uint32_t stride) { - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; unsigned di_src_sel = indexed ? V_0287F0_DI_SRC_SEL_DMA : V_0287F0_DI_SRC_SEL_AUTO_INDEX; - bool draw_id_enable = radv_get_vertex_shader(cmd_buffer->state.pipeline)->info.info.vs.needs_draw_id; + bool draw_id_enable = radv_get_shader(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX)->info.info.vs.needs_draw_id; uint32_t base_reg = cmd_buffer->state.pipeline->graphics.vtx_base_sgpr; assert(base_reg); + /* just reset draw state for vertex data */ + cmd_buffer->state.last_first_instance = -1; + cmd_buffer->state.last_num_instances = -1; + cmd_buffer->state.last_vertex_offset = -1; + if (draw_count == 1 && !count_va && !draw_id_enable) { radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT : PKT3_DRAW_INDIRECT, 3, false)); @@ -3283,7 +3232,7 @@ radv_emit_draw_packets(struct radv_cmd_buffer *cmd_buffer, { struct radv_cmd_state *state = &cmd_buffer->state; struct radeon_winsys *ws = cmd_buffer->device->ws; - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; if (info->indirect) { uint64_t va = radv_buffer_get_va(info->indirect->bo); @@ -3291,7 +3240,7 @@ radv_emit_draw_packets(struct radv_cmd_buffer *cmd_buffer, va += info->indirect->offset + info->indirect_offset; - radv_cs_add_buffer(ws, cs, info->indirect->bo, 8); + radv_cs_add_buffer(ws, cs, info->indirect->bo); radeon_emit(cs, PKT3(PKT3_SET_BASE, 2, 0)); radeon_emit(cs, 1); @@ -3303,7 +3252,7 @@ radv_emit_draw_packets(struct radv_cmd_buffer *cmd_buffer, count_va += info->count_buffer->offset + info->count_buffer_offset; - radv_cs_add_buffer(ws, cs, info->count_buffer->bo, 8); + radv_cs_add_buffer(ws, cs, info->count_buffer->bo); } if (!state->subpass->view_mask) { @@ -3326,15 +3275,25 @@ radv_emit_draw_packets(struct radv_cmd_buffer *cmd_buffer, } } else { assert(state->pipeline->graphics.vtx_base_sgpr); - radeon_set_sh_reg_seq(cs, state->pipeline->graphics.vtx_base_sgpr, - state->pipeline->graphics.vtx_emit_num); - radeon_emit(cs, info->vertex_offset); - radeon_emit(cs, info->first_instance); - if (state->pipeline->graphics.vtx_emit_num == 3) - radeon_emit(cs, 0); - radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, state->predicating)); - radeon_emit(cs, info->instance_count); + if (info->vertex_offset != state->last_vertex_offset || + info->first_instance != state->last_first_instance) { + radeon_set_sh_reg_seq(cs, state->pipeline->graphics.vtx_base_sgpr, + state->pipeline->graphics.vtx_emit_num); + + radeon_emit(cs, info->vertex_offset); + radeon_emit(cs, info->first_instance); + if (state->pipeline->graphics.vtx_emit_num == 3) + radeon_emit(cs, 0); + state->last_first_instance = info->first_instance; + state->last_vertex_offset = info->vertex_offset; + } + + if (state->last_num_instances != info->instance_count) { + radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, false)); + radeon_emit(cs, info->instance_count); + state->last_num_instances = info->instance_count; + } if (info->indexed) { int index_size = state->index_type ? 4 : 2; @@ -3373,10 +3332,59 @@ radv_emit_draw_packets(struct radv_cmd_buffer *cmd_buffer, } } +/* + * Vega and raven have a bug which triggers if there are multiple context + * register contexts active at the same time with different scissor values. + * + * There are two possible workarounds: + * 1) Wait for PS_PARTIAL_FLUSH every time the scissor is changed. That way + * there is only ever 1 active set of scissor values at the same time. + * + * 2) Whenever the hardware switches contexts we have to set the scissor + * registers again even if it is a noop. That way the new context gets + * the correct scissor values. + * + * This implements option 2. radv_need_late_scissor_emission needs to + * return true on affected HW if radv_emit_all_graphics_states sets + * any context registers. + */ +static bool radv_need_late_scissor_emission(struct radv_cmd_buffer *cmd_buffer, + bool indexed_draw) +{ + struct radv_cmd_state *state = &cmd_buffer->state; + + if (!cmd_buffer->device->physical_device->has_scissor_bug) + return false; + + uint32_t used_states = cmd_buffer->state.pipeline->graphics.needed_dynamic_state | ~RADV_CMD_DIRTY_DYNAMIC_ALL; + + /* Index & Vertex buffer don't change context regs, and pipeline is handled later. */ + used_states &= ~(RADV_CMD_DIRTY_INDEX_BUFFER | RADV_CMD_DIRTY_VERTEX_BUFFER | RADV_CMD_DIRTY_PIPELINE); + + /* Assume all state changes except these two can imply context rolls. */ + if (cmd_buffer->state.dirty & used_states) + return true; + + if (cmd_buffer->state.emitted_pipeline != cmd_buffer->state.pipeline) + return true; + + if (indexed_draw && state->pipeline->graphics.prim_restart_enable && + (state->index_type ? 0xffffffffu : 0xffffu) != state->last_primitive_reset_index) + return true; + + return false; +} + static void radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info) { + bool late_scissor_emission = radv_need_late_scissor_emission(cmd_buffer, info->indexed); + + if ((cmd_buffer->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER) || + cmd_buffer->state.emitted_pipeline != cmd_buffer->state.pipeline) + radv_emit_rbplus_state(cmd_buffer); + if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) radv_emit_graphics_pipeline(cmd_buffer); @@ -3402,15 +3410,19 @@ radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, radv_emit_draw_registers(cmd_buffer, info->indexed, info->instance_count > 1, info->indirect, info->indirect ? 0 : info->count); + + if (late_scissor_emission) + radv_emit_scissor(cmd_buffer); } static void radv_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info) { + bool has_prefetch = + cmd_buffer->device->physical_device->rad_info.chip_class >= CIK; bool pipeline_is_dirty = (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) && - cmd_buffer->state.pipeline && cmd_buffer->state.pipeline != cmd_buffer->state.emitted_pipeline; MAYBE_UNUSED unsigned cdw_max = @@ -3435,8 +3447,7 @@ radv_draw(struct radv_cmd_buffer *cmd_buffer, si_emit_cache_flush(cmd_buffer); /* <-- CUs are idle here --> */ - if (!radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty)) - return; + radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty); radv_emit_draw_packets(cmd_buffer, info); /* <-- CUs are busy here --> */ @@ -3445,9 +3456,9 @@ radv_draw(struct radv_cmd_buffer *cmd_buffer, * run in parallel, but starting the draw first is more * important. */ - if (pipeline_is_dirty) { - radv_emit_prefetch(cmd_buffer, - cmd_buffer->state.pipeline); + if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) { + radv_emit_prefetch_L2(cmd_buffer, + cmd_buffer->state.pipeline, false); } } else { /* If we don't wait for idle, start prefetches first, then set @@ -3455,20 +3466,30 @@ radv_draw(struct radv_cmd_buffer *cmd_buffer, */ si_emit_cache_flush(cmd_buffer); - if (pipeline_is_dirty) { - radv_emit_prefetch(cmd_buffer, - cmd_buffer->state.pipeline); + if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) { + /* Only prefetch the vertex shader and VBO descriptors + * in order to start the draw as soon as possible. + */ + radv_emit_prefetch_L2(cmd_buffer, + cmd_buffer->state.pipeline, true); } - if (!radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty)) - return; + radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty); radv_emit_all_graphics_states(cmd_buffer, info); radv_emit_draw_packets(cmd_buffer, info); + + /* Prefetch the remaining shaders after the draw has been + * started. + */ + if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) { + radv_emit_prefetch_L2(cmd_buffer, + cmd_buffer->state.pipeline, false); + } } assert(cmd_buffer->cs->cdw <= cdw_max); - radv_cmd_buffer_after_draw(cmd_buffer); + radv_cmd_buffer_after_draw(cmd_buffer, RADV_CMD_FLAG_PS_PARTIAL_FLUSH); } void radv_CmdDraw( @@ -3598,12 +3619,66 @@ void radv_CmdDrawIndexedIndirectCountAMD( radv_draw(cmd_buffer, &info); } +void radv_CmdDrawIndirectCountKHR( + VkCommandBuffer commandBuffer, + VkBuffer _buffer, + VkDeviceSize offset, + VkBuffer _countBuffer, + VkDeviceSize countBufferOffset, + uint32_t maxDrawCount, + uint32_t stride) +{ + RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + RADV_FROM_HANDLE(radv_buffer, buffer, _buffer); + RADV_FROM_HANDLE(radv_buffer, count_buffer, _countBuffer); + struct radv_draw_info info = {}; + + info.count = maxDrawCount; + info.indirect = buffer; + info.indirect_offset = offset; + info.count_buffer = count_buffer; + info.count_buffer_offset = countBufferOffset; + info.stride = stride; + + radv_draw(cmd_buffer, &info); +} + +void radv_CmdDrawIndexedIndirectCountKHR( + VkCommandBuffer commandBuffer, + VkBuffer _buffer, + VkDeviceSize offset, + VkBuffer _countBuffer, + VkDeviceSize countBufferOffset, + uint32_t maxDrawCount, + uint32_t stride) +{ + RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + RADV_FROM_HANDLE(radv_buffer, buffer, _buffer); + RADV_FROM_HANDLE(radv_buffer, count_buffer, _countBuffer); + struct radv_draw_info info = {}; + + info.indexed = true; + info.count = maxDrawCount; + info.indirect = buffer; + info.indirect_offset = offset; + info.count_buffer = count_buffer; + info.count_buffer_offset = countBufferOffset; + info.stride = stride; + + radv_draw(cmd_buffer, &info); +} + struct radv_dispatch_info { /** * Determine the layout of the grid (in block units) to be used. */ uint32_t blocks[3]; + /** + * A starting offset for the grid. If unaligned is set, the offset + * must still be aligned. + */ + uint32_t offsets[3]; /** * Whether it's an unaligned compute dispatch. */ @@ -3624,8 +3699,8 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_variant *compute_shader = pipeline->shaders[MESA_SHADER_COMPUTE]; unsigned dispatch_initiator = cmd_buffer->device->dispatch_initiator; struct radeon_winsys *ws = cmd_buffer->device->ws; - struct radeon_winsys_cs *cs = cmd_buffer->cs; - struct ac_userdata_info *loc; + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_userdata_info *loc; loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_COMPUTE, AC_UD_CS_GRID_SIZE); @@ -3637,7 +3712,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, va += info->indirect->offset + info->indirect_offset; - radv_cs_add_buffer(ws, cs, info->indirect->bo, 8); + radv_cs_add_buffer(ws, cs, info->indirect->bo); if (loc->sgpr_idx != -1) { for (unsigned i = 0; i < 3; ++i) { @@ -3672,6 +3747,7 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, } } else { unsigned blocks[3] = { info->blocks[0], info->blocks[1], info->blocks[2] }; + unsigned offsets[3] = { info->offsets[0], info->offsets[1], info->offsets[2] }; if (info->unaligned) { unsigned *cs_block_size = compute_shader->info.cs.block_size; @@ -3691,6 +3767,11 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, blocks[1] = round_up_u32(blocks[1], cs_block_size[1]); blocks[2] = round_up_u32(blocks[2], cs_block_size[2]); + for(unsigned i = 0; i < 3; ++i) { + assert(offsets[i] % cs_block_size[i] == 0); + offsets[i] /= cs_block_size[i]; + } + radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); radeon_emit(cs, S_00B81C_NUM_THREAD_FULL(cs_block_size[0]) | @@ -3716,6 +3797,19 @@ radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, radeon_emit(cs, blocks[2]); } + if (offsets[0] || offsets[1] || offsets[2]) { + radeon_set_sh_reg_seq(cs, R_00B810_COMPUTE_START_X, 3); + radeon_emit(cs, offsets[0]); + radeon_emit(cs, offsets[1]); + radeon_emit(cs, offsets[2]); + + /* The blocks in the packet are not counts but end values. */ + for (unsigned i = 0; i < 3; ++i) + blocks[i] += offsets[i]; + } else { + dispatch_initiator |= S_00B800_FORCE_START_AT_000(1); + } + radeon_emit(cs, PKT3(PKT3_DISPATCH_DIRECT, 3, 0) | PKT3_SHADER_TYPE_S(1)); radeon_emit(cs, blocks[0]); @@ -3731,8 +3825,7 @@ static void radv_upload_compute_shader_descriptors(struct radv_cmd_buffer *cmd_buffer) { radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_COMPUTE_BIT); - radv_flush_constants(cmd_buffer, cmd_buffer->state.compute_pipeline, - VK_SHADER_STAGE_COMPUTE_BIT); + radv_flush_constants(cmd_buffer, VK_SHADER_STAGE_COMPUTE_BIT); } static void @@ -3740,6 +3833,8 @@ radv_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_info *info) { struct radv_pipeline *pipeline = cmd_buffer->state.compute_pipeline; + bool has_prefetch = + cmd_buffer->device->physical_device->rad_info.chip_class >= CIK; bool pipeline_is_dirty = pipeline && pipeline != cmd_buffer->state.emitted_compute_pipeline; @@ -3767,7 +3862,7 @@ radv_dispatch(struct radv_cmd_buffer *cmd_buffer, * will run in parallel, but starting the dispatch first is * more important. */ - if (pipeline_is_dirty) { + if (has_prefetch && pipeline_is_dirty) { radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_COMPUTE]); } @@ -3777,7 +3872,7 @@ radv_dispatch(struct radv_cmd_buffer *cmd_buffer, */ si_emit_cache_flush(cmd_buffer); - if (pipeline_is_dirty) { + if (has_prefetch && pipeline_is_dirty) { radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_COMPUTE]); } @@ -3788,11 +3883,14 @@ radv_dispatch(struct radv_cmd_buffer *cmd_buffer, radv_emit_dispatch_packets(cmd_buffer, info); } - radv_cmd_buffer_after_draw(cmd_buffer); + radv_cmd_buffer_after_draw(cmd_buffer, RADV_CMD_FLAG_CS_PARTIAL_FLUSH); } -void radv_CmdDispatch( +void radv_CmdDispatchBase( VkCommandBuffer commandBuffer, + uint32_t base_x, + uint32_t base_y, + uint32_t base_z, uint32_t x, uint32_t y, uint32_t z) @@ -3804,9 +3902,21 @@ void radv_CmdDispatch( info.blocks[1] = y; info.blocks[2] = z; + info.offsets[0] = base_x; + info.offsets[1] = base_y; + info.offsets[2] = base_z; radv_dispatch(cmd_buffer, &info); } +void radv_CmdDispatch( + VkCommandBuffer commandBuffer, + uint32_t x, + uint32_t y, + uint32_t z) +{ + radv_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z); +} + void radv_CmdDispatchIndirect( VkCommandBuffer commandBuffer, VkBuffer _buffer, @@ -3850,7 +3960,7 @@ void radv_CmdEndRenderPass( for (unsigned i = 0; i < cmd_buffer->state.framebuffer->attachment_count; ++i) { VkImageLayout layout = cmd_buffer->state.pass->attachments[i].final_layout; radv_handle_subpass_image_transition(cmd_buffer, - (VkAttachmentReference){i, layout}); + (struct radv_subpass_attachment){i, layout}); } vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments); @@ -3861,10 +3971,17 @@ void radv_CmdEndRenderPass( cmd_buffer->state.framebuffer = NULL; } +void radv_CmdEndRenderPass2KHR( + VkCommandBuffer commandBuffer, + const VkSubpassEndInfoKHR* pSubpassEndInfo) +{ + radv_CmdEndRenderPass(commandBuffer); +} + /* * For HTILE we have the following interesting clear words: - * 0x0000030f: Uncompressed for depth+stencil HTILE. - * 0x0000000f: Uncompressed for depth only HTILE. + * 0xfffff30f: Uncompressed, full depth range, for depth+stencil HTILE + * 0xfffc000f: Uncompressed, full depth range, for depth only HTILE. * 0xfffffff0: Clear depth to 1.0 * 0x00000000: Clear depth to 0.0 */ @@ -3877,9 +3994,11 @@ static void radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, assert(range->levelCount == 1 || range->levelCount == VK_REMAINING_ARRAY_LAYERS); unsigned layer_count = radv_get_layerCount(image, range); uint64_t size = image->surface.htile_slice_size * layer_count; + VkImageAspectFlags aspects = VK_IMAGE_ASPECT_DEPTH_BIT; uint64_t offset = image->offset + image->htile_offset + image->surface.htile_slice_size * range->baseArrayLayer; struct radv_cmd_state *state = &cmd_buffer->state; + VkClearDepthStencilValue value = {}; state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; @@ -3888,6 +4007,11 @@ static void radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, size, clear_word); state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; + + if (vk_format_is_stencil(image->vk_format)) + aspects |= VK_IMAGE_ASPECT_STENCIL_BIT; + + radv_set_ds_clear_metadata(cmd_buffer, image, value, aspects); } static void radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffer, @@ -3899,20 +4023,16 @@ static void radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffe const VkImageSubresourceRange *range, VkImageAspectFlags pending_clears) { - if (dst_layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL && - (pending_clears & vk_format_aspects(image->vk_format)) == vk_format_aspects(image->vk_format) && - cmd_buffer->state.render_area.offset.x == 0 && cmd_buffer->state.render_area.offset.y == 0 && - cmd_buffer->state.render_area.extent.width == image->info.width && - cmd_buffer->state.render_area.extent.height == image->info.height) { - /* The clear will initialize htile. */ + if (!radv_image_has_htile(image)) return; - } else if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED && + + if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED && radv_layout_has_htile(image, dst_layout, dst_queue_mask)) { /* TODO: merge with the clear if applicable */ radv_initialize_htile(cmd_buffer, image, range, 0); } else if (!radv_layout_is_htile_compressed(image, src_layout, src_queue_mask) && radv_layout_is_htile_compressed(image, dst_layout, dst_queue_mask)) { - uint32_t clear_value = vk_format_is_stencil(image->vk_format) ? 0x30f : 0xf; + uint32_t clear_value = vk_format_is_stencil(image->vk_format) ? 0xfffff30f : 0xfffc000f; radv_initialize_htile(cmd_buffer, image, range, clear_value); } else if (radv_layout_is_htile_compressed(image, src_layout, src_queue_mask) && !radv_layout_is_htile_compressed(image, dst_layout, dst_queue_mask)) { @@ -3931,40 +4051,19 @@ static void radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffe } } -void radv_initialise_cmask(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, uint32_t value) +static void radv_initialise_cmask(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, uint32_t value) { struct radv_cmd_state *state = &cmd_buffer->state; state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; - state->flush_bits |= radv_fill_buffer(cmd_buffer, image->bo, - image->offset + image->cmask.offset, - image->cmask.size, value); + state->flush_bits |= radv_clear_cmask(cmd_buffer, image, value); state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; } -static void radv_handle_cmask_image_transition(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, - VkImageLayout src_layout, - VkImageLayout dst_layout, - unsigned src_queue_mask, - unsigned dst_queue_mask, - const VkImageSubresourceRange *range) -{ - if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED) { - if (image->fmask.size) - radv_initialise_cmask(cmd_buffer, image, 0xccccccccu); - else - radv_initialise_cmask(cmd_buffer, image, 0xffffffffu); - } else if (radv_layout_can_fast_clear(image, src_layout, src_queue_mask) && - !radv_layout_can_fast_clear(image, dst_layout, dst_queue_mask)) { - radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); - } -} - void radv_initialize_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, uint32_t value) { @@ -3973,32 +4072,85 @@ void radv_initialize_dcc(struct radv_cmd_buffer *cmd_buffer, state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; - state->flush_bits |= radv_fill_buffer(cmd_buffer, image->bo, - image->offset + image->dcc_offset, - image->surface.dcc_size, value); + state->flush_bits |= radv_clear_dcc(cmd_buffer, image, value); state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; } -static void radv_handle_dcc_image_transition(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, - VkImageLayout src_layout, - VkImageLayout dst_layout, - unsigned src_queue_mask, - unsigned dst_queue_mask, - const VkImageSubresourceRange *range) +/** + * Initialize DCC/FMASK/CMASK metadata for a color image. + */ +static void radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + VkImageLayout src_layout, + VkImageLayout dst_layout, + unsigned src_queue_mask, + unsigned dst_queue_mask) +{ + if (radv_image_has_cmask(image)) { + uint32_t value = 0xffffffffu; /* Fully expanded mode. */ + + /* TODO: clarify this. */ + if (radv_image_has_fmask(image)) { + value = 0xccccccccu; + } + + radv_initialise_cmask(cmd_buffer, image, value); + } + + if (radv_image_has_dcc(image)) { + uint32_t value = 0xffffffffu; /* Fully expanded mode. */ + + if (radv_layout_dcc_compressed(image, dst_layout, + dst_queue_mask)) { + value = 0x20202020u; + } + + radv_initialize_dcc(cmd_buffer, image, value); + + radv_set_dcc_need_cmask_elim_pred(cmd_buffer, image, false); + } + + if (radv_image_has_cmask(image) || radv_image_has_dcc(image)) { + uint32_t color_values[2] = {}; + radv_set_color_clear_metadata(cmd_buffer, image, color_values); + } +} + +/** + * Handle color image transitions for DCC/FMASK/CMASK. + */ +static void radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, + struct radv_image *image, + VkImageLayout src_layout, + VkImageLayout dst_layout, + unsigned src_queue_mask, + unsigned dst_queue_mask, + const VkImageSubresourceRange *range) { if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED) { - radv_initialize_dcc(cmd_buffer, image, - radv_layout_dcc_compressed(image, dst_layout, dst_queue_mask) ? - 0x20202020u : 0xffffffffu); - } else if (radv_layout_dcc_compressed(image, src_layout, src_queue_mask) && - !radv_layout_dcc_compressed(image, dst_layout, dst_queue_mask)) { - radv_decompress_dcc(cmd_buffer, image, range); - } else if (radv_layout_can_fast_clear(image, src_layout, src_queue_mask) && - !radv_layout_can_fast_clear(image, dst_layout, dst_queue_mask)) { - radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); + radv_init_color_image_metadata(cmd_buffer, image, + src_layout, dst_layout, + src_queue_mask, dst_queue_mask); + return; + } + + if (radv_image_has_dcc(image)) { + if (src_layout == VK_IMAGE_LAYOUT_PREINITIALIZED) { + radv_initialize_dcc(cmd_buffer, image, 0xffffffffu); + } else if (radv_layout_dcc_compressed(image, src_layout, src_queue_mask) && + !radv_layout_dcc_compressed(image, dst_layout, dst_queue_mask)) { + radv_decompress_dcc(cmd_buffer, image, range); + } else if (radv_layout_can_fast_clear(image, src_layout, src_queue_mask) && + !radv_layout_can_fast_clear(image, dst_layout, dst_queue_mask)) { + radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); + } + } else if (radv_image_has_cmask(image) || radv_image_has_fmask(image)) { + if (radv_layout_can_fast_clear(image, src_layout, src_queue_mask) && + !radv_layout_can_fast_clear(image, dst_layout, dst_queue_mask)) { + radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); + } } } @@ -4028,62 +4180,82 @@ static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, return; } - unsigned src_queue_mask = radv_image_queue_family_mask(image, src_family, cmd_buffer->queue_family_index); - unsigned dst_queue_mask = radv_image_queue_family_mask(image, dst_family, cmd_buffer->queue_family_index); - - if (image->surface.htile_size) - radv_handle_depth_image_transition(cmd_buffer, image, src_layout, - dst_layout, src_queue_mask, - dst_queue_mask, range, - pending_clears); - - if (image->cmask.size || image->fmask.size) - radv_handle_cmask_image_transition(cmd_buffer, image, src_layout, - dst_layout, src_queue_mask, - dst_queue_mask, range); - - if (image->surface.dcc_size) - radv_handle_dcc_image_transition(cmd_buffer, image, src_layout, - dst_layout, src_queue_mask, - dst_queue_mask, range); + unsigned src_queue_mask = + radv_image_queue_family_mask(image, src_family, + cmd_buffer->queue_family_index); + unsigned dst_queue_mask = + radv_image_queue_family_mask(image, dst_family, + cmd_buffer->queue_family_index); + + if (vk_format_is_depth(image->vk_format)) { + radv_handle_depth_image_transition(cmd_buffer, image, + src_layout, dst_layout, + src_queue_mask, dst_queue_mask, + range, pending_clears); + } else { + radv_handle_color_image_transition(cmd_buffer, image, + src_layout, dst_layout, + src_queue_mask, dst_queue_mask, + range); + } } -void radv_CmdPipelineBarrier( - VkCommandBuffer commandBuffer, - VkPipelineStageFlags srcStageMask, - VkPipelineStageFlags destStageMask, - VkBool32 byRegion, - uint32_t memoryBarrierCount, - const VkMemoryBarrier* pMemoryBarriers, - uint32_t bufferMemoryBarrierCount, - const VkBufferMemoryBarrier* pBufferMemoryBarriers, - uint32_t imageMemoryBarrierCount, - const VkImageMemoryBarrier* pImageMemoryBarriers) -{ - RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); +struct radv_barrier_info { + uint32_t eventCount; + const VkEvent *pEvents; + VkPipelineStageFlags srcStageMask; +}; + +static void +radv_barrier(struct radv_cmd_buffer *cmd_buffer, + uint32_t memoryBarrierCount, + const VkMemoryBarrier *pMemoryBarriers, + uint32_t bufferMemoryBarrierCount, + const VkBufferMemoryBarrier *pBufferMemoryBarriers, + uint32_t imageMemoryBarrierCount, + const VkImageMemoryBarrier *pImageMemoryBarriers, + const struct radv_barrier_info *info) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; enum radv_cmd_flush_bits src_flush_bits = 0; enum radv_cmd_flush_bits dst_flush_bits = 0; + for (unsigned i = 0; i < info->eventCount; ++i) { + RADV_FROM_HANDLE(radv_event, event, info->pEvents[i]); + uint64_t va = radv_buffer_get_va(event->bo); + + radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo); + + MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7); + + si_emit_wait_fence(cs, va, 1, 0xffffffff); + assert(cmd_buffer->cs->cdw <= cdw_max); + } + for (uint32_t i = 0; i < memoryBarrierCount; i++) { - src_flush_bits |= radv_src_access_flush(cmd_buffer, pMemoryBarriers[i].srcAccessMask); + src_flush_bits |= radv_src_access_flush(cmd_buffer, pMemoryBarriers[i].srcAccessMask, + NULL); dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pMemoryBarriers[i].dstAccessMask, NULL); } for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) { - src_flush_bits |= radv_src_access_flush(cmd_buffer, pBufferMemoryBarriers[i].srcAccessMask); + src_flush_bits |= radv_src_access_flush(cmd_buffer, pBufferMemoryBarriers[i].srcAccessMask, + NULL); dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pBufferMemoryBarriers[i].dstAccessMask, NULL); } for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) { RADV_FROM_HANDLE(radv_image, image, pImageMemoryBarriers[i].image); - src_flush_bits |= radv_src_access_flush(cmd_buffer, pImageMemoryBarriers[i].srcAccessMask); + + src_flush_bits |= radv_src_access_flush(cmd_buffer, pImageMemoryBarriers[i].srcAccessMask, + image); dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pImageMemoryBarriers[i].dstAccessMask, image); } - radv_stage_flush(cmd_buffer, srcStageMask); + radv_stage_flush(cmd_buffer, info->srcStageMask); cmd_buffer->state.flush_bits |= src_flush_bits; for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) { @@ -4097,31 +4269,95 @@ void radv_CmdPipelineBarrier( 0); } + /* Make sure CP DMA is idle because the driver might have performed a + * DMA operation for copying or filling buffers/images. + */ + si_cp_dma_wait_for_idle(cmd_buffer); + cmd_buffer->state.flush_bits |= dst_flush_bits; } +void radv_CmdPipelineBarrier( + VkCommandBuffer commandBuffer, + VkPipelineStageFlags srcStageMask, + VkPipelineStageFlags destStageMask, + VkBool32 byRegion, + uint32_t memoryBarrierCount, + const VkMemoryBarrier* pMemoryBarriers, + uint32_t bufferMemoryBarrierCount, + const VkBufferMemoryBarrier* pBufferMemoryBarriers, + uint32_t imageMemoryBarrierCount, + const VkImageMemoryBarrier* pImageMemoryBarriers) +{ + RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + struct radv_barrier_info info; + + info.eventCount = 0; + info.pEvents = NULL; + info.srcStageMask = srcStageMask; + + radv_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers, + bufferMemoryBarrierCount, pBufferMemoryBarriers, + imageMemoryBarrierCount, pImageMemoryBarriers, &info); +} + static void write_event(struct radv_cmd_buffer *cmd_buffer, struct radv_event *event, VkPipelineStageFlags stageMask, unsigned value) { - struct radeon_winsys_cs *cs = cmd_buffer->cs; + struct radeon_cmdbuf *cs = cmd_buffer->cs; uint64_t va = radv_buffer_get_va(event->bo); - radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo, 8); + radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo); MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 18); - /* TODO: this is overkill. Probably should figure something out from - * the stage mask. */ + /* Flags that only require a top-of-pipe event. */ + VkPipelineStageFlags top_of_pipe_flags = + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + + /* Flags that only require a post-index-fetch event. */ + VkPipelineStageFlags post_index_fetch_flags = + top_of_pipe_flags | + VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | + VK_PIPELINE_STAGE_VERTEX_INPUT_BIT; + + /* Make sure CP DMA is idle because the driver might have performed a + * DMA operation for copying or filling buffers/images. + */ + si_cp_dma_wait_for_idle(cmd_buffer); + + /* TODO: Emit EOS events for syncing PS/CS stages. */ - si_cs_emit_write_event_eop(cs, - cmd_buffer->state.predicating, - cmd_buffer->device->physical_device->rad_info.chip_class, - radv_cmd_buffer_uses_mec(cmd_buffer), - V_028A90_BOTTOM_OF_PIPE_TS, 0, - 1, va, 2, value); + if (!(stageMask & ~top_of_pipe_flags)) { + /* Just need to sync the PFP engine. */ + radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); + radeon_emit(cs, S_370_DST_SEL(V_370_MEM_ASYNC) | + S_370_WR_CONFIRM(1) | + S_370_ENGINE_SEL(V_370_PFP)); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + radeon_emit(cs, value); + } else if (!(stageMask & ~post_index_fetch_flags)) { + /* Sync ME because PFP reads index and indirect buffers. */ + radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); + radeon_emit(cs, S_370_DST_SEL(V_370_MEM_ASYNC) | + S_370_WR_CONFIRM(1) | + S_370_ENGINE_SEL(V_370_ME)); + radeon_emit(cs, va); + radeon_emit(cs, va >> 32); + radeon_emit(cs, value); + } else { + /* Otherwise, sync all prior GPU work using an EOP event. */ + si_cs_emit_write_event_eop(cs, + cmd_buffer->device->physical_device->rad_info.chip_class, + radv_cmd_buffer_uses_mec(cmd_buffer), + V_028A90_BOTTOM_OF_PIPE_TS, 0, + EOP_DATA_SEL_VALUE_32BIT, va, 2, value, + cmd_buffer->gfx9_eop_bug_va); + } assert(cmd_buffer->cs->cdw <= cdw_max); } @@ -4159,36 +4395,20 @@ void radv_CmdWaitEvents(VkCommandBuffer commandBuffer, const VkImageMemoryBarrier* pImageMemoryBarriers) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - struct radeon_winsys_cs *cs = cmd_buffer->cs; - - for (unsigned i = 0; i < eventCount; ++i) { - RADV_FROM_HANDLE(radv_event, event, pEvents[i]); - uint64_t va = radv_buffer_get_va(event->bo); - - radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo, 8); - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7); - - si_emit_wait_fence(cs, false, va, 1, 0xffffffff); - assert(cmd_buffer->cs->cdw <= cdw_max); - } + struct radv_barrier_info info; + info.eventCount = eventCount; + info.pEvents = pEvents; + info.srcStageMask = 0; - for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) { - RADV_FROM_HANDLE(radv_image, image, pImageMemoryBarriers[i].image); + radv_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers, + bufferMemoryBarrierCount, pBufferMemoryBarriers, + imageMemoryBarrierCount, pImageMemoryBarriers, &info); +} - radv_handle_image_transition(cmd_buffer, image, - pImageMemoryBarriers[i].oldLayout, - pImageMemoryBarriers[i].newLayout, - pImageMemoryBarriers[i].srcQueueFamilyIndex, - pImageMemoryBarriers[i].dstQueueFamilyIndex, - &pImageMemoryBarriers[i].subresourceRange, - 0); - } - /* TODO: figure out how to do memory barriers without waiting */ - cmd_buffer->state.flush_bits |= RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER | - RADV_CMD_FLAG_INV_GLOBAL_L2 | - RADV_CMD_FLAG_INV_VMEM_L1 | - RADV_CMD_FLAG_INV_SMEM_L1; +void radv_CmdSetDeviceMask(VkCommandBuffer commandBuffer, + uint32_t deviceMask) +{ + /* No-op */ }