X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_compute_blit.c;h=e07042adf3fe5e7951e12e95a02e781b6da4aa21;hb=1909bc526dd0e51d49252a1fc1c2aab0a8a51fb5;hp=c8513557b1ed43783b1b5fc61f934828ce104888;hpb=1f21396431a03dc4e5a542628d7d8370973c967f;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/si_compute_blit.c b/src/gallium/drivers/radeonsi/si_compute_blit.c index c8513557b1e..e07042adf3f 100644 --- a/src/gallium/drivers/radeonsi/si_compute_blit.c +++ b/src/gallium/drivers/radeonsi/si_compute_blit.c @@ -36,7 +36,7 @@ static enum si_cache_policy get_cache_policy(struct si_context *sctx, { if ((sctx->chip_class >= GFX9 && (coher == SI_COHERENCY_CB_META || coher == SI_COHERENCY_CP)) || - (sctx->chip_class >= CIK && coher == SI_COHERENCY_SHADER)) + (sctx->chip_class >= GFX7 && coher == SI_COHERENCY_SHADER)) return size <= 256 * 1024 ? L2_LRU : L2_STREAM; return L2_BYPASS; @@ -51,9 +51,9 @@ unsigned si_get_flush_flags(struct si_context *sctx, enum si_coherency coher, case SI_COHERENCY_CP: return 0; case SI_COHERENCY_SHADER: - return SI_CONTEXT_INV_SMEM_L1 | - SI_CONTEXT_INV_VMEM_L1 | - (cache_policy == L2_BYPASS ? SI_CONTEXT_INV_GLOBAL_L2 : 0); + return SI_CONTEXT_INV_SCACHE | + SI_CONTEXT_INV_VCACHE | + (cache_policy == L2_BYPASS ? SI_CONTEXT_INV_L2 : 0); case SI_COHERENCY_CB_META: return SI_CONTEXT_FLUSH_AND_INV_CB; } @@ -102,6 +102,13 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, struct pipe_shader_buffer saved_sb[2] = {}; si_get_shader_buffers(sctx, PIPE_SHADER_COMPUTE, 0, src ? 2 : 1, saved_sb); + unsigned saved_writable_mask = 0; + for (unsigned i = 0; i < (src ? 2 : 1); i++) { + if (sctx->const_and_shader_buffers[PIPE_SHADER_COMPUTE].writable_mask & + (1u << si_get_shaderbuf_slot(i))) + saved_writable_mask |= 1 << i; + } + /* The memory accesses are coalesced, meaning that the 1st instruction writes * the 1st contiguous block of data for the whole wave, the 2nd instruction * writes the 2nd contiguous block of data, etc. @@ -110,13 +117,14 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, SI_COMPUTE_CLEAR_DW_PER_THREAD; unsigned instructions_per_thread = MAX2(1, dwords_per_thread / 4); unsigned dwords_per_instruction = dwords_per_thread / instructions_per_thread; - unsigned dwords_per_wave = dwords_per_thread * 64; + unsigned wave_size = sctx->screen->compute_wave_size; + unsigned dwords_per_wave = dwords_per_thread * wave_size; unsigned num_dwords = size / 4; unsigned num_instructions = DIV_ROUND_UP(num_dwords, dwords_per_instruction); struct pipe_grid_info info = {}; - info.block[0] = MIN2(64, num_instructions); + info.block[0] = MIN2(wave_size, num_instructions); info.block[1] = 1; info.block[2] = 1; info.grid[0] = DIV_ROUND_UP(num_dwords, dwords_per_wave); @@ -135,7 +143,7 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, sb[1].buffer_offset = src_offset; sb[1].buffer_size = size; - ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 2, sb); + ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 2, sb, 0x1); if (!sctx->cs_copy_buffer) { sctx->cs_copy_buffer = si_create_dma_compute_shader(&sctx->b, @@ -151,7 +159,7 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, for (unsigned i = 0; i < 4; i++) sctx->cs_user_data[i] = clear_value[i % (clear_value_size / 4)]; - ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, sb); + ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, sb, 0x1); if (!sctx->cs_clear_buffer) { sctx->cs_clear_buffer = si_create_dma_compute_shader(&sctx->b, @@ -165,25 +173,27 @@ static void si_compute_do_clear_or_copy(struct si_context *sctx, enum si_cache_policy cache_policy = get_cache_policy(sctx, coher, size); sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | - (cache_policy == L2_BYPASS ? SI_CONTEXT_WRITEBACK_GLOBAL_L2 : 0); + (cache_policy == L2_BYPASS ? SI_CONTEXT_WB_L2 : 0); if (cache_policy != L2_BYPASS) si_resource(dst)->TC_L2_dirty = true; /* Restore states. */ ctx->bind_compute_state(ctx, saved_cs); - ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, src ? 2 : 1, saved_sb); + ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, src ? 2 : 1, saved_sb, + saved_writable_mask); si_compute_internal_end(sctx); } void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, uint64_t offset, uint64_t size, uint32_t *clear_value, - uint32_t clear_value_size, enum si_coherency coher) + uint32_t clear_value_size, enum si_coherency coher, + bool force_cpdma) { if (!size) return; - unsigned clear_alignment = MIN2(clear_value_size, 4); + ASSERTED unsigned clear_alignment = MIN2(clear_value_size, 4); assert(clear_value_size != 3 && clear_value_size != 6); /* 12 is allowed. */ assert(offset % clear_alignment == 0); @@ -242,9 +252,10 @@ void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst, * about buffer placements. */ if (clear_value_size > 4 || - (clear_value_size == 4 && + (!force_cpdma && + clear_value_size == 4 && offset % 4 == 0 && - (size > 32*1024 || sctx->chip_class <= VI))) { + (size > 32*1024 || sctx->chip_class <= GFX8))) { si_compute_do_clear_or_copy(sctx, dst, offset, NULL, 0, aligned_size, clear_value, clear_value_size, coher); @@ -276,7 +287,7 @@ static void si_pipe_clear_buffer(struct pipe_context *ctx, int clear_value_size) { si_clear_buffer((struct si_context*)ctx, dst, offset, size, (uint32_t*)clear_value, - clear_value_size, SI_COHERENCY_SHADER); + clear_value_size, SI_COHERENCY_SHADER, false); } void si_copy_buffer(struct si_context *sctx, @@ -408,7 +419,7 @@ void si_compute_copy_image(struct si_context *sctx, ctx->launch_grid(ctx, &info); sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | - (sctx->chip_class <= VI ? SI_CONTEXT_WRITEBACK_GLOBAL_L2 : 0) | + (sctx->chip_class <= GFX8 ? SI_CONTEXT_WB_L2 : 0) | si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM); ctx->bind_compute_state(ctx, saved_cs); ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 2, saved_image); @@ -424,7 +435,7 @@ void si_retile_dcc(struct si_context *sctx, struct si_texture *tex) SI_CONTEXT_CS_PARTIAL_FLUSH | si_get_flush_flags(sctx, SI_COHERENCY_CB_META, L2_LRU) | si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_LRU); - si_emit_cache_flush(sctx); + sctx->emit_cache_flush(sctx); /* Save states. */ void *saved_cs = sctx->cs_shader_state.program; @@ -440,9 +451,9 @@ void si_retile_dcc(struct si_context *sctx, struct si_texture *tex) unsigned num_elements = tex->surface.u.gfx9.dcc_retile_num_elements; struct pipe_image_view img[3]; - assert(tex->dcc_retile_map_offset && tex->dcc_retile_map_offset <= UINT_MAX); - assert(tex->dcc_offset && tex->dcc_offset <= UINT_MAX); - assert(tex->display_dcc_offset && tex->display_dcc_offset <= UINT_MAX); + assert(tex->surface.dcc_retile_map_offset && tex->surface.dcc_retile_map_offset <= UINT_MAX); + assert(tex->surface.dcc_offset && tex->surface.dcc_offset <= UINT_MAX); + assert(tex->surface.display_dcc_offset && tex->surface.display_dcc_offset <= UINT_MAX); for (unsigned i = 0; i < 3; i++) { img[i].resource = &tex->buffer.b.b; @@ -452,15 +463,15 @@ void si_retile_dcc(struct si_context *sctx, struct si_texture *tex) img[0].format = use_uint16 ? PIPE_FORMAT_R16G16B16A16_UINT : PIPE_FORMAT_R32G32B32A32_UINT; - img[0].u.buf.offset = tex->dcc_retile_map_offset; + img[0].u.buf.offset = tex->surface.dcc_retile_map_offset; img[0].u.buf.size = num_elements * (use_uint16 ? 2 : 4); img[1].format = PIPE_FORMAT_R8_UINT; - img[1].u.buf.offset = tex->dcc_offset; + img[1].u.buf.offset = tex->surface.dcc_offset; img[1].u.buf.size = tex->surface.dcc_size; img[2].format = PIPE_FORMAT_R8_UINT; - img[2].u.buf.offset = tex->display_dcc_offset; + img[2].u.buf.offset = tex->surface.display_dcc_offset; img[2].u.buf.size = tex->surface.u.gfx9.display_dcc_size; ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 3, img); @@ -494,6 +505,91 @@ void si_retile_dcc(struct si_context *sctx, struct si_texture *tex) ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 3, saved_img); } +/* Expand FMASK to make it identity, so that image stores can ignore it. */ +void si_compute_expand_fmask(struct pipe_context *ctx, struct pipe_resource *tex) +{ + struct si_context *sctx = (struct si_context *)ctx; + bool is_array = tex->target == PIPE_TEXTURE_2D_ARRAY; + unsigned log_fragments = util_logbase2(tex->nr_storage_samples); + unsigned log_samples = util_logbase2(tex->nr_samples); + assert(tex->nr_samples >= 2); + + /* EQAA FMASK expansion is unimplemented. */ + if (tex->nr_samples != tex->nr_storage_samples) + return; + + si_compute_internal_begin(sctx); + + /* Flush caches and sync engines. */ + sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | + si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM); + si_make_CB_shader_coherent(sctx, tex->nr_samples, true, + true /* DCC is not possible with image stores */); + + /* Save states. */ + void *saved_cs = sctx->cs_shader_state.program; + struct pipe_image_view saved_image = {0}; + util_copy_image_view(&saved_image, &sctx->images[PIPE_SHADER_COMPUTE].views[0]); + + /* Bind the image. */ + struct pipe_image_view image = {0}; + image.resource = tex; + /* Don't set WRITE so as not to trigger FMASK expansion, causing + * an infinite loop. */ + image.shader_access = image.access = PIPE_IMAGE_ACCESS_READ; + image.format = util_format_linear(tex->format); + if (is_array) + image.u.tex.last_layer = tex->array_size - 1; + + ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &image); + + /* Bind the shader. */ + void **shader = &sctx->cs_fmask_expand[log_samples - 1][is_array]; + if (!*shader) + *shader = si_create_fmask_expand_cs(ctx, tex->nr_samples, is_array); + ctx->bind_compute_state(ctx, *shader); + + /* Dispatch compute. */ + struct pipe_grid_info info = {0}; + info.block[0] = 8; + info.last_block[0] = tex->width0 % 8; + info.block[1] = 8; + info.last_block[1] = tex->height0 % 8; + info.block[2] = 1; + info.grid[0] = DIV_ROUND_UP(tex->width0, 8); + info.grid[1] = DIV_ROUND_UP(tex->height0, 8); + info.grid[2] = is_array ? tex->array_size : 1; + + ctx->launch_grid(ctx, &info); + + /* Flush caches and sync engines. */ + sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | + (sctx->chip_class <= GFX8 ? SI_CONTEXT_WB_L2 : 0) | + si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM); + + /* Restore previous states. */ + ctx->bind_compute_state(ctx, saved_cs); + ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_image); + si_compute_internal_end(sctx); + + /* Array of fully expanded FMASK values, arranged by [log2(fragments)][log2(samples)-1]. */ +#define INVALID 0 /* never used */ + static const uint64_t fmask_expand_values[][4] = { + /* samples */ + /* 2 (8 bpp) 4 (8 bpp) 8 (8-32bpp) 16 (16-64bpp) fragments */ + {0x02020202, 0x0E0E0E0E, 0xFEFEFEFE, 0xFFFEFFFE}, /* 1 */ + {0x02020202, 0xA4A4A4A4, 0xAAA4AAA4, 0xAAAAAAA4}, /* 2 */ + {INVALID, 0xE4E4E4E4, 0x44443210, 0x4444444444443210}, /* 4 */ + {INVALID, INVALID, 0x76543210, 0x8888888876543210}, /* 8 */ + }; + + /* Clear FMASK to identity. */ + struct si_texture *stex = (struct si_texture*)tex; + si_clear_buffer(sctx, tex, stex->surface.fmask_offset, stex->surface.fmask_size, + (uint32_t*)&fmask_expand_values[log_fragments][log_samples - 1], + 4, SI_COHERENCY_SHADER, false); +} + void si_init_compute_blit_functions(struct si_context *sctx) { sctx->b.clear_buffer = si_pipe_clear_buffer; @@ -587,7 +683,7 @@ void si_compute_clear_render_target(struct pipe_context *ctx, ctx->launch_grid(ctx, &info); sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | - (sctx->chip_class <= VI ? SI_CONTEXT_WRITEBACK_GLOBAL_L2 : 0) | + (sctx->chip_class <= GFX8 ? SI_CONTEXT_WB_L2 : 0) | si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM); ctx->bind_compute_state(ctx, saved_cs); ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_image);