X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fr600%2Fr600_state_common.c;h=16d73f619dc717fdd095daba5f6dc53246a2e8be;hb=f718ac62688b555a933c7112f656944288d04edb;hp=fee7a21d27dd039eca461ef43298e27f40c9eceb;hpb=4e7f6437b5359fe41a48fbba510f46ac69db8653;p=mesa.git diff --git a/src/gallium/drivers/r600/r600_state_common.c b/src/gallium/drivers/r600/r600_state_common.c index fee7a21d27d..16d73f619dc 100644 --- a/src/gallium/drivers/r600/r600_state_common.c +++ b/src/gallium/drivers/r600/r600_state_common.c @@ -28,7 +28,7 @@ #include "r600_shader.h" #include "r600d.h" -#include "util/u_format_s3tc.h" +#include "util/format/u_format_s3tc.h" #include "util/u_index_modify.h" #include "util/u_memory.h" #include "util/u_upload_mgr.h" @@ -37,6 +37,10 @@ #include "tgsi/tgsi_scan.h" #include "tgsi/tgsi_ureg.h" +#include "nir.h" +#include "nir/nir_to_tgsi_info.h" +#include "tgsi/tgsi_from_mesa.h" + void r600_init_command_buffer(struct r600_command_buffer *cb, unsigned num_dw) { assert(!cb->buf); @@ -77,7 +81,7 @@ void r600_emit_cso_state(struct r600_context *rctx, struct r600_atom *atom) void r600_emit_alphatest_state(struct r600_context *rctx, struct r600_atom *atom) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; struct r600_alphatest_state *a = (struct r600_alphatest_state*)atom; unsigned alpha_ref = a->sx_alpha_ref; @@ -94,6 +98,10 @@ void r600_emit_alphatest_state(struct r600_context *rctx, struct r600_atom *atom static void r600_memory_barrier(struct pipe_context *ctx, unsigned flags) { struct r600_context *rctx = (struct r600_context *)ctx; + + if (!(flags & ~PIPE_BARRIER_UPDATE)) + return; + if (flags & PIPE_BARRIER_CONSTANT_BUFFER) rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE; @@ -241,7 +249,7 @@ static void r600_set_blend_color(struct pipe_context *ctx, void r600_emit_blend_color(struct r600_context *rctx, struct r600_atom *atom) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; struct pipe_blend_color *state = &rctx->blend_color.state; radeon_set_context_reg_seq(cs, R_028414_CB_BLEND_RED, 4); @@ -253,7 +261,7 @@ void r600_emit_blend_color(struct r600_context *rctx, struct r600_atom *atom) void r600_emit_vgt_state(struct r600_context *rctx, struct r600_atom *atom) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; struct r600_vgt_state *a = (struct r600_vgt_state *)atom; radeon_set_context_reg(cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, a->vgt_multi_prim_ib_reset_en); @@ -287,7 +295,7 @@ static void r600_set_stencil_ref(struct pipe_context *ctx, void r600_emit_stencil_ref(struct r600_context *rctx, struct r600_atom *atom) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; struct r600_stencil_ref_state *a = (struct r600_stencil_ref_state*)atom; radeon_set_context_reg_seq(cs, R_028430_DB_STENCILREFMASK, 2); @@ -415,7 +423,7 @@ static void r600_sampler_view_destroy(struct pipe_context *ctx, if (view->tex_resource->gpu_address && view->tex_resource->b.b.target == PIPE_BUFFER) - LIST_DELINIT(&view->list); + list_delinit(&view->list); pipe_resource_reference(&state->texture, NULL); FREE(view); @@ -542,7 +550,8 @@ static void r600_bind_vertex_elements(struct pipe_context *ctx, void *state) static void r600_delete_vertex_elements(struct pipe_context *ctx, void *state) { struct r600_fetch_shader *shader = (struct r600_fetch_shader*)state; - r600_resource_reference(&shader->buffer, NULL); + if (shader) + r600_resource_reference(&shader->buffer, NULL); FREE(shader); } @@ -570,7 +579,10 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx, /* Set vertex buffers. */ if (input) { for (i = 0; i < count; i++) { - if (memcmp(&input[i], &vb[i], sizeof(struct pipe_vertex_buffer))) { + if ((input[i].buffer.resource != vb[i].buffer.resource) || + (vb[i].stride != input[i].stride) || + (vb[i].buffer_offset != input[i].buffer_offset) || + (vb[i].is_user_buffer != input[i].is_user_buffer)) { if (input[i].buffer.resource) { vb[i].stride = input[i].stride; vb[i].buffer_offset = input[i].buffer_offset; @@ -820,6 +832,8 @@ static inline void r600_shader_selector_key(const struct pipe_context *ctx, key->tcs.prim_mode = rctx->tes_shader->info.properties[TGSI_PROPERTY_TES_PRIM_MODE]; key->tcs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_TESS_CTRL); break; + case PIPE_SHADER_COMPUTE: + break; default: assert(0); } @@ -827,7 +841,7 @@ static inline void r600_shader_selector_key(const struct pipe_context *ctx, /* Select the hw shader variant depending on the current state. * (*dirty) is set to 1 if current variant was changed */ -static int r600_shader_select(struct pipe_context *ctx, +int r600_shader_select(struct pipe_context *ctx, struct r600_pipe_shader_selector* sel, bool *dirty) { @@ -895,17 +909,39 @@ static int r600_shader_select(struct pipe_context *ctx, return 0; } +struct r600_pipe_shader_selector *r600_create_shader_state_tokens(struct pipe_context *ctx, + const void *prog, enum pipe_shader_ir ir, + unsigned pipe_shader_type) +{ + struct r600_pipe_shader_selector *sel = CALLOC_STRUCT(r600_pipe_shader_selector); + + sel->type = pipe_shader_type; + if (ir == PIPE_SHADER_IR_TGSI) { + sel->tokens = tgsi_dup_tokens((const struct tgsi_token *)prog); + tgsi_scan_shader(sel->tokens, &sel->info); + } else if (ir == PIPE_SHADER_IR_NIR){ + sel->nir = nir_shader_clone(NULL, (const nir_shader *)prog); + nir_tgsi_scan_shader(sel->nir, &sel->info, true); + } + return sel; +} + static void *r600_create_shader_state(struct pipe_context *ctx, const struct pipe_shader_state *state, unsigned pipe_shader_type) { - struct r600_pipe_shader_selector *sel = CALLOC_STRUCT(r600_pipe_shader_selector); int i; - - sel->type = pipe_shader_type; - sel->tokens = tgsi_dup_tokens(state->tokens); + struct r600_pipe_shader_selector *sel; + + if (state->type == PIPE_SHADER_IR_TGSI) + sel = r600_create_shader_state_tokens(ctx, state->tokens, state->type, pipe_shader_type); + else if (state->type == PIPE_SHADER_IR_NIR) { + sel = r600_create_shader_state_tokens(ctx, state->ir.nir, state->type, pipe_shader_type); + } else + assert(0 && "Unknown shader type\n"); + + sel->ir_type = state->type; sel->so = state->stream_output; - tgsi_scan_shader(state->tokens, &sel->info); switch (pipe_shader_type) { case PIPE_SHADER_GEOMETRY: @@ -1006,7 +1042,9 @@ static void r600_bind_vs_state(struct pipe_context *ctx, void *state) rctx->vs_shader = (struct r600_pipe_shader_selector *)state; r600_update_vs_writes_viewport_index(&rctx->b, r600_get_vs_info(rctx)); - rctx->b.streamout.stride_in_dw = rctx->vs_shader->so.stride; + + if (rctx->vs_shader->so.num_outputs) + rctx->b.streamout.stride_in_dw = rctx->vs_shader->so.stride; } static void r600_bind_gs_state(struct pipe_context *ctx, void *state) @@ -1021,7 +1059,9 @@ static void r600_bind_gs_state(struct pipe_context *ctx, void *state) if (!state) return; - rctx->b.streamout.stride_in_dw = rctx->gs_shader->so.stride; + + if (rctx->gs_shader->so.num_outputs) + rctx->b.streamout.stride_in_dw = rctx->gs_shader->so.stride; } static void r600_bind_tcs_state(struct pipe_context *ctx, void *state) @@ -1043,11 +1083,13 @@ static void r600_bind_tes_state(struct pipe_context *ctx, void *state) if (!state) return; - rctx->b.streamout.stride_in_dw = rctx->tes_shader->so.stride; + + if (rctx->tes_shader->so.num_outputs) + rctx->b.streamout.stride_in_dw = rctx->tes_shader->so.stride; } -static void r600_delete_shader_selector(struct pipe_context *ctx, - struct r600_pipe_shader_selector *sel) +void r600_delete_shader_selector(struct pipe_context *ctx, + struct r600_pipe_shader_selector *sel) { struct r600_pipe_shader *p = sel->current, *c; while (p) { @@ -1057,7 +1099,14 @@ static void r600_delete_shader_selector(struct pipe_context *ctx, p = c; } - free(sel->tokens); + if (sel->ir_type == PIPE_SHADER_IR_TGSI) { + free(sel->tokens); + /* We might have converted the TGSI shader to a NIR shader */ + if (sel->nir) + ralloc_free(sel->nir); + } + else if (sel->ir_type == PIPE_SHADER_IR_NIR) + ralloc_free(sel->nir); free(sel); } @@ -1204,16 +1253,23 @@ static void r600_set_sample_mask(struct pipe_context *pipe, unsigned sample_mask r600_mark_atom_dirty(rctx, &rctx->sample_mask.atom); } -static void r600_update_driver_const_buffers(struct r600_context *rctx) +void r600_update_driver_const_buffers(struct r600_context *rctx, bool compute_only) { int sh, size; void *ptr; struct pipe_constant_buffer cb; - for (sh = 0; sh < PIPE_SHADER_TYPES; sh++) { + int start, end; + + start = compute_only ? PIPE_SHADER_COMPUTE : 0; + end = compute_only ? PIPE_SHADER_TYPES : PIPE_SHADER_COMPUTE; + + for (sh = start; sh < end; sh++) { struct r600_shader_driver_constants_info *info = &rctx->driver_consts[sh]; if (!info->vs_ucp_dirty && !info->texture_const_dirty && - !info->ps_sample_pos_dirty) + !info->ps_sample_pos_dirty && + !info->tcs_default_levels_dirty && + !info->cs_block_grid_size_dirty) continue; ptr = info->constants; @@ -1229,7 +1285,7 @@ static void r600_update_driver_const_buffers(struct r600_context *rctx) info->vs_ucp_dirty = false; } - if (info->ps_sample_pos_dirty) { + else if (info->ps_sample_pos_dirty) { assert(sh == PIPE_SHADER_FRAGMENT); if (!size) { ptr = rctx->sample_positions; @@ -1240,6 +1296,31 @@ static void r600_update_driver_const_buffers(struct r600_context *rctx) info->ps_sample_pos_dirty = false; } + else if (info->cs_block_grid_size_dirty) { + assert(sh == PIPE_SHADER_COMPUTE); + if (!size) { + ptr = rctx->cs_block_grid_sizes; + size = R600_CS_BLOCK_GRID_SIZE; + } else { + memcpy(ptr, rctx->cs_block_grid_sizes, R600_CS_BLOCK_GRID_SIZE); + } + info->cs_block_grid_size_dirty = false; + } + + else if (info->tcs_default_levels_dirty) { + /* + * We'd only really need this for default tcs shader. + */ + assert(sh == PIPE_SHADER_TESS_CTRL); + if (!size) { + ptr = rctx->tess_state; + size = R600_TCS_DEFAULT_LEVELS_SIZE; + } else { + memcpy(ptr, rctx->tess_state, R600_TCS_DEFAULT_LEVELS_SIZE); + } + info->tcs_default_levels_dirty = false; + } + if (info->texture_const_dirty) { assert (ptr); assert (size); @@ -1247,6 +1328,10 @@ static void r600_update_driver_const_buffers(struct r600_context *rctx) memcpy(ptr, rctx->clip_state.state.ucp, R600_UCP_SIZE); if (sh == PIPE_SHADER_FRAGMENT) memcpy(ptr, rctx->sample_positions, R600_UCP_SIZE); + if (sh == PIPE_SHADER_COMPUTE) + memcpy(ptr, rctx->cs_block_grid_sizes, R600_CS_BLOCK_GRID_SIZE); + if (sh == PIPE_SHADER_TESS_CTRL) + memcpy(ptr, rctx->tess_state, R600_TCS_DEFAULT_LEVELS_SIZE); } info->texture_const_dirty = false; @@ -1260,7 +1345,7 @@ static void r600_update_driver_const_buffers(struct r600_context *rctx) } static void *r600_alloc_buf_consts(struct r600_context *rctx, int shader_type, - int array_size, uint32_t *base_offset) + unsigned array_size, uint32_t *base_offset) { struct r600_shader_driver_constants_info *info = &rctx->driver_consts[shader_type]; if (array_size + R600_UCP_SIZE > info->alloc_size) { @@ -1295,7 +1380,7 @@ static void r600_setup_buffer_constants(struct r600_context *rctx, int shader_ty samplers->views.dirty_buffer_constants = FALSE; bits = util_last_bit(samplers->views.enabled_mask); - array_size = bits * 8 * sizeof(uint32_t) * 4; + array_size = bits * 8 * sizeof(uint32_t); constants = r600_alloc_buf_consts(rctx, shader_type, array_size, &base_offset); @@ -1318,22 +1403,21 @@ static void r600_setup_buffer_constants(struct r600_context *rctx, int shader_ty } else constants[offset + 4] = 0; - constants[offset + 5] = samplers->views.views[i]->base.texture->width0 / util_format_get_blocksize(samplers->views.views[i]->base.format); + constants[offset + 5] = samplers->views.views[i]->base.u.buf.size / + util_format_get_blocksize(samplers->views.views[i]->base.format); constants[offset + 6] = samplers->views.views[i]->base.texture->array_size / 6; } } } -/* On evergreen we store two values - * 1. buffer size for TXQ - * 2. number of cube layers in a cube map array. +/* On evergreen we store one value + * 1. number of cube layers in a cube map array. */ -static void eg_setup_buffer_constants(struct r600_context *rctx, int shader_type) +void eg_setup_buffer_constants(struct r600_context *rctx, int shader_type) { struct r600_textures_info *samplers = &rctx->samplers[shader_type]; struct r600_image_state *images = NULL; - struct r600_image_state *buffers = NULL; int bits, sview_bits, img_bits; uint32_t array_size; int i; @@ -1342,55 +1426,40 @@ static void eg_setup_buffer_constants(struct r600_context *rctx, int shader_type if (shader_type == PIPE_SHADER_FRAGMENT) { images = &rctx->fragment_images; - buffers = &rctx->fragment_buffers; + } else if (shader_type == PIPE_SHADER_COMPUTE) { + images = &rctx->compute_images; } if (!samplers->views.dirty_buffer_constants && - (images && !images->dirty_buffer_constants) && - (buffers && !buffers->dirty_buffer_constants)) + !(images && images->dirty_buffer_constants)) return; if (images) images->dirty_buffer_constants = FALSE; - if (buffers) - buffers->dirty_buffer_constants = FALSE; samplers->views.dirty_buffer_constants = FALSE; bits = sview_bits = util_last_bit(samplers->views.enabled_mask); if (images) bits += util_last_bit(images->enabled_mask); img_bits = bits; - if (buffers) - bits += util_last_bit(buffers->enabled_mask); - array_size = bits * 2 * sizeof(uint32_t) * 4; + + array_size = bits * sizeof(uint32_t); constants = r600_alloc_buf_consts(rctx, shader_type, array_size, &base_offset); for (i = 0; i < sview_bits; i++) { if (samplers->views.enabled_mask & (1 << i)) { - uint32_t offset = (base_offset / 4) + i * 2; - constants[offset] = samplers->views.views[i]->base.texture->width0 / util_format_get_blocksize(samplers->views.views[i]->base.format); - constants[offset + 1] = samplers->views.views[i]->base.texture->array_size / 6; + uint32_t offset = (base_offset / 4) + i; + constants[offset] = samplers->views.views[i]->base.texture->array_size / 6; } } if (images) { for (i = sview_bits; i < img_bits; i++) { int idx = i - sview_bits; if (images->enabled_mask & (1 << idx)) { - uint32_t offset = (base_offset / 4) + i * 2; - constants[offset] = images->views[i].base.resource->width0 / util_format_get_blocksize(images->views[i].base.format); - constants[offset + 1] = images->views[i].base.resource->array_size / 6; - } - } - } - if (buffers) { - for (i = img_bits; i < bits; i++) { - int idx = i - img_bits; - if (buffers->enabled_mask & (1 << idx)) { - uint32_t offset = (base_offset / 4) + i * 2; - constants[offset] = buffers->views[i].base.resource->width0 / util_format_get_blocksize(buffers->views[i].base.format); - constants[offset + 1] = 0; + uint32_t offset = (base_offset / 4) + i; + constants[offset] = images->views[idx].base.resource->array_size / 6; } } } @@ -1399,14 +1468,13 @@ static void eg_setup_buffer_constants(struct r600_context *rctx, int shader_type /* set sample xy locations as array of fragment shader constants */ void r600_set_sample_locations_constant_buffer(struct r600_context *rctx) { - int i; struct pipe_context *ctx = &rctx->b.b; assert(rctx->framebuffer.nr_samples < R600_UCP_SIZE); assert(rctx->framebuffer.nr_samples <= ARRAY_SIZE(rctx->sample_positions)/4); memset(rctx->sample_positions, 0, 4 * 4 * 16); - for (i = 0; i < rctx->framebuffer.nr_samples; i++) { + for (unsigned i = 0; i < rctx->framebuffer.nr_samples; i++) { ctx->get_sample_position(ctx, rctx->framebuffer.nr_samples, i, &rctx->sample_positions[4*i]); /* Also fill in center-zeroed positions used for interpolateAtSample */ rctx->sample_positions[4*i + 2] = rctx->sample_positions[4*i + 0] - 0.5f; @@ -1507,11 +1575,11 @@ static void r600_generate_fixed_func_tcs(struct r600_context *rctx) assert(!rctx->fixed_func_tcs_shader); - ureg_DECL_constant2D(ureg, 0, 3, R600_LDS_INFO_CONST_BUFFER); - const0 = ureg_src_dimension(ureg_src_register(TGSI_FILE_CONSTANT, 2), - R600_LDS_INFO_CONST_BUFFER); - const1 = ureg_src_dimension(ureg_src_register(TGSI_FILE_CONSTANT, 3), - R600_LDS_INFO_CONST_BUFFER); + ureg_DECL_constant2D(ureg, 0, 1, R600_BUFFER_INFO_CONST_BUFFER); + const0 = ureg_src_dimension(ureg_src_register(TGSI_FILE_CONSTANT, 0), + R600_BUFFER_INFO_CONST_BUFFER); + const1 = ureg_src_dimension(ureg_src_register(TGSI_FILE_CONSTANT, 1), + R600_BUFFER_INFO_CONST_BUFFER); tessouter = ureg_DECL_output(ureg, TGSI_SEMANTIC_TESSOUTER, 0); tessinner = ureg_DECL_output(ureg, TGSI_SEMANTIC_TESSINNER, 0); @@ -1524,7 +1592,7 @@ static void r600_generate_fixed_func_tcs(struct r600_context *rctx) ureg_create_shader_and_destroy(ureg, &rctx->b.b); } -static void r600_update_compressed_resource_state(struct r600_context *rctx) +void r600_update_compressed_resource_state(struct r600_context *rctx, bool compute_only) { unsigned i; unsigned counter; @@ -1533,15 +1601,25 @@ static void r600_update_compressed_resource_state(struct r600_context *rctx) if (counter != rctx->b.last_compressed_colortex_counter) { rctx->b.last_compressed_colortex_counter = counter; - for (i = 0; i < PIPE_SHADER_TYPES; ++i) { - r600_update_compressed_colortex_mask(&rctx->samplers[i].views); + if (compute_only) { + r600_update_compressed_colortex_mask(&rctx->samplers[PIPE_SHADER_COMPUTE].views); + } else { + for (i = 0; i < PIPE_SHADER_TYPES; ++i) { + r600_update_compressed_colortex_mask(&rctx->samplers[i].views); + } } - r600_update_compressed_colortex_mask_images(&rctx->fragment_images); + if (!compute_only) + r600_update_compressed_colortex_mask_images(&rctx->fragment_images); + r600_update_compressed_colortex_mask_images(&rctx->compute_images); } /* Decompress textures if needed. */ for (i = 0; i < PIPE_SHADER_TYPES; i++) { struct r600_samplerview_state *views = &rctx->samplers[i].views; + + if (compute_only) + if (i != PIPE_SHADER_COMPUTE) + continue; if (views->compressed_depthtex_mask) { r600_decompress_depth_textures(rctx, views); } @@ -1552,7 +1630,16 @@ static void r600_update_compressed_resource_state(struct r600_context *rctx) { struct r600_image_state *istate; - istate = &rctx->fragment_images; + + if (!compute_only) { + istate = &rctx->fragment_images; + if (istate->compressed_depthtex_mask) + r600_decompress_depth_images(rctx, istate); + if (istate->compressed_colortex_mask) + r600_decompress_color_images(rctx, istate); + } + + istate = &rctx->compute_images; if (istate->compressed_depthtex_mask) r600_decompress_depth_images(rctx, istate); if (istate->compressed_colortex_mask) @@ -1560,6 +1647,104 @@ static void r600_update_compressed_resource_state(struct r600_context *rctx) } } +/* update MEM_SCRATCH buffers if needed */ +void r600_setup_scratch_area_for_shader(struct r600_context *rctx, + struct r600_pipe_shader *shader, struct r600_scratch_buffer *scratch, + unsigned ring_base_reg, unsigned item_size_reg, unsigned ring_size_reg) +{ + unsigned num_ses = rctx->screen->b.info.max_se; + unsigned num_pipes = rctx->screen->b.info.r600_max_quad_pipes; + unsigned nthreads = 128; + + unsigned itemsize = shader->scratch_space_needed * 4; + unsigned size = align(itemsize * nthreads * num_pipes * num_ses * 4, 256); + + if (scratch->dirty || + unlikely(shader->scratch_space_needed != scratch->item_size || + size > scratch->size)) { + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; + + scratch->dirty = false; + + if (size > scratch->size) { + // Release prior one if any + if (scratch->buffer) { + pipe_resource_reference((struct pipe_resource**)&scratch->buffer, NULL); + } + + scratch->buffer = (struct r600_resource *)pipe_buffer_create(rctx->b.b.screen, PIPE_BIND_CUSTOM, + PIPE_USAGE_DEFAULT, size); + if (scratch->buffer) { + scratch->size = size; + } + } + + scratch->item_size = shader->scratch_space_needed; + + radeon_set_config_reg(cs, R_008040_WAIT_UNTIL, S_008040_WAIT_3D_IDLE(1)); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_VGT_FLUSH)); + + // multi-SE chips need programming per SE + for (unsigned se = 0; se < num_ses; se++) { + struct r600_resource *rbuffer = scratch->buffer; + unsigned size_per_se = size / num_ses; + + // Direct to particular SE + if (num_ses > 1) { + radeon_set_config_reg(cs, EG_0802C_GRBM_GFX_INDEX, + S_0802C_INSTANCE_INDEX(0) | + S_0802C_SE_INDEX(se) | + S_0802C_INSTANCE_BROADCAST_WRITES(1) | + S_0802C_SE_BROADCAST_WRITES(0)); + } + + radeon_set_config_reg(cs, ring_base_reg, (rbuffer->gpu_address + size_per_se * se) >> 8); + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, rbuffer, + RADEON_USAGE_READWRITE, + RADEON_PRIO_SCRATCH_BUFFER)); + radeon_set_context_reg(cs, item_size_reg, itemsize); + radeon_set_config_reg(cs, ring_size_reg, size_per_se >> 8); + } + + // Restore broadcast mode + if (num_ses > 1) { + radeon_set_config_reg(cs, EG_0802C_GRBM_GFX_INDEX, + S_0802C_INSTANCE_INDEX(0) | + S_0802C_SE_INDEX(0) | + S_0802C_INSTANCE_BROADCAST_WRITES(1) | + S_0802C_SE_BROADCAST_WRITES(1)); + } + + radeon_set_config_reg(cs, R_008040_WAIT_UNTIL, S_008040_WAIT_3D_IDLE(1)); + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); + radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_VGT_FLUSH)); + } +} + +void r600_setup_scratch_buffers(struct r600_context *rctx) { + static const struct { + unsigned ring_base; + unsigned item_size; + unsigned ring_size; + } regs[R600_NUM_HW_STAGES] = { + [R600_HW_STAGE_PS] = { R_008C68_SQ_PSTMP_RING_BASE, R_0288BC_SQ_PSTMP_RING_ITEMSIZE, R_008C6C_SQ_PSTMP_RING_SIZE }, + [R600_HW_STAGE_VS] = { R_008C60_SQ_VSTMP_RING_BASE, R_0288B8_SQ_VSTMP_RING_ITEMSIZE, R_008C64_SQ_VSTMP_RING_SIZE }, + [R600_HW_STAGE_GS] = { R_008C58_SQ_GSTMP_RING_BASE, R_0288B4_SQ_GSTMP_RING_ITEMSIZE, R_008C5C_SQ_GSTMP_RING_SIZE }, + [R600_HW_STAGE_ES] = { R_008C50_SQ_ESTMP_RING_BASE, R_0288B0_SQ_ESTMP_RING_ITEMSIZE, R_008C54_SQ_ESTMP_RING_SIZE } + }; + + for (unsigned i = 0; i < R600_NUM_HW_STAGES; i++) { + struct r600_pipe_shader *stage = rctx->hw_shader_stages[i].shader; + + if (stage && unlikely(stage->scratch_space_needed)) { + r600_setup_scratch_area_for_shader(rctx, stage, + &rctx->scratch_buffers[i], regs[i].ring_base, regs[i].item_size, regs[i].ring_size); + } + } +} + #define SELECT_SHADER_OR_FAIL(x) do { \ r600_shader_select(ctx, rctx->x##_shader, &x##_dirty); \ if (unlikely(!rctx->x##_shader->current)) \ @@ -1601,7 +1786,7 @@ static bool r600_update_derived_state(struct r600_context *rctx) struct r600_pipe_shader *clip_so_current = NULL; if (!rctx->blitter->running) - r600_update_compressed_resource_state(rctx); + r600_update_compressed_resource_state(rctx, false); SELECT_SHADER_OR_FAIL(ps); @@ -1674,6 +1859,21 @@ static bool r600_update_derived_state(struct r600_context *rctx) } } + /* + * XXX: I believe there's some fatal flaw in the dirty state logic when + * enabling/disabling tes. + * VS/ES share all buffer/resource/sampler slots. If TES is enabled, + * it will therefore overwrite the VS slots. If it now gets disabled, + * the VS needs to rebind all buffer/resource/sampler slots - not only + * has TES overwritten the corresponding slots, but when the VS was + * operating as LS the things with correpsonding dirty bits got bound + * to LS slots and won't reflect what is dirty as VS stage even if the + * TES didn't overwrite it. The story for re-enabled TES is similar. + * In any case, we're not allowed to submit any TES state when + * TES is disabled (the state tracker may not do this but this looks + * like an optimization to me, not something which can be relied on). + */ + /* Update clip misc state. */ if (clip_so_current) { r600_update_clip_state(rctx, clip_so_current); @@ -1684,8 +1884,10 @@ static bool r600_update_derived_state(struct r600_context *rctx) rctx->rasterizer->sprite_coord_enable != rctx->ps_shader->current->sprite_coord_enable || rctx->rasterizer->flatshade != rctx->ps_shader->current->flatshade)) { - if (rctx->cb_misc_state.nr_ps_color_outputs != rctx->ps_shader->current->nr_ps_color_outputs) { + if (rctx->cb_misc_state.nr_ps_color_outputs != rctx->ps_shader->current->nr_ps_color_outputs || + rctx->cb_misc_state.ps_color_export_mask != rctx->ps_shader->current->ps_color_export_mask) { rctx->cb_misc_state.nr_ps_color_outputs = rctx->ps_shader->current->nr_ps_color_outputs; + rctx->cb_misc_state.ps_color_export_mask = rctx->ps_shader->current->ps_color_export_mask; r600_mark_atom_dirty(rctx, &rctx->cb_misc_state.atom); } @@ -1718,6 +1920,13 @@ static bool r600_update_derived_state(struct r600_context *rctx) r600_update_db_shader_control(rctx); } + /* For each shader stage that needs to spill, set up buffer for MEM_SCRATCH */ + if (rctx->b.chip_class >= EVERGREEN) { + evergreen_setup_scratch_buffers(rctx); + } else { + r600_setup_scratch_buffers(rctx); + } + /* on R600 we stuff masks + txq info into one constant buffer */ /* on evergreen we only need a txq info one */ if (rctx->ps_shader) { @@ -1750,7 +1959,23 @@ static bool r600_update_derived_state(struct r600_context *rctx) } } - r600_update_driver_const_buffers(rctx); + if (rctx->tes_shader) { + assert(rctx->b.chip_class >= EVERGREEN); + need_buf_const = rctx->tes_shader->current->shader.uses_tex_buffers || + rctx->tes_shader->current->shader.has_txq_cube_array_z_comp; + if (need_buf_const) { + eg_setup_buffer_constants(rctx, PIPE_SHADER_TESS_EVAL); + } + if (rctx->tcs_shader) { + need_buf_const = rctx->tcs_shader->current->shader.uses_tex_buffers || + rctx->tcs_shader->current->shader.has_txq_cube_array_z_comp; + if (need_buf_const) { + eg_setup_buffer_constants(rctx, PIPE_SHADER_TESS_CTRL); + } + } + } + + r600_update_driver_const_buffers(rctx, false); if (rctx->b.chip_class < EVERGREEN && rctx->ps_shader && rctx->vs_shader) { if (!r600_adjust_gprs(rctx)) { @@ -1781,7 +2006,7 @@ static bool r600_update_derived_state(struct r600_context *rctx) void r600_emit_clip_misc_state(struct r600_context *rctx, struct r600_atom *atom) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; struct r600_clip_misc_state *state = &rctx->clip_misc_state; radeon_set_context_reg(cs, R_028810_PA_CL_CLIP_CNTL, @@ -1801,7 +2026,7 @@ void r600_emit_clip_misc_state(struct r600_context *rctx, struct r600_atom *atom /* rast_prim is the primitive type after GS. */ static inline void r600_emit_rasterizer_prim_state(struct r600_context *rctx) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; enum pipe_prim_type rast_prim = rctx->current_rast_prim; /* Skip this if not rendering lines. */ @@ -1828,7 +2053,7 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info { struct r600_context *rctx = (struct r600_context *)ctx; struct pipe_resource *indexbuf = info->has_user_indices ? NULL : info->index.resource; - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; bool render_cond_bit = rctx->b.render_cond && !rctx->b.render_cond_force_off; bool has_user_indices = info->has_user_indices; uint64_t mask; @@ -1857,6 +2082,11 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info rctx->b.dma.flush(rctx, PIPE_FLUSH_ASYNC, NULL); } + if (rctx->cmd_buf_is_compute) { + rctx->b.gfx.flush(rctx, PIPE_FLUSH_ASYNC, NULL); + rctx->cmd_buf_is_compute = false; + } + /* Re-emit the framebuffer state if needed. */ dirty_tex_counter = p_atomic_read(&rctx->b.screen->dirty_tex_counter); if (unlikely(dirty_tex_counter != rctx->b.last_dirty_tex_counter)) { @@ -1890,8 +2120,9 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info : (rctx->tes_shader)? rctx->tes_shader->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] : info->mode; - if (rctx->b.chip_class >= EVERGREEN) - evergreen_emit_atomic_buffer_setup(rctx, combined_atomics, &atomic_used_mask); + if (rctx->b.chip_class >= EVERGREEN) { + evergreen_emit_atomic_buffer_setup_count(rctx, NULL, combined_atomics, &atomic_used_mask); + } if (index_size) { index_offset += info->start * index_size; @@ -1977,7 +2208,7 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info evergreen_setup_tess_constants(rctx, info, &num_patches); /* Emit states. */ - r600_need_cs_space(rctx, has_user_indices ? 5 : 0, TRUE); + r600_need_cs_space(rctx, has_user_indices ? 5 : 0, TRUE, util_bitcount(atomic_used_mask)); r600_flush_emit(rctx); mask = rctx->dirty_atoms; @@ -1985,6 +2216,10 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info r600_emit_atom(rctx, rctx->atoms[u_bit_scan64(&mask)]); } + if (rctx->b.chip_class >= EVERGREEN) { + evergreen_emit_atomic_buffer_setup(rctx, false, combined_atomics, atomic_used_mask); + } + if (rctx->b.chip_class == CAYMAN) { /* Copied from radeonsi. */ unsigned primgroup_size = 128; /* recommended without a GS */ @@ -2175,7 +2410,7 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info if (rctx->b.chip_class >= EVERGREEN) - evergreen_emit_atomic_buffer_save(rctx, combined_atomics, &atomic_used_mask); + evergreen_emit_atomic_buffer_save(rctx, false, combined_atomics, &atomic_used_mask); if (rctx->trace_buf) eg_trace_emit(rctx); @@ -2338,7 +2573,7 @@ bool sampler_state_needs_border_color(const struct pipe_sampler_state *state) void r600_emit_shader(struct r600_context *rctx, struct r600_atom *a) { - struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct radeon_cmdbuf *cs = rctx->b.gfx.cs; struct r600_pipe_shader *shader = ((struct r600_shader_state*)a)->shader; if (!shader) @@ -2717,6 +2952,7 @@ uint32_t r600_translate_texformat(struct pipe_screen *screen, switch (desc->nr_channels) { case 1: result = FMT_8; + is_srgb_valid = TRUE; goto out_word4; case 2: result = FMT_8_8; @@ -3066,7 +3302,7 @@ static void r600_invalidate_buffer(struct pipe_context *ctx, struct pipe_resourc } -static void r600_set_active_query_state(struct pipe_context *ctx, boolean enable) +static void r600_set_active_query_state(struct pipe_context *ctx, bool enable) { struct r600_context *rctx = (struct r600_context*)ctx; @@ -3089,7 +3325,7 @@ static void r600_set_active_query_state(struct pipe_context *ctx, boolean enable static void r600_need_gfx_cs_space(struct pipe_context *ctx, unsigned num_dw, bool include_draw_vbo) { - r600_need_cs_space((struct r600_context*)ctx, num_dw, include_draw_vbo); + r600_need_cs_space((struct r600_context*)ctx, num_dw, include_draw_vbo, 0); } /* keep this at the end of this file, please */ @@ -3133,6 +3369,7 @@ void r600_init_common_state_functions(struct r600_context *rctx) rctx->b.b.texture_barrier = r600_texture_barrier; rctx->b.b.set_stream_output_targets = r600_set_streamout_targets; rctx->b.b.set_active_query_state = r600_set_active_query_state; + rctx->b.b.draw_vbo = r600_draw_vbo; rctx->b.invalidate_buffer = r600_invalidate_buffer; rctx->b.need_gfx_cs_space = r600_need_gfx_cs_space;