From: Dave Airlie Date: Thu, 2 Nov 2017 00:26:51 +0000 (+1000) Subject: r600: add support for hw atomic counters. (v3) X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=06993e4ee350b9c2ab1e3ee7686878add3900d39;p=mesa.git r600: add support for hw atomic counters. (v3) This adds support for the evergreen/cayman atomic counters. These are implemented using GDS append/consume counters. The values for each counter are loaded before drawing and saved after each draw using special CP packets. v2: move hw atomic assignment into driver. v3: fix messing up caps (Gert Wollny), only store ranges in driver, drop buffers. Signed-off-by: Dave Airlie Acked-by: Nicolai Hähnle Tested-By: Gert Wollny --- diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c index 131778dea9f..eb8b139910d 100644 --- a/src/gallium/drivers/r600/evergreen_state.c +++ b/src/gallium/drivers/r600/evergreen_state.c @@ -3717,6 +3717,38 @@ static void evergreen_set_tess_state(struct pipe_context *ctx, rctx->tess_state_dirty = true; } +static void evergreen_set_hw_atomic_buffers(struct pipe_context *ctx, + unsigned start_slot, + unsigned count, + const struct pipe_shader_buffer *buffers) +{ + struct r600_context *rctx = (struct r600_context *)ctx; + struct r600_atomic_buffer_state *astate; + int i, idx; + + astate = &rctx->atomic_buffer_state; + + /* we'd probably like to expand this to 8 later so put the logic in */ + for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) { + const struct pipe_shader_buffer *buf; + struct pipe_shader_buffer *abuf; + + abuf = &astate->buffer[i]; + + if (!buffers || !buffers[idx].buffer) { + pipe_resource_reference(&abuf->buffer, NULL); + astate->enabled_mask &= ~(1 << i); + continue; + } + buf = &buffers[idx]; + + pipe_resource_reference(&abuf->buffer, buf->buffer); + abuf->buffer_offset = buf->buffer_offset; + abuf->buffer_size = buf->buffer_size; + astate->enabled_mask |= (1 << i); + } +} + void evergreen_init_state_functions(struct r600_context *rctx) { unsigned id = 1; @@ -3802,6 +3834,7 @@ void evergreen_init_state_functions(struct r600_context *rctx) rctx->b.b.set_polygon_stipple = evergreen_set_polygon_stipple; rctx->b.b.set_min_samples = evergreen_set_min_samples; rctx->b.b.set_tess_state = evergreen_set_tess_state; + rctx->b.b.set_hw_atomic_buffers = evergreen_set_hw_atomic_buffers; if (rctx->b.chip_class == EVERGREEN) rctx->b.b.get_sample_position = evergreen_get_sample_position; else @@ -4108,3 +4141,129 @@ void eg_trace_emit(struct r600_context *rctx) radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); radeon_emit(cs, AC_ENCODE_TRACE_POINT(rctx->trace_id)); } + +bool evergreen_emit_atomic_buffer_setup(struct r600_context *rctx, + struct r600_shader_atomic *combined_atomics, + uint8_t *atomic_used_mask_p) +{ + struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct r600_atomic_buffer_state *astate = &rctx->atomic_buffer_state; + unsigned pkt_flags = 0; + uint8_t atomic_used_mask = 0; + int i, j, k; + + for (i = 0; i < EG_NUM_HW_STAGES; i++) { + uint8_t num_atomic_stage; + struct r600_pipe_shader *pshader; + + pshader = rctx->hw_shader_stages[i].shader; + if (!pshader) + continue; + + num_atomic_stage = pshader->shader.nhwatomic_ranges; + if (!num_atomic_stage) + continue; + + for (j = 0; j < num_atomic_stage; j++) { + struct r600_shader_atomic *atomic = &pshader->shader.atomics[j]; + int natomics = atomic->end - atomic->start + 1; + + for (k = 0; k < natomics; k++) { + /* seen this in a previous stage */ + if (atomic_used_mask & (1u << (atomic->hw_idx + k))) + continue; + + combined_atomics[atomic->hw_idx + k].hw_idx = atomic->hw_idx + k; + combined_atomics[atomic->hw_idx + k].buffer_id = atomic->buffer_id; + combined_atomics[atomic->hw_idx + k].start = atomic->start + k; + combined_atomics[atomic->hw_idx + k].end = combined_atomics[atomic->hw_idx + k].start + 1; + atomic_used_mask |= (1u << (atomic->hw_idx + k)); + } + } + } + + uint32_t mask = atomic_used_mask; + while (mask) { + unsigned atomic_index = u_bit_scan(&mask); + struct r600_shader_atomic *atomic = &combined_atomics[atomic_index]; + struct r600_resource *resource = r600_resource(astate->buffer[atomic->buffer_id].buffer); + assert(resource); + unsigned reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, + resource, + RADEON_USAGE_READ, + RADEON_PRIO_SHADER_RW_BUFFER); + uint64_t dst_offset = resource->gpu_address + (atomic->start * 4); + uint32_t base_reg_0 = R_02872C_GDS_APPEND_COUNT_0; + + uint32_t reg_val = (base_reg_0 + atomic->hw_idx * 4 - EVERGREEN_CONTEXT_REG_OFFSET) >> 2; + + radeon_emit(cs, PKT3(PKT3_SET_APPEND_CNT, 2, 0) | pkt_flags); + radeon_emit(cs, (reg_val << 16) | 0x3); + radeon_emit(cs, dst_offset & 0xfffffffc); + radeon_emit(cs, (dst_offset >> 32) & 0xff); + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, reloc); + } + *atomic_used_mask_p = atomic_used_mask; + return true; +} + +void evergreen_emit_atomic_buffer_save(struct r600_context *rctx, + struct r600_shader_atomic *combined_atomics, + uint8_t *atomic_used_mask_p) +{ + struct radeon_winsys_cs *cs = rctx->b.gfx.cs; + struct r600_atomic_buffer_state *astate = &rctx->atomic_buffer_state; + uint32_t pkt_flags = 0; + uint32_t event = EVENT_TYPE_PS_DONE; + uint32_t mask = astate->enabled_mask; + uint64_t dst_offset; + unsigned reloc; + + mask = *atomic_used_mask_p; + while (mask) { + unsigned atomic_index = u_bit_scan(&mask); + struct r600_shader_atomic *atomic = &combined_atomics[atomic_index]; + struct r600_resource *resource = r600_resource(astate->buffer[atomic->buffer_id].buffer); + assert(resource); + + uint32_t base_reg_0 = R_02872C_GDS_APPEND_COUNT_0; + reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, + resource, + RADEON_USAGE_WRITE, + RADEON_PRIO_SHADER_RW_BUFFER); + dst_offset = resource->gpu_address + (atomic->start * 4); + uint32_t reg_val = (base_reg_0 + atomic->hw_idx * 4) >> 2; + + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, 0) | pkt_flags); + radeon_emit(cs, EVENT_TYPE(event) | EVENT_INDEX(6)); + radeon_emit(cs, (dst_offset) & 0xffffffff); + radeon_emit(cs, (0 << 29) | ((dst_offset >> 32) & 0xff)); + radeon_emit(cs, reg_val); + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, reloc); + } + ++rctx->append_fence_id; + reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, + r600_resource(rctx->append_fence), + RADEON_USAGE_READWRITE, + RADEON_PRIO_SHADER_RW_BUFFER); + dst_offset = r600_resource(rctx->append_fence)->gpu_address; + radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, 0) | pkt_flags); + radeon_emit(cs, EVENT_TYPE(event) | EVENT_INDEX(6)); + radeon_emit(cs, dst_offset & 0xffffffff); + radeon_emit(cs, (2 << 29) | ((dst_offset >> 32) & 0xff)); + radeon_emit(cs, rctx->append_fence_id); + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, reloc); + + radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0) | pkt_flags); + radeon_emit(cs, WAIT_REG_MEM_GEQUAL | WAIT_REG_MEM_MEMORY | (1 << 8)); + radeon_emit(cs, dst_offset & 0xffffffff); + radeon_emit(cs, ((dst_offset >> 32) & 0xff)); + radeon_emit(cs, rctx->append_fence_id); + radeon_emit(cs, 0xffffffff); + radeon_emit(cs, 0xa); + radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); + radeon_emit(cs, reloc); +} diff --git a/src/gallium/drivers/r600/r600_pipe.c b/src/gallium/drivers/r600/r600_pipe.c index 0b815d4aef6..96017dc0477 100644 --- a/src/gallium/drivers/r600/r600_pipe.c +++ b/src/gallium/drivers/r600/r600_pipe.c @@ -74,6 +74,8 @@ static void r600_destroy_context(struct pipe_context *context) r600_resource_reference(&rctx->dummy_cmask, NULL); r600_resource_reference(&rctx->dummy_fmask, NULL); + if (rctx->append_fence) + pipe_resource_reference((struct pipe_resource**)&rctx->append_fence, NULL); for (sh = 0; sh < PIPE_SHADER_TYPES; sh++) { rctx->b.b.set_constant_buffer(&rctx->b.b, sh, R600_BUFFER_INFO_CONST_BUFFER, NULL); free(rctx->driver_consts[sh].constants); @@ -186,6 +188,9 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen, rctx->b.family == CHIP_CAICOS || rctx->b.family == CHIP_CAYMAN || rctx->b.family == CHIP_ARUBA); + + rctx->append_fence = pipe_buffer_create(rctx->b.b.screen, PIPE_BIND_CUSTOM, + PIPE_USAGE_DEFAULT, 32); break; default: R600_ERR("Unsupported chip class %d.\n", rctx->b.chip_class); @@ -605,8 +610,17 @@ static int r600_get_shader_param(struct pipe_screen* pscreen, case PIPE_SHADER_CAP_MAX_SHADER_IMAGES: case PIPE_SHADER_CAP_LOWER_IF_THRESHOLD: case PIPE_SHADER_CAP_TGSI_SKIP_MERGE_REGISTERS: + return 0; case PIPE_SHADER_CAP_MAX_HW_ATOMIC_COUNTERS: + if (rscreen->b.family >= CHIP_CEDAR && rscreen->has_atomics) + return 8; + return 0; case PIPE_SHADER_CAP_MAX_HW_ATOMIC_COUNTER_BUFFERS: + /* having to allocate the atomics out amongst shaders stages is messy, + so give compute 8 buffers and all the others one */ + if (rscreen->b.family >= CHIP_CEDAR && rscreen->has_atomics) { + return EG_MAX_ATOMIC_BUFFERS; + } return 0; case PIPE_SHADER_CAP_MAX_UNROLL_ITERATIONS_HINT: /* due to a bug in the shader compiler, some loops hang @@ -741,6 +755,7 @@ struct pipe_screen *r600_screen_create(struct radeon_winsys *ws, /* Create the auxiliary context. This must be done last. */ rscreen->b.aux_context = rscreen->b.b.context_create(&rscreen->b.b, NULL, 0); + rscreen->has_atomics = rscreen->b.info.drm_minor >= 44; #if 0 /* This is for testing whether aux_context and buffer clearing work correctly. */ struct pipe_resource templ = {}; diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h index 0d2551ac566..3dae56e3054 100644 --- a/src/gallium/drivers/r600/r600_pipe.h +++ b/src/gallium/drivers/r600/r600_pipe.h @@ -64,6 +64,8 @@ #define R600_MAX_DRIVER_CONST_BUFFERS 3 #define R600_MAX_CONST_BUFFERS (R600_MAX_USER_CONST_BUFFERS + R600_MAX_DRIVER_CONST_BUFFERS) +#define EG_MAX_ATOMIC_BUFFERS 8 + /* start driver buffers after user buffers */ #define R600_BUFFER_INFO_CONST_BUFFER (R600_MAX_USER_CONST_BUFFERS) #define R600_UCP_SIZE (4*4*8) @@ -247,6 +249,7 @@ struct r600_screen { struct r600_common_screen b; bool has_msaa; bool has_compressed_msaa_texturing; + bool has_atomics; /*for compute global memory binding, we allocate stuff here, instead of * buffers. @@ -416,6 +419,12 @@ struct r600_shader_state { struct r600_pipe_shader *shader; }; +struct r600_atomic_buffer_state { + uint32_t enabled_mask; + uint32_t dirty_mask; + struct pipe_shader_buffer buffer[EG_MAX_ATOMIC_BUFFERS]; +}; + struct r600_context { struct r600_common_context b; struct r600_screen *screen; @@ -470,6 +479,7 @@ struct r600_context { struct r600_config_state config_state; struct r600_stencil_ref_state stencil_ref; struct r600_vgt_state vgt_state; + struct r600_atomic_buffer_state atomic_buffer_state; /* Shaders and shader resources. */ struct r600_cso_state vertex_fetch_shader; struct r600_shader_state hw_shader_stages[EG_NUM_HW_STAGES]; @@ -531,6 +541,9 @@ struct r600_context { struct r600_resource *last_trace_buf; struct r600_resource *trace_buf; unsigned trace_id; + + struct pipe_resource *append_fence; + uint32_t append_fence_id; }; static inline void r600_emit_command_buffer(struct radeon_winsys_cs *cs, @@ -959,4 +972,13 @@ unsigned r600_conv_prim_to_gs_out(unsigned mode); void eg_trace_emit(struct r600_context *rctx); void eg_dump_debug_state(struct pipe_context *ctx, FILE *f, unsigned flags); + +struct r600_shader_atomic; +bool evergreen_emit_atomic_buffer_setup(struct r600_context *rctx, + struct r600_shader_atomic *combined_atomics, + uint8_t *atomic_used_mask_p); +void evergreen_emit_atomic_buffer_save(struct r600_context *rctx, + struct r600_shader_atomic *combined_atomics, + uint8_t *atomic_used_mask_p); + #endif diff --git a/src/gallium/drivers/r600/r600_shader.c b/src/gallium/drivers/r600/r600_shader.c index 188fbc9d47d..af866c4bddb 100644 --- a/src/gallium/drivers/r600/r600_shader.c +++ b/src/gallium/drivers/r600/r600_shader.c @@ -194,6 +194,8 @@ int r600_pipe_shader_create(struct pipe_context *ctx, /* disable SB for shaders using doubles */ use_sb &= !shader->shader.uses_doubles; + use_sb &= !shader->shader.uses_atomics; + /* Check if the bytecode has already been built. */ if (!shader->shader.bc.bytecode) { r = r600_bytecode_build(&shader->shader.bc); @@ -407,6 +409,7 @@ static int tgsi_is_supported(struct r600_shader_ctx *ctx) if (i->Src[j].Register.Dimension) { switch (i->Src[j].Register.File) { case TGSI_FILE_CONSTANT: + case TGSI_FILE_HW_ATOMIC: break; case TGSI_FILE_INPUT: if (ctx->type == PIPE_SHADER_GEOMETRY || @@ -966,6 +969,17 @@ static int tgsi_declaration(struct r600_shader_ctx *ctx) case TGSI_FILE_ADDRESS: break; + case TGSI_FILE_HW_ATOMIC: + i = ctx->shader->nhwatomic_ranges; + ctx->shader->atomics[i].start = d->Range.First; + ctx->shader->atomics[i].end = d->Range.Last; + ctx->shader->atomics[i].hw_idx = ctx->shader->atomic_base + ctx->shader->nhwatomic; + ctx->shader->atomics[i].array_id = d->Array.ArrayID; + ctx->shader->atomics[i].buffer_id = d->Dim.Index2D; + ctx->shader->nhwatomic_ranges++; + ctx->shader->nhwatomic += count; + break; + case TGSI_FILE_SYSTEM_VALUE: if (d->Semantic.Name == TGSI_SEMANTIC_SAMPLEMASK || d->Semantic.Name == TGSI_SEMANTIC_SAMPLEID || @@ -2946,6 +2960,7 @@ static int r600_shader_from_tgsi(struct r600_context *rctx, shader->indirect_files = ctx.info.indirect_files; shader->uses_doubles = ctx.info.uses_doubles; + shader->uses_atomics = ctx.info.file_mask[TGSI_FILE_HW_ATOMIC]; shader->nsys_inputs = 0; indirect_gprs = ctx.info.indirect_files & ~((1 << TGSI_FILE_CONSTANT) | (1 << TGSI_FILE_SAMPLER)); @@ -2959,6 +2974,7 @@ static int r600_shader_from_tgsi(struct r600_context *rctx, shader->vs_as_gs_a = key.vs.as_gs_a; shader->vs_as_es = key.vs.as_es; shader->vs_as_ls = key.vs.as_ls; + shader->atomic_base = key.vs.first_atomic_counter; if (shader->vs_as_es) ring_outputs = true; if (shader->vs_as_ls) @@ -2966,20 +2982,24 @@ static int r600_shader_from_tgsi(struct r600_context *rctx, break; case PIPE_SHADER_GEOMETRY: ring_outputs = true; + shader->atomic_base = key.gs.first_atomic_counter; break; case PIPE_SHADER_TESS_CTRL: shader->tcs_prim_mode = key.tcs.prim_mode; + shader->atomic_base = key.tcs.first_atomic_counter; lds_outputs = true; lds_inputs = true; break; case PIPE_SHADER_TESS_EVAL: shader->tes_as_es = key.tes.as_es; + shader->atomic_base = key.tes.first_atomic_counter; lds_inputs = true; if (shader->tes_as_es) ring_outputs = true; break; case PIPE_SHADER_FRAGMENT: shader->two_side = key.ps.color_two_side; + shader->atomic_base = key.ps.first_atomic_counter; break; default: break; @@ -7533,6 +7553,181 @@ static int tgsi_tex(struct r600_shader_ctx *ctx) return 0; } +static int find_hw_atomic_counter(struct r600_shader_ctx *ctx, + struct tgsi_full_src_register *src) +{ + int i; + + if (src->Register.Indirect) { + for (i = 0; i < ctx->shader->nhwatomic_ranges; i++) { + if (src->Indirect.ArrayID == ctx->shader->atomics[i].array_id) + return ctx->shader->atomics[i].hw_idx; + } + } else { + uint32_t index = src->Register.Index; + for (i = 0; i < ctx->shader->nhwatomic_ranges; i++) { + if (ctx->shader->atomics[i].buffer_id != src->Dimension.Index) + continue; + if (index > ctx->shader->atomics[i].end) + continue; + if (index < ctx->shader->atomics[i].start) + continue; + uint32_t offset = (index - ctx->shader->atomics[i].start); + return ctx->shader->atomics[i].hw_idx + offset; + } + } + assert(0); + return -1; +} + + +static int tgsi_load_gds(struct r600_shader_ctx *ctx) +{ + struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction; + int r; + struct r600_bytecode_gds gds; + int uav_id = 0; + int uav_index_mode = 0; + + uav_id = find_hw_atomic_counter(ctx, &inst->Src[0]); + + if (inst->Src[0].Register.Indirect) + uav_index_mode = 2; + + memset(&gds, 0, sizeof(struct r600_bytecode_gds)); + gds.op = FETCH_OP_GDS_READ_RET; + gds.dst_gpr = ctx->file_offset[inst->Dst[0].Register.File] + inst->Dst[0].Register.Index; + gds.uav_id = uav_id; + gds.uav_index_mode = uav_index_mode; + gds.src_gpr = ctx->temp_reg; + gds.src_sel_x = 4; + gds.src_sel_y = 4; + gds.src_sel_z = 4; + gds.dst_sel_x = 0; + gds.dst_sel_y = 7; + gds.dst_sel_z = 7; + gds.dst_sel_w = 7; + gds.src_gpr2 = ctx->temp_reg; + gds.alloc_consume = 1; + r = r600_bytecode_add_gds(ctx->bc, &gds); + if (r) + return r; + + ctx->bc->cf_last->vpm = 1; + return 0; +} + +static int tgsi_load(struct r600_shader_ctx *ctx) +{ + struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction; + if (inst->Src[0].Register.File == TGSI_FILE_HW_ATOMIC) + return tgsi_load_gds(ctx); + return 0; +} + +static int get_gds_op(int opcode) +{ + switch (opcode) { + case TGSI_OPCODE_ATOMUADD: + return FETCH_OP_GDS_ADD_RET; + case TGSI_OPCODE_ATOMAND: + return FETCH_OP_GDS_AND_RET; + case TGSI_OPCODE_ATOMOR: + return FETCH_OP_GDS_OR_RET; + case TGSI_OPCODE_ATOMXOR: + return FETCH_OP_GDS_XOR_RET; + case TGSI_OPCODE_ATOMUMIN: + return FETCH_OP_GDS_MIN_UINT_RET; + case TGSI_OPCODE_ATOMUMAX: + return FETCH_OP_GDS_MAX_UINT_RET; + case TGSI_OPCODE_ATOMXCHG: + return FETCH_OP_GDS_XCHG_RET; + case TGSI_OPCODE_ATOMCAS: + return FETCH_OP_GDS_CMP_XCHG_RET; + default: + return -1; + } +} + +static int tgsi_atomic_op_gds(struct r600_shader_ctx *ctx) +{ + struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction; + struct r600_bytecode_gds gds; + struct r600_bytecode_alu alu; + int gds_op = get_gds_op(inst->Instruction.Opcode); + int r; + int uav_id = 0; + int uav_index_mode = 0; + + if (gds_op == -1) { + fprintf(stderr, "unknown GDS op for opcode %d\n", inst->Instruction.Opcode); + return -1; + } + + uav_id = find_hw_atomic_counter(ctx, &inst->Src[0]); + + if (inst->Src[0].Register.Indirect) + uav_index_mode = 2; + + if (inst->Src[2].Register.File == TGSI_FILE_IMMEDIATE) { + int value = (ctx->literals[4 * inst->Src[2].Register.Index + inst->Src[2].Register.SwizzleX]); + int abs_value = abs(value); + if (abs_value != value && gds_op == FETCH_OP_GDS_ADD_RET) + gds_op = FETCH_OP_GDS_SUB_RET; + memset(&alu, 0, sizeof(struct r600_bytecode_alu)); + alu.op = ALU_OP1_MOV; + alu.dst.sel = ctx->temp_reg; + alu.dst.chan = 0; + alu.src[0].sel = V_SQ_ALU_SRC_LITERAL; + alu.src[0].value = abs_value; + alu.last = 1; + alu.dst.write = 1; + r = r600_bytecode_add_alu(ctx->bc, &alu); + if (r) + return r; + } else { + memset(&alu, 0, sizeof(struct r600_bytecode_alu)); + alu.op = ALU_OP1_MOV; + alu.dst.sel = ctx->temp_reg; + alu.dst.chan = 0; + r600_bytecode_src(&alu.src[0], &ctx->src[2], 0); + alu.last = 1; + alu.dst.write = 1; + r = r600_bytecode_add_alu(ctx->bc, &alu); + if (r) + return r; + } + + memset(&gds, 0, sizeof(struct r600_bytecode_gds)); + gds.op = gds_op; + gds.dst_gpr = ctx->file_offset[inst->Dst[0].Register.File] + inst->Dst[0].Register.Index; + gds.uav_id = uav_id; + gds.uav_index_mode = uav_index_mode; + gds.src_gpr = ctx->temp_reg; + gds.src_gpr2 = ctx->temp_reg; + gds.src_sel_x = 4; + gds.src_sel_y = 0; + gds.src_sel_z = 4; + gds.dst_sel_x = 0; + gds.dst_sel_y = 7; + gds.dst_sel_z = 7; + gds.dst_sel_w = 7; + gds.alloc_consume = 1; + r = r600_bytecode_add_gds(ctx->bc, &gds); + if (r) + return r; + ctx->bc->cf_last->vpm = 1; + return 0; +} + +static int tgsi_atomic_op(struct r600_shader_ctx *ctx) +{ + struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction; + if (inst->Src[0].Register.File == TGSI_FILE_HW_ATOMIC) + return tgsi_atomic_op_gds(ctx); + return 0; +} + static int tgsi_lrp(struct r600_shader_ctx *ctx) { struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction; @@ -9190,22 +9385,22 @@ static const struct r600_shader_tgsi_instruction eg_shader_tgsi_instruction[] = [TGSI_OPCODE_UCMP] = { ALU_OP0_NOP, tgsi_ucmp}, [TGSI_OPCODE_IABS] = { 0, tgsi_iabs}, [TGSI_OPCODE_ISSG] = { 0, tgsi_issg}, - [TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_unsupported}, + [TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_load}, [TGSI_OPCODE_STORE] = { ALU_OP0_NOP, tgsi_unsupported}, [163] = { ALU_OP0_NOP, tgsi_unsupported}, [164] = { ALU_OP0_NOP, tgsi_unsupported}, [165] = { ALU_OP0_NOP, tgsi_unsupported}, [TGSI_OPCODE_BARRIER] = { ALU_OP0_GROUP_BARRIER, tgsi_barrier}, - [TGSI_OPCODE_ATOMUADD] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMXCHG] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMCAS] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMAND] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMOR] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMXOR] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMUMIN] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMUMAX] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMIMIN] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMIMAX] = { ALU_OP0_NOP, tgsi_unsupported}, + [TGSI_OPCODE_ATOMUADD] = { V_RAT_INST_ADD_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMXCHG] = { V_RAT_INST_XCHG_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMCAS] = { V_RAT_INST_CMPXCHG_INT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMAND] = { V_RAT_INST_AND_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMOR] = { V_RAT_INST_OR_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMXOR] = { V_RAT_INST_XOR_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMUMIN] = { V_RAT_INST_MIN_UINT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMUMAX] = { V_RAT_INST_MAX_UINT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMIMIN] = { V_RAT_INST_MIN_INT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMIMAX] = { V_RAT_INST_MAX_INT_RTN, tgsi_atomic_op}, [TGSI_OPCODE_TEX2] = { FETCH_OP_SAMPLE, tgsi_tex}, [TGSI_OPCODE_TXB2] = { FETCH_OP_SAMPLE_LB, tgsi_tex}, [TGSI_OPCODE_TXL2] = { FETCH_OP_SAMPLE_L, tgsi_tex}, @@ -9413,22 +9608,22 @@ static const struct r600_shader_tgsi_instruction cm_shader_tgsi_instruction[] = [TGSI_OPCODE_UCMP] = { ALU_OP0_NOP, tgsi_ucmp}, [TGSI_OPCODE_IABS] = { 0, tgsi_iabs}, [TGSI_OPCODE_ISSG] = { 0, tgsi_issg}, - [TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_unsupported}, + [TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_load}, [TGSI_OPCODE_STORE] = { ALU_OP0_NOP, tgsi_unsupported}, [163] = { ALU_OP0_NOP, tgsi_unsupported}, [164] = { ALU_OP0_NOP, tgsi_unsupported}, [165] = { ALU_OP0_NOP, tgsi_unsupported}, [TGSI_OPCODE_BARRIER] = { ALU_OP0_GROUP_BARRIER, tgsi_barrier}, - [TGSI_OPCODE_ATOMUADD] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMXCHG] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMCAS] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMAND] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMOR] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMXOR] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMUMIN] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMUMAX] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMIMIN] = { ALU_OP0_NOP, tgsi_unsupported}, - [TGSI_OPCODE_ATOMIMAX] = { ALU_OP0_NOP, tgsi_unsupported}, + [TGSI_OPCODE_ATOMUADD] = { V_RAT_INST_ADD_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMXCHG] = { V_RAT_INST_XCHG_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMCAS] = { V_RAT_INST_CMPXCHG_INT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMAND] = { V_RAT_INST_AND_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMOR] = { V_RAT_INST_OR_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMXOR] = { V_RAT_INST_XOR_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMUMIN] = { V_RAT_INST_MIN_UINT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMUMAX] = { V_RAT_INST_MAX_UINT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMIMIN] = { V_RAT_INST_MIN_INT_RTN, tgsi_atomic_op}, + [TGSI_OPCODE_ATOMIMAX] = { V_RAT_INST_MAX_INT_RTN, tgsi_atomic_op}, [TGSI_OPCODE_TEX2] = { FETCH_OP_SAMPLE, tgsi_tex}, [TGSI_OPCODE_TXB2] = { FETCH_OP_SAMPLE_LB, tgsi_tex}, [TGSI_OPCODE_TXL2] = { FETCH_OP_SAMPLE_L, tgsi_tex}, diff --git a/src/gallium/drivers/r600/r600_shader.h b/src/gallium/drivers/r600/r600_shader.h index 9032d508383..3fecda4c800 100644 --- a/src/gallium/drivers/r600/r600_shader.h +++ b/src/gallium/drivers/r600/r600_shader.h @@ -56,15 +56,25 @@ struct r600_shader_io { int ring_offset; }; +struct r600_shader_atomic { + unsigned start, end; + unsigned buffer_id; + unsigned hw_idx; + unsigned array_id; +}; + struct r600_shader { unsigned processor_type; struct r600_bytecode bc; unsigned ninput; unsigned noutput; + unsigned nhwatomic; unsigned nlds; unsigned nsys_inputs; struct r600_shader_io input[64]; struct r600_shader_io output[64]; + struct r600_shader_atomic atomics[8]; + unsigned nhwatomic_ranges; boolean uses_kill; boolean fs_write_all; boolean two_side; @@ -105,26 +115,35 @@ struct r600_shader { struct r600_shader_array * arrays; boolean uses_doubles; + boolean uses_atomics; + uint8_t atomic_base; }; union r600_shader_key { struct { unsigned nr_cbufs:4; + unsigned first_atomic_counter:4; unsigned color_two_side:1; unsigned alpha_to_one:1; } ps; struct { unsigned prim_id_out:8; + unsigned first_atomic_counter:4; unsigned as_es:1; /* export shader */ unsigned as_ls:1; /* local shader */ unsigned as_gs_a:1; } vs; struct { + unsigned first_atomic_counter:4; unsigned as_es:1; } tes; struct { + unsigned first_atomic_counter:4; unsigned prim_mode:3; } tcs; + struct { + unsigned first_atomic_counter:4; + } gs; }; struct r600_shader_array { diff --git a/src/gallium/drivers/r600/r600_state_common.c b/src/gallium/drivers/r600/r600_state_common.c index 0e8c5d666ef..750fd411baf 100644 --- a/src/gallium/drivers/r600/r600_state_common.c +++ b/src/gallium/drivers/r600/r600_state_common.c @@ -698,6 +698,38 @@ static void r600_update_compressed_colortex_mask(struct r600_samplerview_state * } } +static int r600_get_hw_atomic_count(const struct pipe_context *ctx, + enum pipe_shader_type shader) +{ + const struct r600_context *rctx = (struct r600_context *)ctx; + int value = 0; + switch (shader) { + case PIPE_SHADER_FRAGMENT: + case PIPE_SHADER_COMPUTE: + default: + break; + case PIPE_SHADER_VERTEX: + value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC]; + break; + case PIPE_SHADER_GEOMETRY: + value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC] + + rctx->vs_shader->info.file_count[TGSI_FILE_HW_ATOMIC]; + break; + case PIPE_SHADER_TESS_EVAL: + value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC] + + rctx->vs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] + + (rctx->gs_shader ? rctx->gs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] : 0); + break; + case PIPE_SHADER_TESS_CTRL: + value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC] + + rctx->vs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] + + (rctx->gs_shader ? rctx->gs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] : 0) + + rctx->tes_shader->info.file_count[TGSI_FILE_HW_ATOMIC]; + break; + } + return value; +} + /* Compute the key for the hw shader variant */ static inline void r600_shader_selector_key(const struct pipe_context *ctx, const struct r600_pipe_shader_selector *sel, @@ -716,11 +748,14 @@ static inline void r600_shader_selector_key(const struct pipe_context *ctx, key->vs.as_gs_a = true; key->vs.prim_id_out = rctx->ps_shader->current->shader.input[rctx->ps_shader->current->shader.ps_prim_id_input].spi_sid; } + key->vs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_VERTEX); break; } case PIPE_SHADER_GEOMETRY: + key->gs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_GEOMETRY); break; case PIPE_SHADER_FRAGMENT: { + key->ps.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_FRAGMENT); key->ps.color_two_side = rctx->rasterizer && rctx->rasterizer->two_side; key->ps.alpha_to_one = rctx->alpha_to_one && rctx->rasterizer && rctx->rasterizer->multisample_enable && @@ -733,9 +768,11 @@ static inline void r600_shader_selector_key(const struct pipe_context *ctx, } case PIPE_SHADER_TESS_EVAL: key->tes.as_es = (rctx->gs_shader != NULL); + key->tes.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_TESS_EVAL); break; case PIPE_SHADER_TESS_CTRL: key->tcs.prim_mode = rctx->tes_shader->info.properties[TGSI_PROPERTY_TES_PRIM_MODE]; + key->tcs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_TESS_CTRL); break; default: assert(0); @@ -1700,6 +1737,8 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info unsigned num_patches, dirty_tex_counter, index_offset = 0; unsigned index_size = info->index_size; int index_bias; + struct r600_shader_atomic combined_atomics[8]; + uint8_t atomic_used_mask; if (!info->indirect && !info->count && (index_size || !info->count_from_stream_output)) { return; @@ -1739,6 +1778,9 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info : (rctx->tes_shader)? rctx->tes_shader->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] : info->mode; + if (rctx->b.chip_class >= EVERGREEN) + evergreen_emit_atomic_buffer_setup(rctx, combined_atomics, &atomic_used_mask); + if (index_size) { index_offset += info->start * index_size; @@ -2019,6 +2061,10 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_SQ_NON_EVENT)); } + + if (rctx->b.chip_class >= EVERGREEN) + evergreen_emit_atomic_buffer_save(rctx, combined_atomics, &atomic_used_mask); + if (rctx->trace_buf) eg_trace_emit(rctx); diff --git a/src/gallium/drivers/r600/r600d_common.h b/src/gallium/drivers/r600/r600d_common.h index ed1d46076c0..b06f90f8edd 100644 --- a/src/gallium/drivers/r600/r600d_common.h +++ b/src/gallium/drivers/r600/r600d_common.h @@ -51,6 +51,8 @@ #define STRMOUT_SELECT_BUFFER(x) (((unsigned)(x) & 0x3) << 8) #define PKT3_WAIT_REG_MEM 0x3C #define WAIT_REG_MEM_EQUAL 3 +#define WAIT_REG_MEM_GEQUAL 5 +#define WAIT_REG_MEM_MEMORY (1 << 4) #define WAIT_REG_MEM_MEM_SPACE(x) (((unsigned)(x) & 0x3) << 4) #define PKT3_COPY_DATA 0x40 #define COPY_DATA_SRC_SEL(x) ((x) & 0xf)