X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fgfx10_shader_ngg.c;h=f5899511138158f5c46199dfa0df152a73489d47;hb=6ec7d603f58992904f2e0cbfe9f6e417c34c9ef5;hp=74be6df30f941390c78faaa079aa51f3fa1864a7;hpb=1b4354dab9f81c92ff83f98d8d65e3c9d304d421;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c b/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c index 74be6df30f9..f5899511138 100644 --- a/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c +++ b/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c @@ -44,7 +44,7 @@ static LLVMValueRef get_thread_id_in_tg(struct si_shader_context *ctx) LLVMBuilderRef builder = ctx->ac.builder; LLVMValueRef tmp; tmp = LLVMBuildMul(builder, get_wave_id_in_tg(ctx), - LLVMConstInt(ctx->ac.i32, 64, false), ""); + LLVMConstInt(ctx->ac.i32, ctx->ac.wave_size, false), ""); return LLVMBuildAdd(builder, tmp, ac_get_thread_id(&ctx->ac), ""); } @@ -496,6 +496,20 @@ static void build_streamout(struct si_shader_context *ctx, } } +static unsigned ngg_nogs_vertex_size(struct si_shader *shader) +{ + unsigned lds_vertex_size = 0; + + /* The edgeflag is always stored in the last element that's also + * used for padding to reduce LDS bank conflicts. */ + if (shader->selector->so.num_outputs) + lds_vertex_size = 4 * shader->selector->info.num_outputs + 1; + if (shader->selector->ngg_writes_edgeflag) + lds_vertex_size = MAX2(lds_vertex_size, 1); + + return lds_vertex_size; +} + /** * Returns an `[N x i32] addrspace(LDS)*` pointing at contiguous LDS storage * for the vertex outputs. @@ -504,7 +518,7 @@ static LLVMValueRef ngg_nogs_vertex_ptr(struct si_shader_context *ctx, LLVMValueRef vtxid) { /* The extra dword is used to avoid LDS bank conflicts. */ - unsigned vertex_size = 4 * ctx->shader->selector->info.num_outputs + 1; + unsigned vertex_size = ngg_nogs_vertex_size(ctx->shader); LLVMTypeRef ai32 = LLVMArrayType(ctx->i32, vertex_size); LLVMTypeRef pai32 = LLVMPointerType(ai32, AC_ADDR_SPACE_LDS); LLVMValueRef tmp = LLVMBuildBitCast(ctx->ac.builder, ctx->esgs_ring, pai32, ""); @@ -521,41 +535,50 @@ void gfx10_emit_ngg_epilogue(struct ac_shader_abi *abi, struct si_shader_context *ctx = si_shader_context_from_abi(abi); struct si_shader_selector *sel = ctx->shader->selector; struct tgsi_shader_info *info = &sel->info; - struct si_shader_output_values *outputs = NULL; + struct si_shader_output_values outputs[PIPE_MAX_SHADER_OUTPUTS]; LLVMBuilderRef builder = ctx->ac.builder; - struct lp_build_if_state if_state; LLVMValueRef tmp, tmp2; assert(!ctx->shader->is_gs_copy_shader); assert(info->num_outputs <= max_outputs); - outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0])); - LLVMValueRef vertex_ptr = NULL; - if (sel->so.num_outputs) + if (sel->so.num_outputs || sel->ngg_writes_edgeflag) vertex_ptr = ngg_nogs_vertex_ptr(ctx, get_thread_id_in_tg(ctx)); for (unsigned i = 0; i < info->num_outputs; i++) { outputs[i].semantic_name = info->output_semantic_name[i]; outputs[i].semantic_index = info->output_semantic_index[i]; - /* This is used only by streamout. */ for (unsigned j = 0; j < 4; j++) { - outputs[i].values[j] = - LLVMBuildLoad(builder, - addrs[4 * i + j], - ""); outputs[i].vertex_stream[j] = (info->output_streams[i] >> (2 * j)) & 3; - if (vertex_ptr) { + /* TODO: we may store more outputs than streamout needs, + * but streamout performance isn't that important. + */ + if (sel->so.num_outputs) { tmp = ac_build_gep0(&ctx->ac, vertex_ptr, LLVMConstInt(ctx->i32, 4 * i + j, false)); - tmp2 = ac_to_integer(&ctx->ac, outputs[i].values[j]); + tmp2 = LLVMBuildLoad(builder, addrs[4 * i + j], ""); + tmp2 = ac_to_integer(&ctx->ac, tmp2); LLVMBuildStore(builder, tmp2, tmp); } } + + /* Store the edgeflag at the end (if streamout is enabled) */ + if (info->output_semantic_name[i] == TGSI_SEMANTIC_EDGEFLAG && + sel->ngg_writes_edgeflag) { + LLVMValueRef edgeflag = LLVMBuildLoad(builder, addrs[4 * i], ""); + /* The output is a float, but the hw expects a 1-bit integer. */ + edgeflag = LLVMBuildFPToUI(ctx->ac.builder, edgeflag, ctx->i32, ""); + edgeflag = ac_build_umin(&ctx->ac, edgeflag, ctx->i32_1); + + tmp = LLVMConstInt(ctx->i32, ngg_nogs_vertex_size(ctx->shader) - 1, 0); + tmp = ac_build_gep0(&ctx->ac, vertex_ptr, tmp); + LLVMBuildStore(builder, edgeflag, tmp); + } } lp_build_endif(&ctx->merged_wrap_if_state); @@ -616,13 +639,35 @@ void gfx10_emit_ngg_epilogue(struct ac_shader_abi *abi, emitted_prims = nggso.emit[0]; } + LLVMValueRef user_edgeflags[3] = {}; + + if (sel->ngg_writes_edgeflag) { + /* Streamout already inserted the barrier, so don't insert it again. */ + if (!sel->so.num_outputs) + ac_build_s_barrier(&ctx->ac); + + ac_build_ifcc(&ctx->ac, is_gs_thread, 5400); + /* Load edge flags from ES threads and store them into VGPRs in GS threads. */ + for (unsigned i = 0; i < num_vertices; i++) { + tmp = ngg_nogs_vertex_ptr(ctx, vtxindex[i]); + tmp2 = LLVMConstInt(ctx->i32, ngg_nogs_vertex_size(ctx->shader) - 1, 0); + tmp = ac_build_gep0(&ctx->ac, tmp, tmp2); + tmp = LLVMBuildLoad(builder, tmp, ""); + tmp = LLVMBuildTrunc(builder, tmp, ctx->i1, ""); + + user_edgeflags[i] = ac_build_alloca_undef(&ctx->ac, ctx->i1, ""); + LLVMBuildStore(builder, tmp, user_edgeflags[i]); + } + ac_build_endif(&ctx->ac, 5400); + } + /* Copy Primitive IDs from GS threads to the LDS address corresponding * to the ES thread of the provoking vertex. */ if (ctx->type == PIPE_SHADER_VERTEX && ctx->shader->key.mono.u.vs_export_prim_id) { - /* Streamout uses LDS. We need to wait for it before we can reuse it. */ - if (sel->so.num_outputs) + /* Streamout and edge flags use LDS. Make it idle, so that we can reuse it. */ + if (sel->so.num_outputs || sel->ngg_writes_edgeflag) ac_build_s_barrier(&ctx->ac); ac_build_ifcc(&ctx->ac, is_gs_thread, 5400); @@ -640,8 +685,6 @@ void gfx10_emit_ngg_epilogue(struct ac_shader_abi *abi, ac_build_endif(&ctx->ac, 5400); } - /* TODO: primitive culling */ - build_sendmsg_gs_alloc_req(ctx, ngg_get_vtx_cnt(ctx), ngg_get_prim_cnt(ctx)); /* Update query buffer */ @@ -695,7 +738,7 @@ void gfx10_emit_ngg_epilogue(struct ac_shader_abi *abi, * TODO: culling depends on the primitive type, so can have some * interaction here. */ - lp_build_if(&if_state, &ctx->gallivm, is_gs_thread); + ac_build_ifcc(&ctx->ac, is_gs_thread, 6001); { struct ngg_prim prim = {}; @@ -704,17 +747,28 @@ void gfx10_emit_ngg_epilogue(struct ac_shader_abi *abi, memcpy(prim.index, vtxindex, sizeof(vtxindex[0]) * 3); for (unsigned i = 0; i < num_vertices; ++i) { + if (ctx->type != PIPE_SHADER_VERTEX) { + prim.edgeflag[i] = ctx->i1false; + continue; + } + tmp = LLVMBuildLShr(builder, ctx->abi.gs_invocation_id, LLVMConstInt(ctx->ac.i32, 8 + i, false), ""); prim.edgeflag[i] = LLVMBuildTrunc(builder, tmp, ctx->ac.i1, ""); + + if (sel->ngg_writes_edgeflag) { + tmp2 = LLVMBuildLoad(builder, user_edgeflags[i], ""); + prim.edgeflag[i] = LLVMBuildAnd(builder, prim.edgeflag[i], + tmp2, ""); + } } build_export_prim(ctx, &prim); } - lp_build_endif(&if_state); + ac_build_endif(&ctx->ac, 6001); /* Export per-vertex data (positions and parameters). */ - lp_build_if(&if_state, &ctx->gallivm, is_es_thread); + ac_build_ifcc(&ctx->ac, is_es_thread, 6002); { unsigned i; @@ -755,9 +809,7 @@ void gfx10_emit_ngg_epilogue(struct ac_shader_abi *abi, si_llvm_export_vs(ctx, outputs, i); } - lp_build_endif(&if_state); - - FREE(outputs); + ac_build_endif(&ctx->ac, 6002); } static LLVMValueRef @@ -849,7 +901,6 @@ void gfx10_ngg_gs_emit_vertex(struct si_shader_context *ctx, const struct si_shader_selector *sel = ctx->shader->selector; const struct tgsi_shader_info *info = &sel->info; LLVMBuilderRef builder = ctx->ac.builder; - struct lp_build_if_state if_state; LLVMValueRef tmp; const LLVMValueRef vertexidx = LLVMBuildLoad(builder, ctx->gs_next_vertex[stream], ""); @@ -866,7 +917,7 @@ void gfx10_ngg_gs_emit_vertex(struct si_shader_context *ctx, tmp = LLVMBuildSelect(builder, can_emit, tmp, vertexidx, ""); LLVMBuildStore(builder, tmp, ctx->gs_next_vertex[stream]); - lp_build_if(&if_state, &ctx->gallivm, can_emit); + ac_build_ifcc(&ctx->ac, can_emit, 9001); const LLVMValueRef vertexptr = ngg_gs_emit_vertex_ptr(ctx, get_thread_id_in_tg(ctx), vertexidx); @@ -916,7 +967,7 @@ void gfx10_ngg_gs_emit_vertex(struct si_shader_context *ctx, tmp = LLVMBuildAdd(builder, tmp, LLVMBuildZExt(builder, iscompleteprim, ctx->ac.i32, ""), ""); LLVMBuildStore(builder, tmp, ctx->gs_generated_prims[stream]); - lp_build_endif(&if_state); + ac_build_endif(&ctx->ac, 9001); } void gfx10_ngg_gs_emit_prologue(struct si_shader_context *ctx) @@ -994,7 +1045,7 @@ void gfx10_ngg_gs_emit_epilogue(struct si_shader_context *ctx) LLVMValueRef numprims = LLVMBuildLoad(builder, ctx->gs_generated_prims[stream], ""); - numprims = ac_build_reduce(&ctx->ac, numprims, nir_op_iadd, 64); + numprims = ac_build_reduce(&ctx->ac, numprims, nir_op_iadd, ctx->ac.wave_size); tmp = LLVMBuildICmp(builder, LLVMIntEQ, ac_get_thread_id(&ctx->ac), ctx->i32_0, ""); ac_build_ifcc(&ctx->ac, tmp, 5105); @@ -1084,7 +1135,7 @@ void gfx10_ngg_gs_emit_epilogue(struct si_shader_context *ctx) /* TODO: culling */ /* Determine vertex liveness. */ - LLVMValueRef vertliveptr = lp_build_alloca(&ctx->gallivm, ctx->ac.i1, "vertexlive"); + LLVMValueRef vertliveptr = ac_build_alloca(&ctx->ac, ctx->ac.i1, "vertexlive"); tmp = LLVMBuildICmp(builder, LLVMIntULT, tid, num_emit_threads, ""); ac_build_ifcc(&ctx->ac, tmp, 5120); @@ -1206,8 +1257,7 @@ void gfx10_ngg_gs_emit_epilogue(struct si_shader_context *ctx) tmp = LLVMBuildICmp(builder, LLVMIntULT, tid, vertlive_scan.result_reduce, ""); ac_build_ifcc(&ctx->ac, tmp, 5145); { - struct si_shader_output_values *outputs = NULL; - outputs = MALLOC(info->num_outputs * sizeof(outputs[0])); + struct si_shader_output_values outputs[PIPE_MAX_SHADER_OUTPUTS]; tmp = ngg_gs_vertex_ptr(ctx, tid); LLVMValueRef gep_idx[3] = { @@ -1237,8 +1287,6 @@ void gfx10_ngg_gs_emit_epilogue(struct si_shader_context *ctx) } si_llvm_export_vs(ctx, outputs, info->num_outputs); - - FREE(outputs); } ac_build_endif(&ctx->ac, 5145); } @@ -1265,10 +1313,7 @@ void gfx10_ngg_calculate_subgroup_info(struct si_shader *shader) shader->previous_stage_sel ? shader->previous_stage_sel : gs_sel; const enum pipe_shader_type gs_type = gs_sel->type; const unsigned gs_num_invocations = MAX2(gs_sel->gs_num_invocations, 1); - /* TODO: Use QUADS as the worst case because of reuse, but triangles - * will always have 1 additional unoccupied vector lane. We could use - * that lane if the worst case was TRIANGLES. */ - const unsigned input_prim = si_get_input_prim(gs_sel, PIPE_PRIM_QUADS); + const unsigned input_prim = si_get_input_prim(gs_sel); const bool use_adjacency = input_prim >= PIPE_PRIM_LINES_ADJACENCY && input_prim <= PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY; const unsigned max_verts_per_prim = u_vertices_per_prim(input_prim); @@ -1324,12 +1369,12 @@ void gfx10_ngg_calculate_subgroup_info(struct si_shader *shader) esvert_lds_size = es_sel->esgs_itemsize / 4; gsprim_lds_size = (gs_sel->gsvs_vertex_size / 4 + 1) * max_out_verts_per_gsprim; } else { - /* TODO: This needs to be adjusted once LDS use for compaction - * after culling is implemented. */ - if (es_sel->so.num_outputs) - esvert_lds_size = 4 * es_sel->info.num_outputs + 1; + /* VS and TES. */ + /* LDS size for passing data from ES to GS. */ + esvert_lds_size = ngg_nogs_vertex_size(shader); - /* GS stores Primitive IDs into LDS at the address corresponding + /* LDS size for passing data from GS to ES. + * GS stores Primitive IDs into LDS at the address corresponding * to the ES thread of the provoking vertex. All ES threads * load and export PrimitiveID for their thread. */ @@ -1373,7 +1418,7 @@ void gfx10_ngg_calculate_subgroup_info(struct si_shader *shader) /* Round up towards full wave sizes for better ALU utilization. */ if (!max_vert_out_per_gs_instance) { - const unsigned wavesize = 64; + const unsigned wavesize = gs_sel->screen->ge_wave_size; unsigned orig_max_esverts; unsigned orig_max_gsprims; do {