struct si_shader_output_values
{
LLVMValueRef values[4];
- unsigned name;
- unsigned sid;
+ unsigned semantic_name;
+ unsigned semantic_index;
+ ubyte vertex_stream[4];
};
static void si_init_shader_ctx(struct si_shader_context *ctx,
args[1] = attr_number;
args[2] = params;
args[3] = lp_build_gather_values(gallivm, ij, 2);
- return lp_build_intrinsic(gallivm->builder, "llvm.fs.interp",
+ return lp_build_intrinsic(gallivm->builder, "llvm.SI.fs.interp",
ctx->f32, args, 4,
LP_FUNC_ATTR_READNONE);
}
}
}
-/* On SI, the vertex shader is responsible for writing streamout data
- * to buffers. */
-static void si_llvm_emit_streamout(struct si_shader_context *ctx,
- struct si_shader_output_values *outputs,
- unsigned noutput)
+static void emit_streamout_output(struct si_shader_context *ctx,
+ LLVMValueRef const *so_buffers,
+ LLVMValueRef const *so_write_offsets,
+ struct pipe_stream_output *stream_out,
+ struct si_shader_output_values *shader_out)
{
- struct pipe_stream_output_info *so = &ctx->shader->selector->so;
struct gallivm_state *gallivm = &ctx->gallivm;
LLVMBuilderRef builder = gallivm->builder;
- int i, j;
- struct lp_build_if_state if_ctx;
- LLVMValueRef so_buffers[4];
- LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
- SI_PARAM_RW_BUFFERS);
+ unsigned buf_idx = stream_out->output_buffer;
+ unsigned start = stream_out->start_component;
+ unsigned num_comps = stream_out->num_components;
+ LLVMValueRef out[4];
- /* Load the descriptors. */
- for (i = 0; i < 4; ++i) {
- if (ctx->shader->selector->so.stride[i]) {
- LLVMValueRef offset = lp_build_const_int32(gallivm,
- SI_VS_STREAMOUT_BUF0 + i);
+ assert(num_comps && num_comps <= 4);
+ if (!num_comps || num_comps > 4)
+ return;
- so_buffers[i] = build_indexed_load_const(ctx, buf_ptr, offset);
+ /* Load the output as int. */
+ for (int j = 0; j < num_comps; j++) {
+ assert(stream_out->stream == shader_out->vertex_stream[start + j]);
+
+ out[j] = LLVMBuildBitCast(builder,
+ shader_out->values[start + j],
+ ctx->i32, "");
+ }
+
+ /* Pack the output. */
+ LLVMValueRef vdata = NULL;
+
+ switch (num_comps) {
+ case 1: /* as i32 */
+ vdata = out[0];
+ break;
+ case 2: /* as v2i32 */
+ case 3: /* as v4i32 (aligned to 4) */
+ case 4: /* as v4i32 */
+ vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
+ for (int j = 0; j < num_comps; j++) {
+ vdata = LLVMBuildInsertElement(builder, vdata, out[j],
+ LLVMConstInt(ctx->i32, j, 0), "");
}
+ break;
}
+ build_tbuffer_store_dwords(ctx, so_buffers[buf_idx],
+ vdata, num_comps,
+ so_write_offsets[buf_idx],
+ LLVMConstInt(ctx->i32, 0, 0),
+ stream_out->dst_offset * 4);
+}
+
+/**
+ * Write streamout data to buffers for vertex stream @p stream (different
+ * vertex streams can occur for GS copy shaders).
+ */
+static void si_llvm_emit_streamout(struct si_shader_context *ctx,
+ struct si_shader_output_values *outputs,
+ unsigned noutput, unsigned stream)
+{
+ struct si_shader_selector *sel = ctx->shader->selector;
+ struct pipe_stream_output_info *so = &sel->so;
+ struct gallivm_state *gallivm = &ctx->gallivm;
+ LLVMBuilderRef builder = gallivm->builder;
+ int i;
+ struct lp_build_if_state if_ctx;
+
/* Get bits [22:16], i.e. (so_param >> 16) & 127; */
LLVMValueRef so_vtx_count =
unpack_param(ctx, ctx->param_streamout_config, 16, 7);
LLVMValueRef can_emit =
LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
- LLVMValueRef stream_id =
- unpack_param(ctx, ctx->param_streamout_config, 24, 2);
-
/* Emit the streamout code conditionally. This actually avoids
* out-of-bounds buffer access. The hw tells us via the SGPR
* (so_vtx_count) which threads are allowed to emit streamout data. */
/* Compute (streamout_write_index + thread_id). */
so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
- /* Compute the write offset for each enabled buffer. */
+ /* Load the descriptor and compute the write offset for each
+ * enabled buffer. */
LLVMValueRef so_write_offset[4] = {};
+ LLVMValueRef so_buffers[4];
+ LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
+ SI_PARAM_RW_BUFFERS);
+
for (i = 0; i < 4; i++) {
if (!so->stride[i])
continue;
+ LLVMValueRef offset = lp_build_const_int32(gallivm,
+ SI_VS_STREAMOUT_BUF0 + i);
+
+ so_buffers[i] = build_indexed_load_const(ctx, buf_ptr, offset);
+
LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
ctx->param_streamout_offset[i]);
so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
/* Write streamout data. */
for (i = 0; i < so->num_outputs; i++) {
- unsigned buf_idx = so->output[i].output_buffer;
unsigned reg = so->output[i].register_index;
- unsigned start = so->output[i].start_component;
- unsigned num_comps = so->output[i].num_components;
- unsigned stream = so->output[i].stream;
- LLVMValueRef out[4];
- struct lp_build_if_state if_ctx_stream;
-
- assert(num_comps && num_comps <= 4);
- if (!num_comps || num_comps > 4)
- continue;
if (reg >= noutput)
continue;
- /* Load the output as int. */
- for (j = 0; j < num_comps; j++) {
- out[j] = LLVMBuildBitCast(builder,
- outputs[reg].values[start+j],
- ctx->i32, "");
- }
-
- /* Pack the output. */
- LLVMValueRef vdata = NULL;
-
- switch (num_comps) {
- case 1: /* as i32 */
- vdata = out[0];
- break;
- case 2: /* as v2i32 */
- case 3: /* as v4i32 (aligned to 4) */
- case 4: /* as v4i32 */
- vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
- for (j = 0; j < num_comps; j++) {
- vdata = LLVMBuildInsertElement(builder, vdata, out[j],
- LLVMConstInt(ctx->i32, j, 0), "");
- }
- break;
- }
+ if (stream != so->output[i].stream)
+ continue;
- LLVMValueRef can_emit_stream =
- LLVMBuildICmp(builder, LLVMIntEQ,
- stream_id,
- lp_build_const_int32(gallivm, stream), "");
-
- lp_build_if(&if_ctx_stream, gallivm, can_emit_stream);
- build_tbuffer_store_dwords(ctx, so_buffers[buf_idx],
- vdata, num_comps,
- so_write_offset[buf_idx],
- LLVMConstInt(ctx->i32, 0, 0),
- so->output[i].dst_offset*4);
- lp_build_endif(&if_ctx_stream);
+ emit_streamout_output(ctx, so_buffers, so_write_offset,
+ &so->output[i], &outputs[reg]);
}
}
lp_build_endif(&if_ctx);
unsigned pos_idx;
int i;
- if (outputs && ctx->shader->selector->so.num_outputs) {
- si_llvm_emit_streamout(ctx, outputs, noutput);
- }
-
for (i = 0; i < noutput; i++) {
- semantic_name = outputs[i].name;
- semantic_index = outputs[i].sid;
+ semantic_name = outputs[i].semantic_name;
+ semantic_index = outputs[i].semantic_index;
bool export_param = true;
switch (semantic_name) {
break;
}
+ if (outputs[i].vertex_stream[0] != 0 &&
+ outputs[i].vertex_stream[1] != 0 &&
+ outputs[i].vertex_stream[2] != 0 &&
+ outputs[i].vertex_stream[3] != 0)
+ export_param = false;
+
handle_semantic:
/* Select the correct target */
switch(semantic_name) {
}
for (i = 0; i < info->num_outputs; i++) {
- outputs[i].name = info->output_semantic_name[i];
- outputs[i].sid = info->output_semantic_index[i];
+ outputs[i].semantic_name = info->output_semantic_name[i];
+ outputs[i].semantic_index = info->output_semantic_index[i];
- for (j = 0; j < 4; j++)
+ for (j = 0; j < 4; j++) {
outputs[i].values[j] =
LLVMBuildLoad(gallivm->builder,
ctx->soa.outputs[i][j],
"");
+ outputs[i].vertex_stream[j] =
+ (info->output_streams[i] >> (2 * j)) & 3;
+ }
+
}
/* Return the primitive ID from the LLVM function. */
get_primitive_id(bld_base, 0)),
VS_EPILOG_PRIMID_LOC, "");
+ if (ctx->shader->selector->so.num_outputs)
+ si_llvm_emit_streamout(ctx, outputs, i, 0);
si_llvm_export_vs(bld_base, outputs, i);
FREE(outputs);
}
* point in the program by emitting empty inline assembly that is marked as
* having side effects.
*/
+#if 0 /* unused currently */
static void emit_optimization_barrier(struct si_shader_context *ctx)
{
LLVMBuilderRef builder = ctx->gallivm.builder;
LLVMValueRef inlineasm = LLVMConstInlineAsm(ftype, "", "", true, false);
LLVMBuildCall(builder, inlineasm, NULL, 0, "");
}
+#endif
+
+/* Combine these with & instead of |. */
+#define NOOP_WAITCNT 0xf7f
+#define LGKM_CNT 0x07f
+#define VM_CNT 0xf70
-static void emit_waitcnt(struct si_shader_context *ctx)
+static void emit_waitcnt(struct si_shader_context *ctx, unsigned simm16)
{
struct gallivm_state *gallivm = &ctx->gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef args[1] = {
- lp_build_const_int32(gallivm, 0xf70)
+ lp_build_const_int32(gallivm, simm16)
};
lp_build_intrinsic(builder, "llvm.amdgcn.s.waitcnt",
ctx->voidt, args, 1, 0);
struct lp_build_emit_data *emit_data)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
+ LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
+ unsigned flags = LLVMConstIntGetZExtValue(src0);
+ unsigned waitcnt = NOOP_WAITCNT;
+
+ if (flags & TGSI_MEMBAR_THREAD_GROUP)
+ waitcnt &= VM_CNT & LGKM_CNT;
- emit_waitcnt(ctx);
+ if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
+ TGSI_MEMBAR_SHADER_BUFFER |
+ TGSI_MEMBAR_SHADER_IMAGE))
+ waitcnt &= VM_CNT;
+
+ if (flags & TGSI_MEMBAR_SHARED)
+ waitcnt &= LGKM_CNT;
+
+ if (waitcnt != NOOP_WAITCNT)
+ emit_waitcnt(ctx, waitcnt);
}
static LLVMValueRef
}
if (inst->Memory.Qualifier & TGSI_MEMORY_VOLATILE)
- emit_waitcnt(ctx);
+ emit_waitcnt(ctx, VM_CNT);
if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
load_emit_buffer(ctx, emit_data);
}
if (inst->Memory.Qualifier & TGSI_MEMORY_VOLATILE)
- emit_waitcnt(ctx);
+ emit_waitcnt(ctx, VM_CNT);
if (inst->Dst[0].Register.File == TGSI_FILE_BUFFER) {
store_emit_buffer(ctx, emit_data);
struct si_shader *shader = ctx->shader;
struct tgsi_shader_info *info = &shader->selector->info;
struct gallivm_state *gallivm = bld_base->base.gallivm;
+ struct lp_build_if_state if_state;
LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
SI_PARAM_GS2VS_OFFSET);
LLVMValueRef gs_next_vertex;
"");
/* If this thread has already emitted the declared maximum number of
- * vertices, kill it: excessive vertex emissions are not supposed to
- * have any effect, and GS threads have no externally observable
- * effects other than emitting vertices.
+ * vertices, skip the write: excessive vertex emissions are not
+ * supposed to have any effect.
+ *
+ * If the shader has no writes to memory, kill it instead. This skips
+ * further memory loads and may allow LLVM to skip to the end
+ * altogether.
*/
- can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULE, gs_next_vertex,
+ can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULT, gs_next_vertex,
lp_build_const_int32(gallivm,
shader->selector->gs_max_out_vertices), "");
- kill = lp_build_select(&bld_base->base, can_emit,
- lp_build_const_float(gallivm, 1.0f),
- lp_build_const_float(gallivm, -1.0f));
- lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
- ctx->voidt, &kill, 1, 0);
+ bool use_kill = !info->writes_memory;
+ if (use_kill) {
+ kill = lp_build_select(&bld_base->base, can_emit,
+ lp_build_const_float(gallivm, 1.0f),
+ lp_build_const_float(gallivm, -1.0f));
+
+ lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
+ ctx->voidt, &kill, 1, 0);
+ } else {
+ lp_build_if(&if_state, gallivm, can_emit);
+ }
for (i = 0; i < info->num_outputs; i++) {
LLVMValueRef *out_ptr =
ctx->soa.outputs[i];
for (chan = 0; chan < 4; chan++) {
+ if (((info->output_streams[i] >> (2 * chan)) & 3) != stream)
+ continue;
+
LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
LLVMValueRef voffset =
lp_build_const_int32(gallivm, (i * 4 + chan) *
1, 0, 1, 1, 0);
}
}
+
gs_next_vertex = lp_build_add(uint, gs_next_vertex,
lp_build_const_int32(gallivm, 1));
args[1] = LLVMGetParam(ctx->main_fn, SI_PARAM_GS_WAVE_ID);
lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
ctx->voidt, args, 2, 0);
+
+ if (!use_kill)
+ lp_build_endif(&if_state);
}
/* Cut one primitive from the geometry shader */
* always fits into a single wave.
*/
if (ctx->type == PIPE_SHADER_TESS_CTRL) {
- emit_optimization_barrier(ctx);
+ emit_waitcnt(ctx, LGKM_CNT & VM_CNT);
return;
}
}
static void si_shader_dump_stats(struct si_screen *sscreen,
- struct si_shader_config *conf,
- unsigned num_inputs,
- unsigned code_size,
+ struct si_shader *shader,
struct pipe_debug_callback *debug,
unsigned processor,
FILE *file)
{
+ struct si_shader_config *conf = &shader->config;
+ unsigned num_inputs = shader->selector ? shader->selector->info.num_inputs : 0;
+ unsigned code_size = si_get_shader_binary_size(shader);
unsigned lds_increment = sscreen->b.chip_class >= CIK ? 512 : 256;
unsigned lds_per_wave = 0;
unsigned max_simd_waves = 10;
/* Compute LDS usage for PS. */
- if (processor == PIPE_SHADER_FRAGMENT) {
+ switch (processor) {
+ case PIPE_SHADER_FRAGMENT:
/* The minimum usage per wave is (num_inputs * 48). The maximum
* usage is (num_inputs * 48 * 16).
* We can get anything in between and it varies between waves.
*/
lds_per_wave = conf->lds_size * lds_increment +
align(num_inputs * 48, lds_increment);
+ break;
+ case PIPE_SHADER_COMPUTE:
+ if (shader->selector) {
+ unsigned max_workgroup_size =
+ si_get_max_workgroup_size(shader);
+ lds_per_wave = (conf->lds_size * lds_increment) /
+ DIV_ROUND_UP(max_workgroup_size, 64);
+ }
+ break;
}
/* Compute the per-SIMD wave counts. */
if (conf->num_vgprs)
max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
- /* LDS is 64KB per CU (4 SIMDs), divided into 16KB blocks per SIMD
- * that PS can use.
- */
+ /* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
+ * 16KB makes some SIMDs unoccupied). */
if (lds_per_wave)
max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
fprintf(file, "\n");
}
- si_shader_dump_stats(sscreen, &shader->config,
- shader->selector ? shader->selector->info.num_inputs : 0,
- si_get_shader_binary_size(shader), debug, processor,
- file);
+ si_shader_dump_stats(sscreen, shader, debug, processor, file);
}
int si_compile_llvm(struct si_screen *sscreen,
struct si_shader_context ctx;
struct si_shader *shader;
struct gallivm_state *gallivm = &ctx.gallivm;
+ LLVMBuilderRef builder;
struct lp_build_tgsi_context *bld_base = &ctx.soa.bld_base;
struct lp_build_context *uint = &bld_base->uint_bld;
struct si_shader_output_values *outputs;
si_init_shader_ctx(&ctx, sscreen, shader, tm);
ctx.type = PIPE_SHADER_VERTEX;
+ builder = gallivm->builder;
+
create_meta_data(&ctx);
create_function(&ctx);
preload_ring_buffers(&ctx);
args[7] = uint->one; /* SLC */
args[8] = uint->zero; /* TFE */
- /* Fetch vertex data from GSVS ring */
+ /* Fetch the vertex stream ID.*/
+ LLVMValueRef stream_id;
+
+ if (gs_selector->so.num_outputs)
+ stream_id = unpack_param(&ctx, ctx.param_streamout_config, 24, 2);
+ else
+ stream_id = uint->zero;
+
+ /* Fill in output information. */
for (i = 0; i < gsinfo->num_outputs; ++i) {
- unsigned chan;
+ outputs[i].semantic_name = gsinfo->output_semantic_name[i];
+ outputs[i].semantic_index = gsinfo->output_semantic_index[i];
+
+ for (int chan = 0; chan < 4; chan++) {
+ outputs[i].vertex_stream[chan] =
+ (gsinfo->output_streams[i] >> (2 * chan)) & 3;
+ }
+ }
- outputs[i].name = gsinfo->output_semantic_name[i];
- outputs[i].sid = gsinfo->output_semantic_index[i];
+ LLVMBasicBlockRef end_bb;
+ LLVMValueRef switch_inst;
- for (chan = 0; chan < 4; chan++) {
- args[2] = lp_build_const_int32(gallivm,
- (i * 4 + chan) *
- gs_selector->gs_max_out_vertices * 16 * 4);
+ end_bb = LLVMAppendBasicBlockInContext(gallivm->context, ctx.main_fn, "end");
+ switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
+
+ for (int stream = 0; stream < 4; stream++) {
+ LLVMBasicBlockRef bb;
+
+ if (!gsinfo->num_stream_output_components[stream])
+ continue;
- outputs[i].values[chan] =
- LLVMBuildBitCast(gallivm->builder,
+ if (stream > 0 && !gs_selector->so.num_outputs)
+ continue;
+
+ bb = LLVMInsertBasicBlockInContext(gallivm->context, end_bb, "out");
+ LLVMAddCase(switch_inst, lp_build_const_int32(gallivm, stream), bb);
+ LLVMPositionBuilderAtEnd(builder, bb);
+
+ /* Fetch vertex data from GSVS ring */
+ for (i = 0; i < gsinfo->num_outputs; ++i) {
+ for (unsigned chan = 0; chan < 4; chan++) {
+ if (outputs[i].vertex_stream[chan] != stream) {
+ outputs[i].values[chan] = ctx.soa.bld_base.base.undef;
+ continue;
+ }
+
+ args[2] = lp_build_const_int32(
+ gallivm,
+ (i * 4 + chan) * gs_selector->gs_max_out_vertices * 16 * 4);
+
+ outputs[i].values[chan] =
+ LLVMBuildBitCast(gallivm->builder,
lp_build_intrinsic(gallivm->builder,
"llvm.SI.buffer.load.dword.i32.i32",
ctx.i32, args, 9,
LP_FUNC_ATTR_READONLY),
ctx.f32, "");
+ }
+ }
+
+ /* Streamout and exports. */
+ if (gs_selector->so.num_outputs) {
+ si_llvm_emit_streamout(&ctx, outputs,
+ gsinfo->num_outputs,
+ stream);
}
+
+ if (stream == 0)
+ si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
+
+ LLVMBuildBr(builder, end_bb);
}
- si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
+ LLVMPositionBuilderAtEnd(builder, end_bb);
LLVMBuildRetVoid(gallivm->builder);
projects / mesa.git / blobdiff