-
/*
* Copyright 2012 Advanced Micro Devices, Inc.
*
* Christian König <christian.koenig@amd.com>
*/
-#include "gallivm/lp_bld_tgsi_action.h"
#include "gallivm/lp_bld_const.h"
#include "gallivm/lp_bld_gather.h"
#include "gallivm/lp_bld_intr.h"
#include "gallivm/lp_bld_logic.h"
-#include "gallivm/lp_bld_tgsi.h"
#include "gallivm/lp_bld_arit.h"
#include "gallivm/lp_bld_flow.h"
#include "radeon_llvm.h"
#include "radeon_llvm_emit.h"
#include "util/u_memory.h"
-#include "tgsi/tgsi_info.h"
#include "tgsi/tgsi_parse.h"
-#include "tgsi/tgsi_scan.h"
#include "tgsi/tgsi_util.h"
#include "tgsi/tgsi_dump.h"
#include "si_pipe.h"
#include "si_shader.h"
-#include "si_state.h"
#include "sid.h"
-#include <assert.h>
#include <errno.h>
-#include <stdio.h>
+
+struct si_shader_output_values
+{
+ LLVMValueRef values[4];
+ unsigned name;
+ unsigned index;
+ unsigned sid;
+ unsigned usage;
+};
struct si_shader_context
{
struct tgsi_parse_context parse;
struct tgsi_token * tokens;
struct si_pipe_shader *shader;
+ struct si_shader *gs_for_vs;
unsigned type; /* TGSI_PROCESSOR_* specifies the type of shader. */
int param_streamout_config;
int param_streamout_write_index;
LLVMValueRef *resources;
LLVMValueRef *samplers;
LLVMValueRef so_buffers[4];
+ LLVMValueRef gs_next_vertex;
};
static struct si_shader_context * si_shader_context(
#define LOCAL_ADDR_SPACE 3
#define USER_SGPR_ADDR_SPACE 8
+
+#define SENDMSG_GS 2
+#define SENDMSG_GS_DONE 3
+
+#define SENDMSG_GS_OP_NOP (0 << 4)
+#define SENDMSG_GS_OP_CUT (1 << 4)
+#define SENDMSG_GS_OP_EMIT (2 << 4)
+#define SENDMSG_GS_OP_EMIT_CUT (3 << 4)
+
+
/**
* Build an LLVM bytecode indexed load using LLVMBuildGEP + LLVMBuildLoad
*
return result;
}
+static int si_store_shader_io_attribs(struct si_shader *shader,
+ const struct tgsi_full_declaration *d)
+{
+ int i = -1;
+
+ switch (d->Declaration.File) {
+ case TGSI_FILE_INPUT:
+ i = shader->ninput++;
+ assert(i < Elements(shader->input));
+ shader->input[i].name = d->Semantic.Name;
+ shader->input[i].sid = d->Semantic.Index;
+ shader->input[i].index = d->Range.First;
+ shader->input[i].interpolate = d->Interp.Interpolate;
+ shader->input[i].centroid = d->Interp.Centroid;
+ return -1;
+
+ case TGSI_FILE_OUTPUT:
+ i = shader->noutput++;
+ assert(i < Elements(shader->output));
+ shader->output[i].name = d->Semantic.Name;
+ shader->output[i].sid = d->Semantic.Index;
+ shader->output[i].index = d->Range.First;
+ shader->output[i].usage = d->Declaration.UsageMask;
+ break;
+ }
+
+ return i;
+}
+
static void declare_input_vs(
- struct si_shader_context * si_shader_ctx,
+ struct radeon_llvm_context *radeon_bld,
unsigned input_index,
const struct tgsi_full_declaration *decl)
{
- struct lp_build_context * base = &si_shader_ctx->radeon_bld.soa.bld_base.base;
+ struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
+ struct gallivm_state *gallivm = base->gallivm;
+ struct si_shader_context *si_shader_ctx =
+ si_shader_context(&radeon_bld->soa.bld_base);
unsigned divisor = si_shader_ctx->shader->key.vs.instance_divisors[input_index];
unsigned chan;
/* Load the T list */
t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_BUFFER);
- t_offset = lp_build_const_int32(base->gallivm, input_index);
+ t_offset = lp_build_const_int32(gallivm, input_index);
t_list = build_indexed_load(si_shader_ctx, t_list_ptr, t_offset);
/* Build the attribute offset */
- attribute_offset = lp_build_const_int32(base->gallivm, 0);
+ attribute_offset = lp_build_const_int32(gallivm, 0);
if (divisor) {
/* Build index from instance ID, start instance and divisor */
args[0] = t_list;
args[1] = attribute_offset;
args[2] = buffer_index;
- input = build_intrinsic(base->gallivm->builder,
+ input = build_intrinsic(gallivm->builder,
"llvm.SI.vs.load.input", vec4_type, args, 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
/* Break up the vec4 into individual components */
for (chan = 0; chan < 4; chan++) {
- LLVMValueRef llvm_chan = lp_build_const_int32(base->gallivm, chan);
+ LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
/* XXX: Use a helper function for this. There is one in
* tgsi_llvm.c. */
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, chan)] =
- LLVMBuildExtractElement(base->gallivm->builder,
+ LLVMBuildExtractElement(gallivm->builder,
input, llvm_chan, "");
}
}
+static void declare_input_gs(
+ struct radeon_llvm_context *radeon_bld,
+ unsigned input_index,
+ const struct tgsi_full_declaration *decl)
+{
+ struct si_shader_context *si_shader_ctx =
+ si_shader_context(&radeon_bld->soa.bld_base);
+ struct si_shader *shader = &si_shader_ctx->shader->shader;
+
+ si_store_shader_io_attribs(shader, decl);
+
+ if (decl->Semantic.Name != TGSI_SEMANTIC_PRIMID)
+ shader->input[input_index].param_offset = shader->nparam++;
+}
+
+static LLVMValueRef fetch_input_gs(
+ struct lp_build_tgsi_context *bld_base,
+ const struct tgsi_full_src_register *reg,
+ enum tgsi_opcode_type type,
+ unsigned swizzle)
+{
+ struct lp_build_context *base = &bld_base->base;
+ struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
+ struct si_shader *shader = &si_shader_ctx->shader->shader;
+ struct lp_build_context *uint = &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
+ struct gallivm_state *gallivm = base->gallivm;
+ LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
+ LLVMValueRef vtx_offset;
+ LLVMValueRef t_list_ptr;
+ LLVMValueRef t_list;
+ LLVMValueRef args[9];
+ unsigned vtx_offset_param;
+
+ if (swizzle != ~0 &&
+ shader->input[reg->Register.Index].name == TGSI_SEMANTIC_PRIMID) {
+ if (swizzle == 0)
+ return LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_PRIMITIVE_ID);
+ else
+ return uint->zero;
+ }
+
+ if (!reg->Register.Dimension)
+ return NULL;
+
+ if (swizzle == ~0) {
+ LLVMValueRef values[TGSI_NUM_CHANNELS];
+ unsigned chan;
+ for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+ values[chan] = fetch_input_gs(bld_base, reg, type, chan);
+ }
+ return lp_build_gather_values(bld_base->base.gallivm, values,
+ TGSI_NUM_CHANNELS);
+ }
+
+ /* Get the vertex offset parameter */
+ vtx_offset_param = reg->Dimension.Index;
+ if (vtx_offset_param < 2) {
+ vtx_offset_param += SI_PARAM_VTX0_OFFSET;
+ } else {
+ assert(vtx_offset_param < 6);
+ vtx_offset_param += SI_PARAM_VTX2_OFFSET - 2;
+ }
+ vtx_offset = lp_build_mul_imm(uint,
+ LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ vtx_offset_param),
+ 4);
+
+ /* Load the ESGS ring resource descriptor */
+ t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_RW_BUFFERS);
+ t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
+ lp_build_const_int32(gallivm, SI_RING_ESGS));
+
+ args[0] = t_list;
+ args[1] = vtx_offset;
+ args[2] = lp_build_const_int32(gallivm,
+ ((shader->input[reg->Register.Index].param_offset * 4) +
+ swizzle) * 256);
+ args[3] = uint->zero;
+ args[4] = uint->one; /* OFFEN */
+ args[5] = uint->zero; /* IDXEN */
+ args[6] = uint->one; /* GLC */
+ args[7] = uint->zero; /* SLC */
+ args[8] = uint->zero; /* TFE */
+
+ return LLVMBuildBitCast(gallivm->builder,
+ build_intrinsic(gallivm->builder,
+ "llvm.SI.buffer.load.dword.i32.i32",
+ i32, args, 9,
+ LLVMReadOnlyAttribute | LLVMNoUnwindAttribute),
+ tgsi2llvmtype(bld_base, type), "");
+}
+
static void declare_input_fs(
- struct si_shader_context * si_shader_ctx,
+ struct radeon_llvm_context *radeon_bld,
unsigned input_index,
const struct tgsi_full_declaration *decl)
{
+ struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
+ struct si_shader_context *si_shader_ctx =
+ si_shader_context(&radeon_bld->soa.bld_base);
struct si_shader *shader = &si_shader_ctx->shader->shader;
- struct lp_build_context * base =
- &si_shader_ctx->radeon_bld.soa.bld_base.base;
- struct lp_build_context *uint =
- &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
- struct gallivm_state * gallivm = base->gallivm;
+ struct lp_build_context *uint = &radeon_bld->soa.bld_base.uint_bld;
+ struct gallivm_state *gallivm = base->gallivm;
LLVMTypeRef input_type = LLVMFloatTypeInContext(gallivm->context);
- LLVMValueRef main_fn = si_shader_ctx->radeon_bld.main_fn;
+ LLVMValueRef main_fn = radeon_bld->main_fn;
LLVMValueRef interp_param;
const char * intr_name;
* [32:16] ParamOffset
*
*/
- LLVMValueRef params = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_PRIM_MASK);
+ LLVMValueRef params = LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK);
LLVMValueRef attr_number;
unsigned chan;
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
unsigned soa_index =
radeon_llvm_reg_index_soa(input_index, chan);
- si_shader_ctx->radeon_bld.inputs[soa_index] =
+ radeon_bld->inputs[soa_index] =
LLVMGetParam(main_fn, SI_PARAM_POS_X_FLOAT + chan);
if (chan == 3)
/* RCP for fragcoord.w */
- si_shader_ctx->radeon_bld.inputs[soa_index] =
+ radeon_bld->inputs[soa_index] =
LLVMBuildFDiv(gallivm->builder,
lp_build_const_float(gallivm, 1.0f),
- si_shader_ctx->radeon_bld.inputs[soa_index],
+ radeon_bld->inputs[soa_index],
"");
}
return;
lp_build_const_float(gallivm, 0.0f),
"");
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
LLVMBuildSelect(gallivm->builder,
is_face_positive,
lp_build_const_float(gallivm, 1.0f),
lp_build_const_float(gallivm, 0.0f),
"");
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
lp_build_const_float(gallivm, 0.0f);
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
lp_build_const_float(gallivm, 1.0f);
return;
}
- shader->input[input_index].param_offset = shader->ninterp++;
+ shader->input[input_index].param_offset = shader->nparam++;
attr_number = lp_build_const_int32(gallivm,
shader->input[input_index].param_offset);
args[0] = llvm_chan;
args[1] = attr_number;
- front = build_intrinsic(base->gallivm->builder, intr_name,
+ front = build_intrinsic(gallivm->builder, intr_name,
input_type, args, args[3] ? 4 : 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
args[1] = back_attr_number;
- back = build_intrinsic(base->gallivm->builder, intr_name,
+ back = build_intrinsic(gallivm->builder, intr_name,
input_type, args, args[3] ? 4 : 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
- si_shader_ctx->radeon_bld.inputs[soa_index] =
+ radeon_bld->inputs[soa_index] =
LLVMBuildSelect(gallivm->builder,
is_face_positive,
front,
"");
}
- shader->ninterp++;
+ shader->nparam++;
} else if (decl->Semantic.Name == TGSI_SEMANTIC_FOG) {
LLVMValueRef args[4];
args[1] = attr_number;
args[2] = params;
args[3] = interp_param;
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
- build_intrinsic(base->gallivm->builder, intr_name,
- input_type, args, args[3] ? 4 : 3,
- LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
+ build_intrinsic(gallivm->builder, intr_name,
+ input_type, args, args[3] ? 4 : 3,
+ LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
lp_build_const_float(gallivm, 0.0f);
- si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
+ radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
lp_build_const_float(gallivm, 1.0f);
} else {
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
args[1] = attr_number;
args[2] = params;
args[3] = interp_param;
- si_shader_ctx->radeon_bld.inputs[soa_index] =
- build_intrinsic(base->gallivm->builder, intr_name,
+ radeon_bld->inputs[soa_index] =
+ build_intrinsic(gallivm->builder, intr_name,
input_type, args, args[3] ? 4 : 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
}
}
}
-static void declare_input(
- struct radeon_llvm_context * radeon_bld,
- unsigned input_index,
- const struct tgsi_full_declaration *decl)
-{
- struct si_shader_context * si_shader_ctx =
- si_shader_context(&radeon_bld->soa.bld_base);
- if (si_shader_ctx->type == TGSI_PROCESSOR_VERTEX) {
- declare_input_vs(si_shader_ctx, input_index, decl);
- } else if (si_shader_ctx->type == TGSI_PROCESSOR_FRAGMENT) {
- declare_input_fs(si_shader_ctx, input_index, decl);
- } else {
- fprintf(stderr, "Warning: Unsupported shader type,\n");
- }
-}
-
static void declare_system_value(
struct radeon_llvm_context * radeon_bld,
unsigned index,
/* Initialize arguments for the shader export intrinsic */
static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
- struct tgsi_full_declaration *d,
- unsigned index,
+ LLVMValueRef *values,
unsigned target,
LLVMValueRef *args)
{
if (compressed) {
/* Pixel shader needs to pack output values before export */
for (chan = 0; chan < 2; chan++ ) {
- LLVMValueRef *out_ptr =
- si_shader_ctx->radeon_bld.soa.outputs[index];
- args[0] = LLVMBuildLoad(base->gallivm->builder,
- out_ptr[2 * chan], "");
- args[1] = LLVMBuildLoad(base->gallivm->builder,
- out_ptr[2 * chan + 1], "");
+ args[0] = values[2 * chan];
+ args[1] = values[2 * chan + 1];
args[chan + 5] =
build_intrinsic(base->gallivm->builder,
"llvm.SI.packf16",
/* Set COMPR flag */
args[4] = uint->one;
} else {
- for (chan = 0; chan < 4; chan++ ) {
- LLVMValueRef out_ptr =
- si_shader_ctx->radeon_bld.soa.outputs[index][chan];
+ for (chan = 0; chan < 4; chan++ )
/* +5 because the first output value will be
* the 6th argument to the intrinsic. */
- args[chan + 5] = LLVMBuildLoad(base->gallivm->builder,
- out_ptr, "");
- }
+ args[chan + 5] = values[chan];
/* Clear COMPR flag */
args[4] = uint->zero;
* stage. */
}
+/* Load from output pointers and initialize arguments for the shader export intrinsic */
+static void si_llvm_init_export_args_load(struct lp_build_tgsi_context *bld_base,
+ LLVMValueRef *out_ptr,
+ unsigned target,
+ LLVMValueRef *args)
+{
+ struct gallivm_state *gallivm = bld_base->base.gallivm;
+ LLVMValueRef values[4];
+ int i;
+
+ for (i = 0; i < 4; i++)
+ values[i] = LLVMBuildLoad(gallivm->builder, out_ptr[i], "");
+
+ si_llvm_init_export_args(bld_base, values, target, args);
+}
+
static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
- unsigned index)
+ LLVMValueRef *out_ptr)
{
struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
struct gallivm_state *gallivm = bld_base->base.gallivm;
if (si_shader_ctx->shader->key.ps.alpha_func != PIPE_FUNC_NEVER) {
- LLVMValueRef out_ptr = si_shader_ctx->radeon_bld.soa.outputs[index][3];
LLVMValueRef alpha_ref = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
SI_PARAM_ALPHA_REF);
LLVMValueRef alpha_pass =
lp_build_cmp(&bld_base->base,
si_shader_ctx->shader->key.ps.alpha_func,
- LLVMBuildLoad(gallivm->builder, out_ptr, ""),
+ LLVMBuildLoad(gallivm->builder, out_ptr[3], ""),
alpha_ref);
LLVMValueRef arg =
lp_build_select(&bld_base->base,
}
}
-static void si_alpha_to_one(struct lp_build_tgsi_context *bld_base,
- unsigned index)
-{
- struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
-
- /* set alpha to one */
- LLVMBuildStore(bld_base->base.gallivm->builder,
- bld_base->base.one,
- si_shader_ctx->radeon_bld.soa.outputs[index][3]);
-}
-
static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context * bld_base,
- LLVMValueRef (*pos)[9], unsigned index)
+ LLVMValueRef (*pos)[9], LLVMValueRef *out_elts)
{
struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
struct si_pipe_shader *shader = si_shader_ctx->shader;
unsigned reg_index;
unsigned chan;
unsigned const_chan;
- LLVMValueRef out_elts[4];
LLVMValueRef base_elt;
LLVMValueRef ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_CONST);
LLVMValueRef constbuf_index = lp_build_const_int32(base->gallivm, NUM_PIPE_CONST_BUFFERS);
LLVMValueRef const_resource = build_indexed_load(si_shader_ctx, ptr, constbuf_index);
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- LLVMValueRef out_ptr = si_shader_ctx->radeon_bld.soa.outputs[index][chan];
- out_elts[chan] = LLVMBuildLoad(base->gallivm->builder, out_ptr, "");
- }
-
for (reg_index = 0; reg_index < 2; reg_index ++) {
LLVMValueRef *args = pos[2 + reg_index];
LLVMConstInt(i32, tfe, 0)
};
+ /* The instruction offset field has 12 bits */
+ assert(offen || inst_offset < (1 << 12));
+
/* The intrinsic is overloaded, we need to add a type suffix for overloading to work. */
unsigned func = CLAMP(num_channels, 1, 3) - 1;
const char *types[] = {"i32", "v2i32", "v4i32"};
/* On SI, the vertex shader is responsible for writing streamout data
* to buffers. */
-static void si_llvm_emit_streamout(struct si_shader_context *shader)
+static void si_llvm_emit_streamout(struct si_shader_context *shader,
+ struct si_shader_output_values *outputs,
+ unsigned noutput)
{
struct pipe_stream_output_info *so = &shader->shader->selector->so;
struct gallivm_state *gallivm = &shader->radeon_bld.gallivm;
so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
}
- LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS] = shader->radeon_bld.soa.outputs;
-
/* Write streamout data. */
for (i = 0; i < so->num_outputs; i++) {
unsigned buf_idx = so->output[i].output_buffer;
/* Load the output as int. */
for (j = 0; j < num_comps; j++) {
- out[j] = LLVMBuildLoad(builder, outputs[reg][start+j], "");
- out[j] = LLVMBuildBitCast(builder, out[j], i32, "");
+ unsigned outidx = 0;
+
+ while (outidx < noutput && outputs[outidx].index != reg)
+ outidx++;
+
+ if (outidx < noutput)
+ out[j] = LLVMBuildBitCast(builder,
+ outputs[outidx].values[start+j],
+ i32, "");
+ else
+ out[j] = NULL;
}
+ if (!out[0])
+ continue;
+
/* Pack the output. */
LLVMValueRef vdata = NULL;
}
-static void si_llvm_emit_epilogue(struct lp_build_tgsi_context * bld_base)
+/* Generate export instructions for hardware VS shader stage */
+static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
+ struct si_shader_output_values *outputs,
+ unsigned noutput)
{
struct si_shader_context * si_shader_ctx = si_shader_context(bld_base);
struct si_shader * shader = &si_shader_ctx->shader->shader;
struct lp_build_context * base = &bld_base->base;
struct lp_build_context * uint =
&si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
- struct tgsi_parse_context *parse = &si_shader_ctx->parse;
LLVMValueRef args[9];
- LLVMValueRef last_args[9] = { 0 };
LLVMValueRef pos_args[4][9] = { { 0 } };
- unsigned semantic_name;
+ LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL;
+ unsigned semantic_name, semantic_index, semantic_usage;
+ unsigned target;
unsigned param_count = 0;
- int depth_index = -1, stencil_index = -1, psize_index = -1, edgeflag_index = -1;
- int layer_index = -1;
+ unsigned pos_idx;
+ int i;
+
+ if (outputs && si_shader_ctx->shader->selector->so.num_outputs) {
+ si_llvm_emit_streamout(si_shader_ctx, outputs, noutput);
+ }
+
+ for (i = 0; i < noutput; i++) {
+ semantic_name = outputs[i].name;
+ semantic_index = outputs[i].sid;
+ semantic_usage = outputs[i].usage;
+
+handle_semantic:
+ /* Select the correct target */
+ switch(semantic_name) {
+ case TGSI_SEMANTIC_PSIZE:
+ shader->vs_out_misc_write = true;
+ shader->vs_out_point_size = true;
+ psize_value = outputs[i].values[0];
+ continue;
+ case TGSI_SEMANTIC_EDGEFLAG:
+ shader->vs_out_misc_write = true;
+ shader->vs_out_edgeflag = true;
+ edgeflag_value = outputs[i].values[0];
+ continue;
+ case TGSI_SEMANTIC_LAYER:
+ shader->vs_out_misc_write = true;
+ shader->vs_out_layer = true;
+ layer_value = outputs[i].values[0];
+ continue;
+ case TGSI_SEMANTIC_POSITION:
+ target = V_008DFC_SQ_EXP_POS;
+ break;
+ case TGSI_SEMANTIC_COLOR:
+ case TGSI_SEMANTIC_BCOLOR:
+ target = V_008DFC_SQ_EXP_PARAM + param_count;
+ shader->output[i].param_offset = param_count;
+ param_count++;
+ break;
+ case TGSI_SEMANTIC_CLIPDIST:
+ if (!(si_shader_ctx->shader->key.vs.ucps_enabled &
+ (1 << semantic_index)))
+ continue;
+ shader->clip_dist_write |=
+ semantic_usage << (semantic_index << 2);
+ target = V_008DFC_SQ_EXP_POS + 2 + semantic_index;
+ break;
+ case TGSI_SEMANTIC_CLIPVERTEX:
+ si_llvm_emit_clipvertex(bld_base, pos_args, outputs[i].values);
+ continue;
+ case TGSI_SEMANTIC_PRIMID:
+ case TGSI_SEMANTIC_FOG:
+ case TGSI_SEMANTIC_GENERIC:
+ target = V_008DFC_SQ_EXP_PARAM + param_count;
+ shader->output[i].param_offset = param_count;
+ param_count++;
+ break;
+ default:
+ target = 0;
+ fprintf(stderr,
+ "Warning: SI unhandled vs output type:%d\n",
+ semantic_name);
+ }
+
+ si_llvm_init_export_args(bld_base, outputs[i].values, target, args);
+
+ if (target >= V_008DFC_SQ_EXP_POS &&
+ target <= (V_008DFC_SQ_EXP_POS + 3)) {
+ memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
+ args, sizeof(args));
+ } else {
+ lp_build_intrinsic(base->gallivm->builder,
+ "llvm.SI.export",
+ LLVMVoidTypeInContext(base->gallivm->context),
+ args, 9);
+ }
+
+ if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
+ semantic_name = TGSI_SEMANTIC_GENERIC;
+ goto handle_semantic;
+ }
+ }
+
+ /* We need to add the position output manually if it's missing. */
+ if (!pos_args[0][0]) {
+ pos_args[0][0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
+ pos_args[0][1] = uint->zero; /* EXEC mask */
+ pos_args[0][2] = uint->zero; /* last export? */
+ pos_args[0][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS);
+ pos_args[0][4] = uint->zero; /* COMPR flag */
+ pos_args[0][5] = base->zero; /* X */
+ pos_args[0][6] = base->zero; /* Y */
+ pos_args[0][7] = base->zero; /* Z */
+ pos_args[0][8] = base->one; /* W */
+ }
+
+ /* Write the misc vector (point size, edgeflag, layer, viewport). */
+ if (shader->vs_out_misc_write) {
+ pos_args[1][0] = lp_build_const_int32(base->gallivm, /* writemask */
+ shader->vs_out_point_size |
+ (shader->vs_out_edgeflag << 1) |
+ (shader->vs_out_layer << 2));
+ pos_args[1][1] = uint->zero; /* EXEC mask */
+ pos_args[1][2] = uint->zero; /* last export? */
+ pos_args[1][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + 1);
+ pos_args[1][4] = uint->zero; /* COMPR flag */
+ pos_args[1][5] = base->zero; /* X */
+ pos_args[1][6] = base->zero; /* Y */
+ pos_args[1][7] = base->zero; /* Z */
+ pos_args[1][8] = base->zero; /* W */
+
+ if (shader->vs_out_point_size)
+ pos_args[1][5] = psize_value;
+
+ if (shader->vs_out_edgeflag) {
+ /* The output is a float, but the hw expects an integer
+ * with the first bit containing the edge flag. */
+ edgeflag_value = LLVMBuildFPToUI(base->gallivm->builder,
+ edgeflag_value,
+ bld_base->uint_bld.elem_type, "");
+ edgeflag_value = lp_build_min(&bld_base->int_bld,
+ edgeflag_value,
+ bld_base->int_bld.one);
+
+ /* The LLVM intrinsic expects a float. */
+ pos_args[1][6] = LLVMBuildBitCast(base->gallivm->builder,
+ edgeflag_value,
+ base->elem_type, "");
+ }
+
+ if (shader->vs_out_layer)
+ pos_args[1][7] = layer_value;
+ }
+
+ for (i = 0; i < 4; i++)
+ if (pos_args[i][0])
+ shader->nr_pos_exports++;
+
+ pos_idx = 0;
+ for (i = 0; i < 4; i++) {
+ if (!pos_args[i][0])
+ continue;
+
+ /* Specify the target we are exporting */
+ pos_args[i][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + pos_idx++);
+
+ if (pos_idx == shader->nr_pos_exports)
+ /* Specify that this is the last export */
+ pos_args[i][2] = uint->one;
+
+ lp_build_intrinsic(base->gallivm->builder,
+ "llvm.SI.export",
+ LLVMVoidTypeInContext(base->gallivm->context),
+ pos_args[i], 9);
+ }
+}
+
+static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context * bld_base)
+{
+ struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
+ struct gallivm_state *gallivm = bld_base->base.gallivm;
+ struct si_shader *es = &si_shader_ctx->shader->shader;
+ struct si_shader *gs = si_shader_ctx->gs_for_vs;
+ struct tgsi_parse_context *parse = &si_shader_ctx->parse;
+ LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
+ LLVMValueRef soffset = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_ES2GS_OFFSET);
+ LLVMValueRef t_list_ptr;
+ LLVMValueRef t_list;
+ unsigned chan;
int i;
- if (si_shader_ctx->shader->selector->so.num_outputs) {
- si_llvm_emit_streamout(si_shader_ctx);
+ while (!tgsi_parse_end_of_tokens(parse)) {
+ struct tgsi_full_declaration *d =
+ &parse->FullToken.FullDeclaration;
+
+ tgsi_parse_token(parse);
+
+ if (parse->FullToken.Token.Type != TGSI_TOKEN_TYPE_DECLARATION)
+ continue;
+
+ si_store_shader_io_attribs(es, d);
+ }
+
+ /* Load the ESGS ring resource descriptor */
+ t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_RW_BUFFERS);
+ t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
+ lp_build_const_int32(gallivm, SI_RING_ESGS));
+
+ for (i = 0; i < es->noutput; i++) {
+ LLVMValueRef *out_ptr =
+ si_shader_ctx->radeon_bld.soa.outputs[es->output[i].index];
+ int j;
+
+ for (j = 0; j < gs->ninput; j++) {
+ if (gs->input[j].name == es->output[i].name &&
+ gs->input[j].sid == es->output[i].sid)
+ break;
+ }
+ if (j == gs->ninput)
+ continue;
+
+ for (chan = 0; chan < 4; chan++) {
+ LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
+ out_val = LLVMBuildBitCast(gallivm->builder, out_val, i32, "");
+
+ build_tbuffer_store(si_shader_ctx, t_list, out_val, 1,
+ LLVMGetUndef(i32), soffset,
+ (4 * gs->input[j].param_offset + chan) * 4,
+ V_008F0C_BUF_DATA_FORMAT_32,
+ V_008F0C_BUF_NUM_FORMAT_UINT,
+ 0, 0, 1, 1, 0);
+ }
}
+}
+
+static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
+{
+ struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
+ struct gallivm_state *gallivm = bld_base->base.gallivm;
+ LLVMValueRef args[2];
+
+ args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_NOP | SENDMSG_GS_DONE);
+ args[1] = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
+ build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
+ LLVMVoidTypeInContext(gallivm->context), args, 2,
+ LLVMNoUnwindAttribute);
+}
+
+static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context * bld_base)
+{
+ struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
+ struct gallivm_state *gallivm = bld_base->base.gallivm;
+ struct si_pipe_shader *shader = si_shader_ctx->shader;
+ struct tgsi_parse_context *parse = &si_shader_ctx->parse;
+ struct si_shader_output_values *outputs = NULL;
+ unsigned noutput = 0;
+ int i;
+
+ while (!tgsi_parse_end_of_tokens(parse)) {
+ struct tgsi_full_declaration *d =
+ &parse->FullToken.FullDeclaration;
+ unsigned index;
+
+ tgsi_parse_token(parse);
+
+ if (parse->FullToken.Token.Type != TGSI_TOKEN_TYPE_DECLARATION)
+ continue;
+
+ i = si_store_shader_io_attribs(&shader->shader, d);
+ if (i < 0)
+ continue;
+
+ outputs = REALLOC(outputs, noutput * sizeof(outputs[0]),
+ (noutput + 1) * sizeof(outputs[0]));
+ for (index = d->Range.First; index <= d->Range.Last; index++) {
+ outputs[noutput].index = index;
+ outputs[noutput].name = d->Semantic.Name;
+ outputs[noutput].sid = d->Semantic.Index;
+ outputs[noutput].usage = d->Declaration.UsageMask;
+
+ for (i = 0; i < 4; i++)
+ outputs[noutput].values[i] =
+ LLVMBuildLoad(gallivm->builder,
+ si_shader_ctx->radeon_bld.soa.outputs[index][i],
+ "");
+ }
+ noutput++;
+ }
+
+ si_llvm_export_vs(bld_base, outputs, noutput);
+ FREE(outputs);
+}
+
+static void si_llvm_emit_fs_epilogue(struct lp_build_tgsi_context * bld_base)
+{
+ struct si_shader_context * si_shader_ctx = si_shader_context(bld_base);
+ struct si_shader * shader = &si_shader_ctx->shader->shader;
+ struct lp_build_context * base = &bld_base->base;
+ struct lp_build_context * uint =
+ &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
+ struct tgsi_parse_context *parse = &si_shader_ctx->parse;
+ LLVMValueRef args[9];
+ LLVMValueRef last_args[9] = { 0 };
+ unsigned semantic_name;
+ int depth_index = -1, stencil_index = -1;
+ int i;
while (!tgsi_parse_end_of_tokens(parse)) {
struct tgsi_full_declaration *d =
if (parse->FullToken.Token.Type != TGSI_TOKEN_TYPE_DECLARATION)
continue;
- switch (d->Declaration.File) {
- case TGSI_FILE_INPUT:
- i = shader->ninput++;
- assert(i < Elements(shader->input));
- shader->input[i].name = d->Semantic.Name;
- shader->input[i].sid = d->Semantic.Index;
- shader->input[i].interpolate = d->Interp.Interpolate;
- shader->input[i].centroid = d->Interp.Centroid;
- continue;
-
- case TGSI_FILE_OUTPUT:
- i = shader->noutput++;
- assert(i < Elements(shader->output));
- shader->output[i].name = d->Semantic.Name;
- shader->output[i].sid = d->Semantic.Index;
- shader->output[i].interpolate = d->Interp.Interpolate;
- break;
-
- default:
+ i = si_store_shader_io_attribs(shader, d);
+ if (i < 0)
continue;
- }
semantic_name = d->Semantic.Name;
-handle_semantic:
for (index = d->Range.First; index <= d->Range.Last; index++) {
/* Select the correct target */
switch(semantic_name) {
- case TGSI_SEMANTIC_PSIZE:
- shader->vs_out_misc_write = true;
- shader->vs_out_point_size = true;
- psize_index = index;
- continue;
- case TGSI_SEMANTIC_EDGEFLAG:
- shader->vs_out_misc_write = true;
- shader->vs_out_edgeflag = true;
- edgeflag_index = index;
- continue;
- case TGSI_SEMANTIC_LAYER:
- shader->vs_out_misc_write = true;
- shader->vs_out_layer = true;
- layer_index = index;
- continue;
case TGSI_SEMANTIC_POSITION:
- if (si_shader_ctx->type == TGSI_PROCESSOR_VERTEX) {
- target = V_008DFC_SQ_EXP_POS;
- break;
- } else {
- depth_index = index;
- continue;
- }
+ depth_index = index;
+ continue;
case TGSI_SEMANTIC_STENCIL:
stencil_index = index;
continue;
case TGSI_SEMANTIC_COLOR:
- if (si_shader_ctx->type == TGSI_PROCESSOR_VERTEX) {
- case TGSI_SEMANTIC_BCOLOR:
- target = V_008DFC_SQ_EXP_PARAM + param_count;
- shader->output[i].param_offset = param_count;
- param_count++;
- } else {
- target = V_008DFC_SQ_EXP_MRT + shader->output[i].sid;
- if (si_shader_ctx->shader->key.ps.alpha_to_one) {
- si_alpha_to_one(bld_base, index);
- }
- if (shader->output[i].sid == 0 &&
- si_shader_ctx->shader->key.ps.alpha_func != PIPE_FUNC_ALWAYS)
- si_alpha_test(bld_base, index);
- }
- break;
- case TGSI_SEMANTIC_CLIPDIST:
- if (!(si_shader_ctx->shader->key.vs.ucps_enabled &
- (1 << d->Semantic.Index)))
- continue;
- shader->clip_dist_write |=
- d->Declaration.UsageMask << (d->Semantic.Index << 2);
- target = V_008DFC_SQ_EXP_POS + 2 + d->Semantic.Index;
- break;
- case TGSI_SEMANTIC_CLIPVERTEX:
- si_llvm_emit_clipvertex(bld_base, pos_args, index);
- continue;
- case TGSI_SEMANTIC_FOG:
- case TGSI_SEMANTIC_GENERIC:
- target = V_008DFC_SQ_EXP_PARAM + param_count;
- shader->output[i].param_offset = param_count;
- param_count++;
+ target = V_008DFC_SQ_EXP_MRT + d->Semantic.Index;
+ if (si_shader_ctx->shader->key.ps.alpha_to_one)
+ LLVMBuildStore(bld_base->base.gallivm->builder,
+ bld_base->base.one,
+ si_shader_ctx->radeon_bld.soa.outputs[index][3]);
+
+ if (d->Semantic.Index == 0 &&
+ si_shader_ctx->shader->key.ps.alpha_func != PIPE_FUNC_ALWAYS)
+ si_alpha_test(bld_base,
+ si_shader_ctx->radeon_bld.soa.outputs[index]);
break;
default:
target = 0;
fprintf(stderr,
- "Warning: SI unhandled output type:%d\n",
+ "Warning: SI unhandled fs output type:%d\n",
semantic_name);
}
- si_llvm_init_export_args(bld_base, d, index, target, args);
+ si_llvm_init_export_args_load(bld_base,
+ si_shader_ctx->radeon_bld.soa.outputs[index],
+ target, args);
- if (si_shader_ctx->type == TGSI_PROCESSOR_VERTEX &&
- target >= V_008DFC_SQ_EXP_POS &&
- target <= (V_008DFC_SQ_EXP_POS + 3)) {
- memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
- args, sizeof(args));
- } else if (si_shader_ctx->type == TGSI_PROCESSOR_FRAGMENT &&
- semantic_name == TGSI_SEMANTIC_COLOR) {
+ if (semantic_name == TGSI_SEMANTIC_COLOR) {
/* If there is an export instruction waiting to be emitted, do so now. */
if (last_args[0]) {
lp_build_intrinsic(base->gallivm->builder,
if (shader->fs_write_all && shader->output[i].sid == 0 &&
si_shader_ctx->shader->key.ps.nr_cbufs > 1) {
for (int c = 1; c < si_shader_ctx->shader->key.ps.nr_cbufs; c++) {
- si_llvm_init_export_args(bld_base, d, index,
- V_008DFC_SQ_EXP_MRT + c, args);
+ si_llvm_init_export_args_load(bld_base,
+ si_shader_ctx->radeon_bld.soa.outputs[index],
+ V_008DFC_SQ_EXP_MRT + c, args);
lp_build_intrinsic(base->gallivm->builder,
"llvm.SI.export",
LLVMVoidTypeInContext(base->gallivm->context),
args, 9);
}
}
-
- if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
- semantic_name = TGSI_SEMANTIC_GENERIC;
- goto handle_semantic;
- }
}
if (depth_index >= 0 || stencil_index >= 0) {
memcpy(last_args, args, sizeof(args));
}
- if (si_shader_ctx->type == TGSI_PROCESSOR_VERTEX) {
- unsigned pos_idx = 0;
-
- /* We need to add the position output manually if it's missing. */
- if (!pos_args[0][0]) {
- pos_args[0][0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
- pos_args[0][1] = uint->zero; /* EXEC mask */
- pos_args[0][2] = uint->zero; /* last export? */
- pos_args[0][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS);
- pos_args[0][4] = uint->zero; /* COMPR flag */
- pos_args[0][5] = base->zero; /* X */
- pos_args[0][6] = base->zero; /* Y */
- pos_args[0][7] = base->zero; /* Z */
- pos_args[0][8] = base->one; /* W */
- }
-
- /* Write the misc vector (point size, edgeflag, layer, viewport). */
- if (shader->vs_out_misc_write) {
- pos_args[1][0] = lp_build_const_int32(base->gallivm, /* writemask */
- shader->vs_out_point_size |
- (shader->vs_out_edgeflag << 1) |
- (shader->vs_out_layer << 2));
- pos_args[1][1] = uint->zero; /* EXEC mask */
- pos_args[1][2] = uint->zero; /* last export? */
- pos_args[1][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + 1);
- pos_args[1][4] = uint->zero; /* COMPR flag */
- pos_args[1][5] = base->zero; /* X */
- pos_args[1][6] = base->zero; /* Y */
- pos_args[1][7] = base->zero; /* Z */
- pos_args[1][8] = base->zero; /* W */
-
- if (shader->vs_out_point_size) {
- pos_args[1][5] = LLVMBuildLoad(base->gallivm->builder,
- si_shader_ctx->radeon_bld.soa.outputs[psize_index][0], "");
- }
-
- if (shader->vs_out_edgeflag) {
- LLVMValueRef output = LLVMBuildLoad(base->gallivm->builder,
- si_shader_ctx->radeon_bld.soa.outputs[edgeflag_index][0], "");
-
- /* The output is a float, but the hw expects an integer
- * with the first bit containing the edge flag. */
- output = LLVMBuildFPToUI(base->gallivm->builder, output,
- bld_base->uint_bld.elem_type, "");
-
- output = lp_build_min(&bld_base->int_bld, output, bld_base->int_bld.one);
-
- /* The LLVM intrinsic expects a float. */
- pos_args[1][6] = LLVMBuildBitCast(base->gallivm->builder, output,
- base->elem_type, "");
- }
-
- if (shader->vs_out_layer) {
- pos_args[1][7] = LLVMBuildLoad(base->gallivm->builder,
- si_shader_ctx->radeon_bld.soa.outputs[layer_index][0], "");
- }
- }
-
- for (i = 0; i < 4; i++)
- if (pos_args[i][0])
- shader->nr_pos_exports++;
-
- for (i = 0; i < 4; i++) {
- if (!pos_args[i][0])
- continue;
-
- /* Specify the target we are exporting */
- pos_args[i][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + pos_idx++);
-
- if (pos_idx == shader->nr_pos_exports)
- /* Specify that this is the last export */
- pos_args[i][2] = uint->one;
-
- lp_build_intrinsic(base->gallivm->builder,
- "llvm.SI.export",
- LLVMVoidTypeInContext(base->gallivm->context),
- pos_args[i], 9);
- }
- } else {
- if (!last_args[0]) {
- /* Specify which components to enable */
- last_args[0] = lp_build_const_int32(base->gallivm, 0x0);
+ if (!last_args[0]) {
+ /* Specify which components to enable */
+ last_args[0] = lp_build_const_int32(base->gallivm, 0x0);
- /* Specify the target we are exporting */
- last_args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_MRT);
+ /* Specify the target we are exporting */
+ last_args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_MRT);
- /* Set COMPR flag to zero to export data as 32-bit */
- last_args[4] = uint->zero;
+ /* Set COMPR flag to zero to export data as 32-bit */
+ last_args[4] = uint->zero;
- /* dummy bits */
- last_args[5]= uint->zero;
- last_args[6]= uint->zero;
- last_args[7]= uint->zero;
- last_args[8]= uint->zero;
+ /* dummy bits */
+ last_args[5]= uint->zero;
+ last_args[6]= uint->zero;
+ last_args[7]= uint->zero;
+ last_args[8]= uint->zero;
- si_shader_ctx->shader->spi_shader_col_format |=
- V_028714_SPI_SHADER_32_ABGR;
- si_shader_ctx->shader->cb_shader_mask |= S_02823C_OUTPUT0_ENABLE(0xf);
- }
+ si_shader_ctx->shader->spi_shader_col_format |=
+ V_028714_SPI_SHADER_32_ABGR;
+ si_shader_ctx->shader->cb_shader_mask |= S_02823C_OUTPUT0_ENABLE(0xf);
+ }
- /* Specify whether the EXEC mask represents the valid mask */
- last_args[1] = uint->one;
+ /* Specify whether the EXEC mask represents the valid mask */
+ last_args[1] = uint->one;
- /* Specify that this is the last export */
- last_args[2] = lp_build_const_int32(base->gallivm, 1);
+ /* Specify that this is the last export */
+ last_args[2] = lp_build_const_int32(base->gallivm, 1);
- lp_build_intrinsic(base->gallivm->builder,
- "llvm.SI.export",
- LLVMVoidTypeInContext(base->gallivm->context),
- last_args, 9);
- }
+ lp_build_intrinsic(base->gallivm->builder,
+ "llvm.SI.export",
+ LLVMVoidTypeInContext(base->gallivm->context),
+ last_args, 9);
}
static const struct lp_build_tgsi_action txf_action;
#endif /* HAVE_LLVM >= 0x0304 */
+/* Emit one vertex from the geometry shader */
+static void si_llvm_emit_vertex(
+ const struct lp_build_tgsi_action *action,
+ struct lp_build_tgsi_context *bld_base,
+ struct lp_build_emit_data *emit_data)
+{
+ struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
+ struct lp_build_context *uint = &bld_base->uint_bld;
+ struct si_shader *shader = &si_shader_ctx->shader->shader;
+ struct gallivm_state *gallivm = bld_base->base.gallivm;
+ LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
+ LLVMValueRef soffset = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_GS2VS_OFFSET);
+ LLVMValueRef gs_next_vertex;
+ LLVMValueRef can_emit, kill;
+ LLVMValueRef t_list_ptr;
+ LLVMValueRef t_list;
+ LLVMValueRef args[2];
+ unsigned chan;
+ int i;
+
+ /* Load the GSVS ring resource descriptor */
+ t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_RW_BUFFERS);
+ t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
+ lp_build_const_int32(gallivm, SI_RING_GSVS));
+
+ if (shader->noutput == 0) {
+ struct tgsi_parse_context *parse = &si_shader_ctx->parse;
+
+ while (!tgsi_parse_end_of_tokens(parse)) {
+ tgsi_parse_token(parse);
+
+ if (parse->FullToken.Token.Type == TGSI_TOKEN_TYPE_DECLARATION) {
+ struct tgsi_full_declaration *d = &parse->FullToken.FullDeclaration;
+
+ if (d->Declaration.File == TGSI_FILE_OUTPUT)
+ si_store_shader_io_attribs(shader, d);
+ }
+ }
+ }
+
+ /* Write vertex attribute values to GSVS ring */
+ gs_next_vertex = LLVMBuildLoad(gallivm->builder, si_shader_ctx->gs_next_vertex, "");
+
+ /* If this thread has already emitted the declared maximum number of
+ * vertices, kill it: excessive vertex emissions are not supposed to
+ * have any effect, and GS threads have no externally observable
+ * effects other than emitting vertices.
+ */
+ can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULE, gs_next_vertex,
+ lp_build_const_int32(gallivm,
+ shader->gs_max_out_vertices), "");
+ kill = lp_build_select(&bld_base->base, can_emit,
+ lp_build_const_float(gallivm, 1.0f),
+ lp_build_const_float(gallivm, -1.0f));
+ build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
+ LLVMVoidTypeInContext(gallivm->context), &kill, 1, 0);
+
+ for (i = 0; i < shader->noutput; i++) {
+ LLVMValueRef *out_ptr =
+ si_shader_ctx->radeon_bld.soa.outputs[shader->output[i].index];
+
+ for (chan = 0; chan < 4; chan++) {
+ LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
+ LLVMValueRef voffset =
+ lp_build_const_int32(gallivm, (i * 4 + chan) *
+ shader->gs_max_out_vertices);
+
+ voffset = lp_build_add(uint, voffset, gs_next_vertex);
+ voffset = lp_build_mul_imm(uint, voffset, 4);
+
+ out_val = LLVMBuildBitCast(gallivm->builder, out_val, i32, "");
+
+ build_tbuffer_store(si_shader_ctx, t_list, out_val, 1,
+ voffset, soffset, 0,
+ V_008F0C_BUF_DATA_FORMAT_32,
+ V_008F0C_BUF_NUM_FORMAT_UINT,
+ 1, 0, 1, 1, 0);
+ }
+ }
+ gs_next_vertex = lp_build_add(uint, gs_next_vertex,
+ lp_build_const_int32(gallivm, 1));
+ LLVMBuildStore(gallivm->builder, gs_next_vertex, si_shader_ctx->gs_next_vertex);
+
+ /* Signal vertex emission */
+ args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_EMIT | SENDMSG_GS);
+ args[1] = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
+ build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
+ LLVMVoidTypeInContext(gallivm->context), args, 2,
+ LLVMNoUnwindAttribute);
+}
+
+/* Cut one primitive from the geometry shader */
+static void si_llvm_emit_primitive(
+ const struct lp_build_tgsi_action *action,
+ struct lp_build_tgsi_context *bld_base,
+ struct lp_build_emit_data *emit_data)
+{
+ struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
+ struct gallivm_state *gallivm = bld_base->base.gallivm;
+ LLVMValueRef args[2];
+
+ /* Signal primitive cut */
+ args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_CUT | SENDMSG_GS);
+ args[1] = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
+ build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
+ LLVMVoidTypeInContext(gallivm->context), args, 2,
+ LLVMNoUnwindAttribute);
+}
+
static const struct lp_build_tgsi_action tex_action = {
.fetch_args = tex_fetch_args,
.emit = build_tex_intrinsic,
{
struct lp_build_tgsi_context *bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
struct gallivm_state *gallivm = bld_base->base.gallivm;
- LLVMTypeRef params[21], f32, i8, i32, v2i32, v3i32;
+ struct si_pipe_shader *shader = si_shader_ctx->shader;
+ LLVMTypeRef params[SI_NUM_PARAMS], f32, i8, i32, v2i32, v3i32;
unsigned i, last_sgpr, num_params;
i8 = LLVMInt8TypeInContext(gallivm->context);
params[SI_PARAM_CONST] = LLVMPointerType(
LLVMArrayType(LLVMVectorType(i8, 16), NUM_CONST_BUFFERS), CONST_ADDR_SPACE);
+ params[SI_PARAM_RW_BUFFERS] = params[SI_PARAM_CONST];
+
/* We assume at most 16 textures per program at the moment.
* This need probably need to be changed to support bindless textures */
params[SI_PARAM_SAMPLER] = LLVMPointerType(
switch (si_shader_ctx->type) {
case TGSI_PROCESSOR_VERTEX:
params[SI_PARAM_VERTEX_BUFFER] = params[SI_PARAM_CONST];
- params[SI_PARAM_SO_BUFFER] = params[SI_PARAM_CONST];
params[SI_PARAM_START_INSTANCE] = i32;
num_params = SI_PARAM_START_INSTANCE+1;
+ if (shader->key.vs.as_es) {
+ params[SI_PARAM_ES2GS_OFFSET] = i32;
+ num_params++;
+ } else {
+ /* The locations of the other parameters are assigned dynamically. */
- /* The locations of the other parameters are assigned dynamically. */
-
- /* Streamout SGPRs. */
- if (si_shader_ctx->shader->selector->so.num_outputs) {
- params[si_shader_ctx->param_streamout_config = num_params++] = i32;
- params[si_shader_ctx->param_streamout_write_index = num_params++] = i32;
- }
- /* A streamout buffer offset is loaded if the stride is non-zero. */
- for (i = 0; i < 4; i++) {
- if (!si_shader_ctx->shader->selector->so.stride[i])
- continue;
+ /* Streamout SGPRs. */
+ if (shader->selector->so.num_outputs) {
+ params[si_shader_ctx->param_streamout_config = num_params++] = i32;
+ params[si_shader_ctx->param_streamout_write_index = num_params++] = i32;
+ }
+ /* A streamout buffer offset is loaded if the stride is non-zero. */
+ for (i = 0; i < 4; i++) {
+ if (!shader->selector->so.stride[i])
+ continue;
- params[si_shader_ctx->param_streamout_offset[i] = num_params++] = i32;
+ params[si_shader_ctx->param_streamout_offset[i] = num_params++] = i32;
+ }
}
last_sgpr = num_params-1;
params[si_shader_ctx->param_instance_id = num_params++] = i32;
break;
+ case TGSI_PROCESSOR_GEOMETRY:
+ params[SI_PARAM_GS2VS_OFFSET] = i32;
+ params[SI_PARAM_GS_WAVE_ID] = i32;
+ last_sgpr = SI_PARAM_GS_WAVE_ID;
+
+ /* VGPRs */
+ params[SI_PARAM_VTX0_OFFSET] = i32;
+ params[SI_PARAM_VTX1_OFFSET] = i32;
+ params[SI_PARAM_PRIMITIVE_ID] = i32;
+ params[SI_PARAM_VTX2_OFFSET] = i32;
+ params[SI_PARAM_VTX3_OFFSET] = i32;
+ params[SI_PARAM_VTX4_OFFSET] = i32;
+ params[SI_PARAM_VTX5_OFFSET] = i32;
+ params[SI_PARAM_GS_INSTANCE_ID] = i32;
+ num_params = SI_PARAM_GS_INSTANCE_ID+1;
+ break;
+
case TGSI_PROCESSOR_FRAGMENT:
params[SI_PARAM_ALPHA_REF] = f32;
params[SI_PARAM_PRIM_MASK] = i32;
}
#if HAVE_LLVM >= 0x0304
- if (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
- bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0)
+ if (bld_base->info &&
+ (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
+ bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0))
si_shader_ctx->ddxy_lds =
LLVMAddGlobalInAddressSpace(gallivm->module,
LLVMArrayType(i32, 64),
struct gallivm_state * gallivm = bld_base->base.gallivm;
unsigned i;
- if (!si_shader_ctx->shader->selector->so.num_outputs)
+ if (si_shader_ctx->type != TGSI_PROCESSOR_VERTEX ||
+ si_shader_ctx->shader->key.vs.as_es ||
+ !si_shader_ctx->shader->selector->so.num_outputs)
return;
LLVMValueRef buf_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
- SI_PARAM_SO_BUFFER);
+ SI_PARAM_RW_BUFFERS);
/* Load the resources, we rely on the code sinking to do the rest */
for (i = 0; i < 4; ++i) {
if (si_shader_ctx->shader->selector->so.stride[i]) {
- LLVMValueRef offset = lp_build_const_int32(gallivm, i);
+ LLVMValueRef offset = lp_build_const_int32(gallivm,
+ SI_RW_SO + i);
si_shader_ctx->so_buffers[i] = build_indexed_load(si_shader_ctx, buf_ptr, offset);
}
}
}
-int si_compile_llvm(struct si_context *rctx, struct si_pipe_shader *shader,
+int si_compile_llvm(struct si_context *sctx, struct si_pipe_shader *shader,
LLVMModuleRef mod)
{
unsigned i;
uint32_t *ptr;
- struct radeon_llvm_binary binary;
- bool dump = r600_can_dump_shader(&rctx->screen->b,
+ struct radeon_shader_binary binary;
+ bool dump = r600_can_dump_shader(&sctx->screen->b,
shader->selector ? shader->selector->tokens : NULL);
memset(&binary, 0, sizeof(binary));
radeon_llvm_compile(mod, &binary,
- si_get_llvm_processor_name(rctx->screen->b.family), dump);
+ r600_get_llvm_processor_name(sctx->screen->b.family), dump);
if (dump && ! binary.disassembled) {
fprintf(stderr, "SI CODE:\n");
for (i = 0; i < binary.code_size; i+=4 ) {
/* copy new shader */
r600_resource_reference(&shader->bo, NULL);
- shader->bo = si_resource_create_custom(rctx->b.b.screen, PIPE_USAGE_IMMUTABLE,
+ shader->bo = si_resource_create_custom(sctx->b.b.screen, PIPE_USAGE_IMMUTABLE,
binary.code_size);
if (shader->bo == NULL) {
return -ENOMEM;
}
- ptr = (uint32_t*)rctx->b.ws->buffer_map(shader->bo->cs_buf, rctx->b.rings.gfx.cs, PIPE_TRANSFER_WRITE);
- if (0 /*R600_BIG_ENDIAN*/) {
+ ptr = (uint32_t*)sctx->b.ws->buffer_map(shader->bo->cs_buf, sctx->b.rings.gfx.cs, PIPE_TRANSFER_WRITE);
+ if (SI_BIG_ENDIAN) {
for (i = 0; i < binary.code_size / 4; ++i) {
- ptr[i] = util_bswap32(*(uint32_t*)(binary.code + i*4));
+ ptr[i] = util_cpu_to_le32((*(uint32_t*)(binary.code + i*4)));
}
} else {
memcpy(ptr, binary.code, binary.code_size);
}
- rctx->b.ws->buffer_unmap(shader->bo->cs_buf);
+ sctx->b.ws->buffer_unmap(shader->bo->cs_buf);
free(binary.code);
free(binary.config);
return 0;
}
+/* Generate code for the hardware VS shader stage to go with a geometry shader */
+static int si_generate_gs_copy_shader(struct si_context *sctx,
+ struct si_shader_context *si_shader_ctx,
+ bool dump)
+{
+ struct gallivm_state *gallivm = &si_shader_ctx->radeon_bld.gallivm;
+ struct lp_build_tgsi_context *bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
+ struct lp_build_context *base = &bld_base->base;
+ struct lp_build_context *uint = &bld_base->uint_bld;
+ struct si_shader *shader = &si_shader_ctx->shader->shader;
+ struct si_shader *gs = &si_shader_ctx->shader->selector->current->shader;
+ struct si_shader_output_values *outputs;
+ LLVMValueRef t_list_ptr, t_list;
+ LLVMValueRef args[9];
+ int i, r;
+
+ outputs = MALLOC(gs->noutput * sizeof(outputs[0]));
+
+ si_shader_ctx->type = TGSI_PROCESSOR_VERTEX;
+ si_shader_ctx->gs_for_vs = gs;
+
+ radeon_llvm_context_init(&si_shader_ctx->radeon_bld);
+
+ create_meta_data(si_shader_ctx);
+ create_function(si_shader_ctx);
+ preload_streamout_buffers(si_shader_ctx);
+
+ /* Load the GSVS ring resource descriptor */
+ t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ SI_PARAM_RW_BUFFERS);
+ t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
+ lp_build_const_int32(gallivm, SI_RING_GSVS));
+
+ args[0] = t_list;
+ args[1] = lp_build_mul_imm(uint,
+ LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
+ si_shader_ctx->param_vertex_id),
+ 4);
+ args[3] = uint->zero;
+ args[4] = uint->one; /* OFFEN */
+ args[5] = uint->zero; /* IDXEN */
+ args[6] = uint->one; /* GLC */
+ args[7] = uint->one; /* SLC */
+ args[8] = uint->zero; /* TFE */
+
+ /* Fetch vertex data from GSVS ring */
+ for (i = 0; i < gs->noutput; ++i) {
+ struct si_shader_output *out = gs->output + i;
+ unsigned chan;
+
+ shader->output[i] = *out;
+
+ outputs[i].name = out->name;
+ outputs[i].index = out->index;
+ outputs[i].sid = out->sid;
+ outputs[i].usage = out->usage;
+
+ for (chan = 0; chan < 4; chan++) {
+ args[2] = lp_build_const_int32(gallivm,
+ (i * 4 + chan) *
+ gs->gs_max_out_vertices * 16 * 4);
+
+ outputs[i].values[chan] =
+ LLVMBuildBitCast(gallivm->builder,
+ build_intrinsic(gallivm->builder,
+ "llvm.SI.buffer.load.dword.i32.i32",
+ LLVMInt32TypeInContext(gallivm->context),
+ args, 9,
+ LLVMReadOnlyAttribute | LLVMNoUnwindAttribute),
+ base->elem_type, "");
+ }
+ }
+ shader->noutput = gs->noutput;
+
+ si_llvm_export_vs(bld_base, outputs, gs->noutput);
+
+ radeon_llvm_finalize_module(&si_shader_ctx->radeon_bld);
+
+ if (dump)
+ fprintf(stderr, "Copy Vertex Shader for Geometry Shader:\n\n");
+
+ r = si_compile_llvm(sctx, si_shader_ctx->shader,
+ bld_base->base.gallivm->module);
+
+ radeon_llvm_dispose(&si_shader_ctx->radeon_bld);
+
+ FREE(outputs);
+ return r;
+}
+
int si_pipe_shader_create(
struct pipe_context *ctx,
struct si_pipe_shader *shader)
{
- struct si_context *rctx = (struct si_context*)ctx;
+ struct si_context *sctx = (struct si_context*)ctx;
struct si_pipe_shader_selector *sel = shader->selector;
struct si_shader_context si_shader_ctx;
struct tgsi_shader_info shader_info;
struct lp_build_tgsi_context * bld_base;
LLVMModuleRef mod;
int r = 0;
- bool dump = r600_can_dump_shader(&rctx->screen->b, shader->selector->tokens);
+ bool dump = r600_can_dump_shader(&sctx->screen->b, sel->tokens);
+
+ /* Dump TGSI code before doing TGSI->LLVM conversion in case the
+ * conversion fails. */
+ if (dump) {
+ tgsi_dump(sel->tokens, 0);
+ si_dump_streamout(&sel->so);
+ }
assert(shader->shader.noutput == 0);
- assert(shader->shader.ninterp == 0);
+ assert(shader->shader.nparam == 0);
assert(shader->shader.ninput == 0);
memset(&si_shader_ctx, 0, sizeof(si_shader_ctx));
shader->shader.uses_instanceid = shader_info.uses_instanceid;
bld_base->info = &shader_info;
bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
- bld_base->emit_epilogue = si_llvm_emit_epilogue;
bld_base->op_actions[TGSI_OPCODE_TEX] = tex_action;
bld_base->op_actions[TGSI_OPCODE_TXB] = txb_action;
bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
#endif
- si_shader_ctx.radeon_bld.load_input = declare_input;
+ bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
+ bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
+
si_shader_ctx.radeon_bld.load_system_value = declare_system_value;
si_shader_ctx.tokens = sel->tokens;
tgsi_parse_init(&si_shader_ctx.parse, si_shader_ctx.tokens);
si_shader_ctx.shader = shader;
si_shader_ctx.type = si_shader_ctx.parse.FullHeader.Processor.Processor;
+ switch (si_shader_ctx.type) {
+ case TGSI_PROCESSOR_VERTEX:
+ si_shader_ctx.radeon_bld.load_input = declare_input_vs;
+ if (shader->key.vs.as_es) {
+ si_shader_ctx.gs_for_vs = &sctx->gs_shader->current->shader;
+ bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
+ } else {
+ bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
+ }
+ break;
+ case TGSI_PROCESSOR_GEOMETRY: {
+ int i;
+
+ si_shader_ctx.radeon_bld.load_input = declare_input_gs;
+ bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
+ bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
+
+ for (i = 0; i < shader_info.num_properties; i++) {
+ switch (shader_info.properties[i].name) {
+ case TGSI_PROPERTY_GS_INPUT_PRIM:
+ shader->shader.gs_input_prim = shader_info.properties[i].data[0];
+ break;
+ case TGSI_PROPERTY_GS_OUTPUT_PRIM:
+ shader->shader.gs_output_prim = shader_info.properties[i].data[0];
+ break;
+ case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
+ shader->shader.gs_max_out_vertices = shader_info.properties[i].data[0];
+ break;
+ }
+ }
+ break;
+ }
+ case TGSI_PROCESSOR_FRAGMENT:
+ si_shader_ctx.radeon_bld.load_input = declare_input_fs;
+ bld_base->emit_epilogue = si_llvm_emit_fs_epilogue;
+ break;
+ default:
+ assert(!"Unsupported shader type");
+ return -1;
+ }
+
create_meta_data(&si_shader_ctx);
create_function(&si_shader_ctx);
preload_constants(&si_shader_ctx);
preload_samplers(&si_shader_ctx);
preload_streamout_buffers(&si_shader_ctx);
- /* Dump TGSI code before doing TGSI->LLVM conversion in case the
- * conversion fails. */
- if (dump) {
- tgsi_dump(sel->tokens, 0);
- si_dump_streamout(&sel->so);
+ if (si_shader_ctx.type == TGSI_PROCESSOR_GEOMETRY) {
+ si_shader_ctx.gs_next_vertex =
+ lp_build_alloca(bld_base->base.gallivm,
+ bld_base->uint_bld.elem_type, "");
}
if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
- for (int i = 0; i < NUM_CONST_BUFFERS; i++)
- FREE(si_shader_ctx.constants[i]);
- FREE(si_shader_ctx.resources);
- FREE(si_shader_ctx.samplers);
- return -EINVAL;
+ goto out;
}
radeon_llvm_finalize_module(&si_shader_ctx.radeon_bld);
mod = bld_base->base.gallivm->module;
- r = si_compile_llvm(rctx, shader, mod);
+ r = si_compile_llvm(sctx, shader, mod);
+ if (r) {
+ fprintf(stderr, "LLVM failed to compile shader\n");
+ goto out;
+ }
radeon_llvm_dispose(&si_shader_ctx.radeon_bld);
+
+ if (si_shader_ctx.type == TGSI_PROCESSOR_GEOMETRY) {
+ shader->gs_copy_shader = CALLOC_STRUCT(si_pipe_shader);
+ shader->gs_copy_shader->selector = shader->selector;
+ shader->gs_copy_shader->key = shader->key;
+ si_shader_ctx.shader = shader->gs_copy_shader;
+ if ((r = si_generate_gs_copy_shader(sctx, &si_shader_ctx, dump))) {
+ free(shader->gs_copy_shader);
+ shader->gs_copy_shader = NULL;
+ goto out;
+ }
+ }
+
tgsi_parse_free(&si_shader_ctx.parse);
+out:
for (int i = 0; i < NUM_CONST_BUFFERS; i++)
FREE(si_shader_ctx.constants[i]);
FREE(si_shader_ctx.resources);
projects / mesa.git / blobdiff