*/
#include "ac_nir_to_llvm.h"
+#include "ac_llvm_build.h"
#include "ac_llvm_util.h"
#include "ac_binary.h"
#include "sid.h"
#define RADEON_LLVM_MAX_INPUTS (VARYING_SLOT_VAR31 + 1)
#define RADEON_LLVM_MAX_OUTPUTS (VARYING_SLOT_VAR31 + 1)
-#define SENDMSG_GS 2
-#define SENDMSG_GS_DONE 3
-
-#define SENDMSG_GS_OP_NOP (0 << 4)
-#define SENDMSG_GS_OP_CUT (1 << 4)
-#define SENDMSG_GS_OP_EMIT (2 << 4)
-#define SENDMSG_GS_OP_EMIT_CUT (3 << 4)
-
enum desc_type {
DESC_IMAGE,
DESC_FMASK,
LLVMValueRef linear_sample, linear_center, linear_centroid;
LLVMValueRef front_face;
LLVMValueRef ancillary;
+ LLVMValueRef sample_coverage;
LLVMValueRef frag_pos[4];
LLVMBasicBlockRef continue_block;
LLVMValueRef f32one;
LLVMValueRef v4f32empty;
- unsigned range_md_kind;
unsigned uniform_md_kind;
LLVMValueRef empty_md;
gl_shader_stage stage;
int num_locals;
LLVMValueRef *locals;
bool has_ddxy;
- unsigned num_clips;
- unsigned num_culls;
+ uint8_t num_input_clips;
+ uint8_t num_input_culls;
+ uint8_t num_output_clips;
+ uint8_t num_output_culls;
bool has_ds_bpermute;
return 0;
if (slot == VARYING_SLOT_PSIZ)
return 1;
- if (slot == VARYING_SLOT_CLIP_DIST0)
+ if (slot == VARYING_SLOT_CLIP_DIST0 ||
+ slot == VARYING_SLOT_CULL_DIST0)
return 2;
- if (slot == VARYING_SLOT_CLIP_DIST1)
+ if (slot == VARYING_SLOT_CLIP_DIST1 ||
+ slot == VARYING_SLOT_CULL_DIST1)
return 3;
if (slot >= VARYING_SLOT_VAR0 && slot <= VARYING_SLOT_VAR31)
return 4 + (slot - VARYING_SLOT_VAR0);
for (unsigned i = 0; i < sgpr_params; ++i) {
if (array_params_mask & (1 << i)) {
LLVMValueRef P = LLVMGetParam(main_function, i);
- ac_add_function_attr(main_function, i + 1, AC_FUNC_ATTR_BYVAL);
+ ac_add_function_attr(ctx, main_function, i + 1, AC_FUNC_ATTR_BYVAL);
ac_add_attr_dereferenceable(P, UINT64_MAX);
}
else {
- ac_add_function_attr(main_function, i + 1, AC_FUNC_ATTR_INREG);
+ ac_add_function_attr(ctx, main_function, i + 1, AC_FUNC_ATTR_INREG);
}
}
arg_types[arg_idx++] = ctx->f32; /* pos w float */
arg_types[arg_idx++] = ctx->i32; /* front face */
arg_types[arg_idx++] = ctx->i32; /* ancillary */
- arg_types[arg_idx++] = ctx->f32; /* sample coverage */
+ arg_types[arg_idx++] = ctx->i32; /* sample coverage */
arg_types[arg_idx++] = ctx->i32; /* fixed pt */
break;
default:
ctx->frag_pos[3] = LLVMGetParam(ctx->main_function, arg_idx++);
ctx->front_face = LLVMGetParam(ctx->main_function, arg_idx++);
ctx->ancillary = LLVMGetParam(ctx->main_function, arg_idx++);
+ ctx->sample_coverage = LLVMGetParam(ctx->main_function, arg_idx++);
break;
default:
unreachable("Shader stage not implemented");
args[3] = ctx->f32one;
ctx->v4f32empty = LLVMConstVector(args, 4);
- ctx->range_md_kind = LLVMGetMDKindIDInContext(ctx->context,
- "range", 5);
ctx->uniform_md_kind =
LLVMGetMDKindIDInContext(ctx->context, "amdgpu.uniform", 14);
ctx->empty_md = LLVMMDNodeInContext(ctx->context, NULL, 0);
static LLVMValueRef emit_ifind_msb(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
- LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.flbit.i32",
- ctx->i32, &src0, 1,
- AC_FUNC_ATTR_READNONE);
-
- /* The HW returns the last bit index from MSB, but NIR wants
- * the index from LSB. Invert it by doing "31 - msb". */
- msb = LLVMBuildSub(ctx->builder, LLVMConstInt(ctx->i32, 31, false),
- msb, "");
-
- LLVMValueRef all_ones = LLVMConstInt(ctx->i32, -1, true);
- LLVMValueRef cond = LLVMBuildOr(ctx->builder,
- LLVMBuildICmp(ctx->builder, LLVMIntEQ,
- src0, ctx->i32zero, ""),
- LLVMBuildICmp(ctx->builder, LLVMIntEQ,
- src0, all_ones, ""), "");
-
- return LLVMBuildSelect(ctx->builder, cond, all_ones, msb, "");
+ return ac_emit_imsb(&ctx->ac, src0, ctx->i32);
}
static LLVMValueRef emit_ufind_msb(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
- LLVMValueRef args[2] = {
- src0,
- ctx->i32one,
- };
- LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctlz.i32",
- ctx->i32, args, ARRAY_SIZE(args),
- AC_FUNC_ATTR_READNONE);
-
- /* The HW returns the last bit index from MSB, but NIR wants
- * the index from LSB. Invert it by doing "31 - msb". */
- msb = LLVMBuildSub(ctx->builder, LLVMConstInt(ctx->i32, 31, false),
- msb, "");
-
- return LLVMBuildSelect(ctx->builder,
- LLVMBuildICmp(ctx->builder, LLVMIntEQ, src0,
- ctx->i32zero, ""),
- LLVMConstInt(ctx->i32, -1, true), msb, "");
+ return ac_emit_umsb(&ctx->ac, src0, ctx->i32);
}
static LLVMValueRef emit_minmax_int(struct nir_to_llvm_context *ctx,
}
static LLVMValueRef emit_bitfield_extract(struct nir_to_llvm_context *ctx,
- const char *intrin,
+ const char *intrin, unsigned attr_mask,
LLVMValueRef srcs[3])
{
LLVMValueRef result;
LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2], LLVMConstInt(ctx->i32, 32, false), "");
- result = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->i32, srcs, 3, AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->i32, srcs, 3,
+ AC_FUNC_ATTR_READNONE | attr_mask);
result = LLVMBuildSelect(ctx->builder, icond, srcs[0], result, "");
return result;
return result;
}
-/**
- * Set range metadata on an instruction. This can only be used on load and
- * call instructions. If you know an instruction can only produce the values
- * 0, 1, 2, you would do set_range_metadata(value, 0, 3);
- * \p lo is the minimum value inclusive.
- * \p hi is the maximum value exclusive.
- */
-static void set_range_metadata(struct nir_to_llvm_context *ctx,
- LLVMValueRef value, unsigned lo, unsigned hi)
-{
- LLVMValueRef range_md, md_args[2];
- LLVMTypeRef type = LLVMTypeOf(value);
- LLVMContextRef context = LLVMGetTypeContext(type);
-
- md_args[0] = LLVMConstInt(type, lo, false);
- md_args[1] = LLVMConstInt(type, hi, false);
- range_md = LLVMMDNodeInContext(context, md_args, 2);
- LLVMSetMetadata(value, ctx->range_md_kind, range_md);
-}
-
-static LLVMValueRef get_thread_id(struct nir_to_llvm_context *ctx)
-{
- LLVMValueRef tid;
- LLVMValueRef tid_args[2];
- tid_args[0] = LLVMConstInt(ctx->i32, 0xffffffff, false);
- tid_args[1] = ctx->i32zero;
- tid_args[1] = ac_emit_llvm_intrinsic(&ctx->ac,
- "llvm.amdgcn.mbcnt.lo", ctx->i32,
- tid_args, 2, AC_FUNC_ATTR_READNONE);
-
- tid = ac_emit_llvm_intrinsic(&ctx->ac,
- "llvm.amdgcn.mbcnt.hi", ctx->i32,
- tid_args, 2, AC_FUNC_ATTR_READNONE);
- set_range_metadata(ctx, tid, 0, 64);
- return tid;
-}
-
-/*
- * SI implements derivatives using the local data store (LDS)
- * All writes to the LDS happen in all executing threads at
- * the same time. TID is the Thread ID for the current
- * thread and is a value between 0 and 63, representing
- * the thread's position in the wavefront.
- *
- * For the pixel shader threads are grouped into quads of four pixels.
- * The TIDs of the pixels of a quad are:
- *
- * +------+------+
- * |4n + 0|4n + 1|
- * +------+------+
- * |4n + 2|4n + 3|
- * +------+------+
- *
- * So, masking the TID with 0xfffffffc yields the TID of the top left pixel
- * of the quad, masking with 0xfffffffd yields the TID of the top pixel of
- * the current pixel's column, and masking with 0xfffffffe yields the TID
- * of the left pixel of the current pixel's row.
- *
- * Adding 1 yields the TID of the pixel to the right of the left pixel, and
- * adding 2 yields the TID of the pixel below the top pixel.
- */
-/* masks for thread ID. */
-#define TID_MASK_TOP_LEFT 0xfffffffc
-#define TID_MASK_TOP 0xfffffffd
-#define TID_MASK_LEFT 0xfffffffe
static LLVMValueRef emit_ddxy(struct nir_to_llvm_context *ctx,
nir_op op,
LLVMValueRef src0)
{
- LLVMValueRef tl, trbl, result;
- LLVMValueRef tl_tid, trbl_tid;
- LLVMValueRef args[2];
- LLVMValueRef thread_id;
unsigned mask;
int idx;
+ LLVMValueRef result;
ctx->has_ddxy = true;
if (!ctx->lds && !ctx->has_ds_bpermute)
LLVMArrayType(ctx->i32, 64),
"ddxy_lds", LOCAL_ADDR_SPACE);
- thread_id = get_thread_id(ctx);
if (op == nir_op_fddx_fine || op == nir_op_fddx)
- mask = TID_MASK_LEFT;
+ mask = AC_TID_MASK_LEFT;
else if (op == nir_op_fddy_fine || op == nir_op_fddy)
- mask = TID_MASK_TOP;
+ mask = AC_TID_MASK_TOP;
else
- mask = TID_MASK_TOP_LEFT;
+ mask = AC_TID_MASK_TOP_LEFT;
- tl_tid = LLVMBuildAnd(ctx->builder, thread_id,
- LLVMConstInt(ctx->i32, mask, false), "");
/* for DDX we want to next X pixel, DDY next Y pixel. */
if (op == nir_op_fddx_fine ||
op == nir_op_fddx_coarse ||
else
idx = 2;
- trbl_tid = LLVMBuildAdd(ctx->builder, tl_tid,
- LLVMConstInt(ctx->i32, idx, false), "");
-
- if (ctx->has_ds_bpermute) {
- args[0] = LLVMBuildMul(ctx->builder, tl_tid,
- LLVMConstInt(ctx->i32, 4, false), "");
- args[1] = src0;
- tl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
- ctx->i32, args, 2,
- AC_FUNC_ATTR_READNONE);
-
- args[0] = LLVMBuildMul(ctx->builder, trbl_tid,
- LLVMConstInt(ctx->i32, 4, false), "");
- trbl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
- ctx->i32, args, 2,
- AC_FUNC_ATTR_READNONE);
- } else {
- LLVMValueRef store_ptr, load_ptr0, load_ptr1;
-
- store_ptr = ac_build_gep0(&ctx->ac, ctx->lds, thread_id);
- load_ptr0 = ac_build_gep0(&ctx->ac, ctx->lds, tl_tid);
- load_ptr1 = ac_build_gep0(&ctx->ac, ctx->lds, trbl_tid);
-
- LLVMBuildStore(ctx->builder, src0, store_ptr);
- tl = LLVMBuildLoad(ctx->builder, load_ptr0, "");
- trbl = LLVMBuildLoad(ctx->builder, load_ptr1, "");
- }
- tl = LLVMBuildBitCast(ctx->builder, tl, ctx->f32, "");
- trbl = LLVMBuildBitCast(ctx->builder, trbl, ctx->f32, "");
- result = LLVMBuildFSub(ctx->builder, trbl, tl, "");
+ result = ac_emit_ddxy(&ctx->ac, ctx->has_ds_bpermute,
+ mask, idx, ctx->lds,
+ src0);
return result;
}
to_float_type(ctx, def_type), src[0], src[1], src[2]);
break;
case nir_op_ibitfield_extract:
- result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.i32", src);
+ result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.i32",
+ AC_FUNC_ATTR_LEGACY, src);
break;
case nir_op_ubitfield_extract:
- result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.u32", src);
+ result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.u32",
+ AC_FUNC_ATTR_LEGACY, src);
break;
case nir_op_bitfield_insert:
result = emit_bitfield_insert(ctx, src[0], src[1], src[2], src[3]);
case nir_op_d2f:
result = LLVMBuildFPTrunc(ctx->builder, src[0], to_float_type(ctx, def_type), "");
break;
+ case nir_op_u2u32:
+ case nir_op_u2u64:
+ case nir_op_u2i32:
+ case nir_op_u2i64:
+ if (get_elem_bits(ctx, LLVMTypeOf(src[0])) < get_elem_bits(ctx, def_type))
+ result = LLVMBuildZExt(ctx->builder, src[0], def_type, "");
+ else
+ result = LLVMBuildTrunc(ctx->builder, src[0], def_type, "");
+ break;
+ case nir_op_i2u32:
+ case nir_op_i2u64:
+ case nir_op_i2i32:
+ case nir_op_i2i64:
+ if (get_elem_bits(ctx, LLVMTypeOf(src[0])) < get_elem_bits(ctx, def_type))
+ result = LLVMBuildSExt(ctx->builder, src[0], def_type, "");
+ else
+ result = LLVMBuildTrunc(ctx->builder, src[0], def_type, "");
+ break;
case nir_op_bcsel:
result = emit_bcsel(ctx, src[0], src[1], src[2]);
break;
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* tfe */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* lwe */
size = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32", ctx->v4i32,
- txq_args, txq_arg_count,
- AC_FUNC_ATTR_READNONE);
+ txq_args, txq_arg_count,
+ AC_FUNC_ATTR_READNONE |
+ AC_FUNC_ATTR_LEGACY);
for (c = 0; c < 2; c++) {
half_texel[c] = LLVMBuildExtractElement(ctx->builder, size,
tinfo->args[0] = coord;
return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
- AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
+ AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND |
+ AC_FUNC_ATTR_LEGACY);
}
}
}
return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
- AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
+ AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND |
+ AC_FUNC_ATTR_LEGACY);
}
offset, "")
};
results[i] = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.load.const", ctx->f32,
- params, 2, AC_FUNC_ATTR_READNONE);
+ params, 2,
+ AC_FUNC_ATTR_READNONE |
+ AC_FUNC_ATTR_LEGACY);
}
unsigned param, vtx_offset_param;
LLVMValueRef value[4], result;
unsigned vertex_index;
+ unsigned cull_offset = 0;
radv_get_deref_offset(ctx, &instr->variables[0]->deref,
false, &vertex_index,
&const_index, &indir_index);
vtx_offset = LLVMBuildMul(ctx->builder, ctx->gs_vtx_offset[vtx_offset_param],
LLVMConstInt(ctx->i32, 4, false), "");
+ param = shader_io_get_unique_index(instr->variables[0]->var->data.location);
+ if (instr->variables[0]->var->data.location == VARYING_SLOT_CULL_DIST0)
+ cull_offset += ctx->num_input_clips;
for (unsigned i = 0; i < instr->num_components; i++) {
- param = shader_io_get_unique_index(instr->variables[0]->var->data.location);
+
args[0] = ctx->esgs_ring;
args[1] = vtx_offset;
- args[2] = LLVMConstInt(ctx->i32, (param * 4 + i + const_index) * 256, false);
+ args[2] = LLVMConstInt(ctx->i32, (param * 4 + i + const_index + cull_offset) * 256, false);
args[3] = ctx->i32zero;
args[4] = ctx->i32one; /* OFFEN */
args[5] = ctx->i32zero; /* IDXEN */
args[8] = ctx->i32zero; /* TFE */
value[i] = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.buffer.load.dword.i32.i32",
- ctx->i32, args, 9, AC_FUNC_ATTR_READONLY);
+ ctx->i32, args, 9,
+ AC_FUNC_ATTR_READONLY |
+ AC_FUNC_ATTR_LEGACY);
}
result = ac_build_gather_values(&ctx->ac, value, instr->num_components);
LLVMValueRef ptr = get_shared_memory_ptr(ctx, idx, ctx->i32);
LLVMValueRef derived_ptr;
+ if (indir_index)
+ indir_index = LLVMBuildMul(ctx->builder, indir_index, LLVMConstInt(ctx->i32, 4, false), "");
+
for (unsigned chan = 0; chan < ve; chan++) {
LLVMValueRef index = LLVMConstInt(ctx->i32, chan, false);
if (indir_index)
break;
case nir_var_shared: {
LLVMValueRef ptr = get_shared_memory_ptr(ctx, idx, ctx->i32);
+
+ if (indir_index)
+ indir_index = LLVMBuildMul(ctx->builder, indir_index, LLVMConstInt(ctx->i32, 4, false), "");
+
for (unsigned chan = 0; chan < 8; chan++) {
if (!(writemask & (1 << chan)))
continue;
if (indir_index)
index = LLVMBuildAdd(ctx->builder, index, indir_index, "");
+
+ value = llvm_extract_elem(ctx, src, chan);
derived_ptr = LLVMBuildGEP(ctx->builder, ptr, &index, 1, "");
LLVMBuildStore(ctx->builder,
- to_integer(ctx, src), derived_ptr);
+ to_integer(ctx, value), derived_ptr);
}
break;
}
return 0;
}
+
+static void get_image_intr_name(const char *base_name,
+ LLVMTypeRef data_type,
+ LLVMTypeRef coords_type,
+ LLVMTypeRef rsrc_type,
+ char *out_name, unsigned out_len)
+{
+ char coords_type_name[8];
+
+ ac_build_type_name_for_intr(coords_type, coords_type_name,
+ sizeof(coords_type_name));
+
+ if (HAVE_LLVM <= 0x0309) {
+ snprintf(out_name, out_len, "%s.%s", base_name, coords_type_name);
+ } else {
+ char data_type_name[8];
+ char rsrc_type_name[8];
+
+ ac_build_type_name_for_intr(data_type, data_type_name,
+ sizeof(data_type_name));
+ ac_build_type_name_for_intr(rsrc_type, rsrc_type_name,
+ sizeof(rsrc_type_name));
+ snprintf(out_name, out_len, "%s.%s.%s.%s", base_name,
+ data_type_name, coords_type_name, rsrc_type_name);
+ }
+}
+
+/* Adjust the sample index according to FMASK.
+ *
+ * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
+ * which is the identity mapping. Each nibble says which physical sample
+ * should be fetched to get that sample.
+ *
+ * For example, 0x11111100 means there are only 2 samples stored and
+ * the second sample covers 3/4 of the pixel. When reading samples 0
+ * and 1, return physical sample 0 (determined by the first two 0s
+ * in FMASK), otherwise return physical sample 1.
+ *
+ * The sample index should be adjusted as follows:
+ * sample_index = (fmask >> (sample_index * 4)) & 0xF;
+ */
+static LLVMValueRef adjust_sample_index_using_fmask(struct nir_to_llvm_context *ctx,
+ LLVMValueRef coord_x, LLVMValueRef coord_y,
+ LLVMValueRef coord_z,
+ LLVMValueRef sample_index,
+ LLVMValueRef fmask_desc_ptr)
+{
+ LLVMValueRef fmask_load_address[4], params[7];
+ LLVMValueRef glc = LLVMConstInt(ctx->i1, 0, false);
+ LLVMValueRef slc = LLVMConstInt(ctx->i1, 0, false);
+ LLVMValueRef da = coord_z ? ctx->i32one : ctx->i32zero;
+ LLVMValueRef res;
+ char intrinsic_name[64];
+
+ fmask_load_address[0] = coord_x;
+ fmask_load_address[1] = coord_y;
+ if (coord_z) {
+ fmask_load_address[2] = coord_z;
+ fmask_load_address[3] = LLVMGetUndef(ctx->i32);
+ }
+
+ params[0] = ac_build_gather_values(&ctx->ac, fmask_load_address, coord_z ? 4 : 2);
+ params[1] = fmask_desc_ptr;
+ params[2] = LLVMConstInt(ctx->i32, 15, false); /* dmask */
+ LLVMValueRef lwe = LLVMConstInt(ctx->i1, 0, false);
+ params[3] = glc;
+ params[4] = slc;
+ params[5] = lwe;
+ params[6] = da;
+
+ get_image_intr_name("llvm.amdgcn.image.load",
+ ctx->v4f32, /* vdata */
+ LLVMTypeOf(params[0]), /* coords */
+ LLVMTypeOf(params[1]), /* rsrc */
+ intrinsic_name, sizeof(intrinsic_name));
+
+ res = ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->v4f32,
+ params, 7, AC_FUNC_ATTR_READONLY);
+
+ res = to_integer(ctx, res);
+ LLVMValueRef four = LLVMConstInt(ctx->i32, 4, false);
+ LLVMValueRef F = LLVMConstInt(ctx->i32, 0xf, false);
+
+ LLVMValueRef fmask = LLVMBuildExtractElement(ctx->builder,
+ res,
+ ctx->i32zero, "");
+
+ LLVMValueRef sample_index4 =
+ LLVMBuildMul(ctx->builder, sample_index, four, "");
+ LLVMValueRef shifted_fmask =
+ LLVMBuildLShr(ctx->builder, fmask, sample_index4, "");
+ LLVMValueRef final_sample =
+ LLVMBuildAnd(ctx->builder, shifted_fmask, F, "");
+
+ /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
+ * resource descriptor is 0 (invalid),
+ */
+ LLVMValueRef fmask_desc =
+ LLVMBuildBitCast(ctx->builder, params[1],
+ ctx->v8i32, "");
+
+ LLVMValueRef fmask_word1 =
+ LLVMBuildExtractElement(ctx->builder, fmask_desc,
+ ctx->i32one, "");
+
+ LLVMValueRef word1_is_nonzero =
+ LLVMBuildICmp(ctx->builder, LLVMIntNE,
+ fmask_word1, ctx->i32zero, "");
+
+ /* Replace the MSAA sample index. */
+ sample_index =
+ LLVMBuildSelect(ctx->builder, word1_is_nonzero,
+ final_sample, sample_index, "");
+ return sample_index;
+}
+
static LLVMValueRef get_image_coords(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
LLVMConstInt(ctx->i32, 2, false), LLVMConstInt(ctx->i32, 3, false),
};
LLVMValueRef res;
+ LLVMValueRef sample_index = llvm_extract_elem(ctx, get_src(ctx, instr->src[1]), 0);
+
int count;
enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS ||
count = image_type_to_components_count(dim,
glsl_sampler_type_is_array(type));
+ if (is_ms) {
+ LLVMValueRef fmask_load_address[3];
+ int chan;
+
+ fmask_load_address[0] = LLVMBuildExtractElement(ctx->builder, src0, masks[0], "");
+ fmask_load_address[1] = LLVMBuildExtractElement(ctx->builder, src0, masks[1], "");
+ if (glsl_sampler_type_is_array(type))
+ fmask_load_address[2] = LLVMBuildExtractElement(ctx->builder, src0, masks[2], "");
+ else
+ fmask_load_address[2] = NULL;
+ if (add_frag_pos) {
+ for (chan = 0; chan < 2; ++chan)
+ fmask_load_address[chan] = LLVMBuildAdd(ctx->builder, fmask_load_address[chan], LLVMBuildFPToUI(ctx->builder, ctx->frag_pos[chan], ctx->i32, ""), "");
+ }
+ sample_index = adjust_sample_index_using_fmask(ctx,
+ fmask_load_address[0],
+ fmask_load_address[1],
+ fmask_load_address[2],
+ sample_index,
+ get_sampler_desc(ctx, instr->variables[0], DESC_FMASK));
+ }
if (count == 1) {
if (instr->src[0].ssa->num_components)
res = LLVMBuildExtractElement(ctx->builder, src0, masks[0], "");
coords[chan] = LLVMBuildAdd(ctx->builder, coords[chan], LLVMBuildFPToUI(ctx->builder, ctx->frag_pos[chan], ctx->i32, ""), "");
}
if (is_ms) {
- coords[count] = llvm_extract_elem(ctx, get_src(ctx, instr->src[1]), 0);
+ coords[count] = sample_index;
count++;
}
return res;
}
-static void build_type_name_for_intr(
- LLVMTypeRef type,
- char *buf, unsigned bufsize)
-{
- LLVMTypeRef elem_type = type;
-
- assert(bufsize >= 8);
-
- if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
- int ret = snprintf(buf, bufsize, "v%u",
- LLVMGetVectorSize(type));
- if (ret < 0) {
- char *type_name = LLVMPrintTypeToString(type);
- fprintf(stderr, "Error building type name for: %s\n",
- type_name);
- return;
- }
- elem_type = LLVMGetElementType(type);
- buf += ret;
- bufsize -= ret;
- }
- switch (LLVMGetTypeKind(elem_type)) {
- default: break;
- case LLVMIntegerTypeKind:
- snprintf(buf, bufsize, "i%d", LLVMGetIntTypeWidth(elem_type));
- break;
- case LLVMFloatTypeKind:
- snprintf(buf, bufsize, "f32");
- break;
- case LLVMDoubleTypeKind:
- snprintf(buf, bufsize, "f64");
- break;
- }
-}
-
-static void get_image_intr_name(const char *base_name,
- LLVMTypeRef data_type,
- LLVMTypeRef coords_type,
- LLVMTypeRef rsrc_type,
- char *out_name, unsigned out_len)
-{
- char coords_type_name[8];
-
- build_type_name_for_intr(coords_type, coords_type_name,
- sizeof(coords_type_name));
-
- if (HAVE_LLVM <= 0x0309) {
- snprintf(out_name, out_len, "%s.%s", base_name, coords_type_name);
- } else {
- char data_type_name[8];
- char rsrc_type_name[8];
-
- build_type_name_for_intr(data_type, data_type_name,
- sizeof(data_type_name));
- build_type_name_for_intr(rsrc_type, rsrc_type_name,
- sizeof(rsrc_type_name));
- snprintf(out_name, out_len, "%s.%s.%s.%s", base_name,
- data_type_name, coords_type_name, rsrc_type_name);
- }
-}
-
static LLVMValueRef visit_image_load(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
params[9] = ctx->i32zero;
res = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32", ctx->v4i32,
- params, 10, AC_FUNC_ATTR_READNONE);
+ params, 10,
+ AC_FUNC_ATTR_READNONE |
+ AC_FUNC_ATTR_LEGACY);
if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
glsl_sampler_type_is_array(type)) {
ctx->f32zero, "");
ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kill",
ctx->voidt,
- &cond, 1, 0);
+ &cond, 1, AC_FUNC_ATTR_LEGACY);
}
static LLVMValueRef
visit_load_local_invocation_index(struct nir_to_llvm_context *ctx)
{
LLVMValueRef result;
- LLVMValueRef thread_id = get_thread_id(ctx);
+ LLVMValueRef thread_id = ac_get_thread_id(&ctx->ac);
result = LLVMBuildAnd(ctx->builder, ctx->tg_size,
LLVMConstInt(ctx->i32, 0xfc0, false), "");
unsigned location;
unsigned chan;
LLVMValueRef src_c0, src_c1;
- const char *intr_name;
LLVMValueRef src0;
int input_index = instr->variables[0]->var->data.location - VARYING_SLOT_VAR0;
switch (instr->intrinsic) {
location = INTERP_CENTROID;
break;
case nir_intrinsic_interp_var_at_sample:
- case nir_intrinsic_interp_var_at_offset:
location = INTERP_SAMPLE;
src0 = get_src(ctx, instr->src[0]);
break;
+ case nir_intrinsic_interp_var_at_offset:
+ location = INTERP_CENTER;
+ src0 = get_src(ctx, instr->src[0]);
default:
break;
}
interp_param = lookup_interp_param(ctx, instr->variables[0]->var->data.interpolation, location);
attr_number = LLVMConstInt(ctx->i32, input_index, false);
- if (location == INTERP_SAMPLE) {
+ if (location == INTERP_SAMPLE || location == INTERP_CENTER) {
LLVMValueRef ij_out[2];
LLVMValueRef ddxy_out = emit_ddxy_interp(ctx, interp_param);
interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
}
- intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
+
for (chan = 0; chan < 2; chan++) {
- LLVMValueRef args[4];
LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
- args[0] = llvm_chan;
- args[1] = attr_number;
- args[2] = ctx->prim_mask;
- args[3] = interp_param;
- result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
- ctx->f32, args, args[3] ? 4 : 3,
- AC_FUNC_ATTR_READNONE);
+ if (interp_param) {
+ interp_param = LLVMBuildBitCast(ctx->builder,
+ interp_param, LLVMVectorType(ctx->f32, 2), "");
+ LLVMValueRef i = LLVMBuildExtractElement(
+ ctx->builder, interp_param, ctx->i32zero, "");
+ LLVMValueRef j = LLVMBuildExtractElement(
+ ctx->builder, interp_param, ctx->i32one, "");
+
+ result[chan] = ac_build_fs_interp(&ctx->ac,
+ llvm_chan, attr_number,
+ ctx->prim_mask, i, j);
+ } else {
+ result[chan] = ac_build_fs_interp_mov(&ctx->ac,
+ LLVMConstInt(ctx->i32, 2, false),
+ llvm_chan, attr_number,
+ ctx->prim_mask);
+ }
}
return ac_build_gather_values(&ctx->ac, result, 2);
}
{
LLVMValueRef gs_next_vertex;
LLVMValueRef can_emit, kill;
- LLVMValueRef args[2];
int idx;
-
+ int clip_cull_slot = -1;
assert(instr->const_index[0] == 0);
/* Write vertex attribute values to GSVS ring */
gs_next_vertex = LLVMBuildLoad(ctx->builder,
LLVMConstReal(ctx->f32, 1.0f),
LLVMConstReal(ctx->f32, -1.0f), "");
ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kill",
- ctx->voidt, &kill, 1, 0);
+ ctx->voidt, &kill, 1, AC_FUNC_ATTR_LEGACY);
/* loop num outputs */
idx = 0;
for (unsigned i = 0; i < RADEON_LLVM_MAX_OUTPUTS; ++i) {
LLVMValueRef *out_ptr = &ctx->outputs[i * 4];
+ int length = 4;
+ int start = 0;
+ int slot = idx;
+ int slot_inc = 1;
+
if (!(ctx->output_mask & (1ull << i)))
continue;
- for (unsigned j = 0; j < 4; j++) {
+ if (i == VARYING_SLOT_CLIP_DIST1 ||
+ i == VARYING_SLOT_CULL_DIST1)
+ continue;
+
+ if (i == VARYING_SLOT_CLIP_DIST0 ||
+ i == VARYING_SLOT_CULL_DIST0) {
+ /* pack clip and cull into a single set of slots */
+ if (clip_cull_slot == -1) {
+ clip_cull_slot = idx;
+ if (ctx->num_output_clips + ctx->num_output_culls > 4)
+ slot_inc = 2;
+ } else {
+ slot = clip_cull_slot;
+ slot_inc = 0;
+ }
+ if (i == VARYING_SLOT_CLIP_DIST0)
+ length = ctx->num_output_clips;
+ if (i == VARYING_SLOT_CULL_DIST0) {
+ start = ctx->num_output_clips;
+ length = ctx->num_output_culls;
+ }
+ }
+ for (unsigned j = 0; j < length; j++) {
LLVMValueRef out_val = LLVMBuildLoad(ctx->builder,
out_ptr[j], "");
- LLVMValueRef voffset = LLVMConstInt(ctx->i32, (idx * 4 + j) * ctx->gs_max_out_vertices, false);
+ LLVMValueRef voffset = LLVMConstInt(ctx->i32, (slot * 4 + j + start) * ctx->gs_max_out_vertices, false);
voffset = LLVMBuildAdd(ctx->builder, voffset, gs_next_vertex, "");
voffset = LLVMBuildMul(ctx->builder, voffset, LLVMConstInt(ctx->i32, 4, false), "");
V_008F0C_BUF_NUM_FORMAT_UINT,
1, 0, 1, 1, 0);
}
- idx++;
+ idx += slot_inc;
}
gs_next_vertex = LLVMBuildAdd(ctx->builder, gs_next_vertex,
ctx->i32one, "");
LLVMBuildStore(ctx->builder, gs_next_vertex, ctx->gs_next_vertex);
- args[0] = LLVMConstInt(ctx->i32, SENDMSG_GS_OP_EMIT | SENDMSG_GS | (0 << 8), false);
- args[1] = ctx->gs_wave_id;
- ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.sendmsg",
- ctx->voidt, args, 2, 0);
+
+ ac_emit_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (0 << 8), ctx->gs_wave_id);
}
static void
visit_end_primitive(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
- LLVMValueRef args[2];
-
- assert(instr->const_index[0] == 0);
- args[0] = LLVMConstInt(ctx->i32, SENDMSG_GS_OP_CUT | SENDMSG_GS | (0 << 8), false);
- args[1] = ctx->gs_wave_id;
-
- ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.sendmsg", ctx->voidt,
- args, 2, 0);
+ ac_emit_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (0 << 8), ctx->gs_wave_id);
}
static void visit_intrinsic(struct nir_to_llvm_context *ctx,
ctx->shader_info->fs.force_persample = true;
result = load_sample_pos(ctx);
break;
+ case nir_intrinsic_load_sample_mask_in:
+ result = ctx->sample_coverage;
+ break;
case nir_intrinsic_load_front_face:
result = ctx->front_face;
break;
ctx->shader_info->fs.can_discard = true;
ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kilp",
ctx->voidt,
- NULL, 0, 0);
+ NULL, 0, AC_FUNC_ATTR_LEGACY);
break;
case nir_intrinsic_discard_if:
emit_discard_if(ctx, instr);
LLVMBuilderRef builder = ctx->builder;
LLVMTypeRef type;
LLVMValueRef index = NULL;
+ unsigned constant_index = 0;
assert(deref->var->data.binding < layout->binding_count);
if (child->deref_array_type == nir_deref_array_type_indirect) {
index = get_src(ctx, child->indirect);
}
+
+ constant_index = child->base_offset;
+ }
+ if (desc_type == DESC_SAMPLER && binding->immutable_samplers &&
+ (!index || binding->immutable_samplers_equal)) {
+ if (binding->immutable_samplers_equal)
+ constant_index = 0;
+
+ LLVMValueRef constants[] = {
+ LLVMConstInt(ctx->i32, binding->immutable_samplers[constant_index * 4 + 0], 0),
+ LLVMConstInt(ctx->i32, binding->immutable_samplers[constant_index * 4 + 1], 0),
+ LLVMConstInt(ctx->i32, binding->immutable_samplers[constant_index * 4 + 2], 0),
+ LLVMConstInt(ctx->i32, binding->immutable_samplers[constant_index * 4 + 3], 0),
+ };
+ return ac_build_gather_values(&ctx->ac, constants, 4);
}
assert(stride % type_size == 0);
}
if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
- result = get_buffer_size(ctx, res_ptr, false);
+ result = get_buffer_size(ctx, res_ptr, true);
goto write_result;
}
goto write_result;
}
- /* Adjust the sample index according to FMASK.
- *
- * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
- * which is the identity mapping. Each nibble says which physical sample
- * should be fetched to get that sample.
- *
- * For example, 0x11111100 means there are only 2 samples stored and
- * the second sample covers 3/4 of the pixel. When reading samples 0
- * and 1, return physical sample 0 (determined by the first two 0s
- * in FMASK), otherwise return physical sample 1.
- *
- * The sample index should be adjusted as follows:
- * sample_index = (fmask >> (sample_index * 4)) & 0xF;
- */
- if (instr->sampler_dim == GLSL_SAMPLER_DIM_MS) {
- LLVMValueRef txf_address[4];
- struct ac_tex_info txf_info = { 0 };
- unsigned txf_count = count;
- memcpy(txf_address, address, sizeof(txf_address));
-
- if (!instr->is_array)
- txf_address[2] = ctx->i32zero;
- txf_address[3] = ctx->i32zero;
-
- set_tex_fetch_args(ctx, &txf_info, instr, nir_texop_txf,
- fmask_ptr, NULL,
- txf_address, txf_count, 0xf);
-
- result = build_tex_intrinsic(ctx, instr, &txf_info);
- LLVMValueRef four = LLVMConstInt(ctx->i32, 4, false);
- LLVMValueRef F = LLVMConstInt(ctx->i32, 0xf, false);
-
- LLVMValueRef fmask = LLVMBuildExtractElement(ctx->builder,
- result,
- ctx->i32zero, "");
-
+ if (instr->sampler_dim == GLSL_SAMPLER_DIM_MS &&
+ instr->op != nir_texop_txs) {
unsigned sample_chan = instr->is_array ? 3 : 2;
-
- LLVMValueRef sample_index4 =
- LLVMBuildMul(ctx->builder, address[sample_chan], four, "");
- LLVMValueRef shifted_fmask =
- LLVMBuildLShr(ctx->builder, fmask, sample_index4, "");
- LLVMValueRef final_sample =
- LLVMBuildAnd(ctx->builder, shifted_fmask, F, "");
-
- /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
- * resource descriptor is 0 (invalid),
- */
- LLVMValueRef fmask_desc =
- LLVMBuildBitCast(ctx->builder, fmask_ptr,
- ctx->v8i32, "");
-
- LLVMValueRef fmask_word1 =
- LLVMBuildExtractElement(ctx->builder, fmask_desc,
- ctx->i32one, "");
-
- LLVMValueRef word1_is_nonzero =
- LLVMBuildICmp(ctx->builder, LLVMIntNE,
- fmask_word1, ctx->i32zero, "");
-
- /* Replace the MSAA sample index. */
- address[sample_chan] =
- LLVMBuildSelect(ctx->builder, word1_is_nonzero,
- final_sample, address[sample_chan], "");
+ address[sample_chan] = adjust_sample_index_using_fmask(ctx,
+ address[0],
+ address[1],
+ instr->is_array ? address[2] : NULL,
+ address[sample_chan],
+ fmask_ptr);
}
if (offsets && instr->op == nir_texop_txf) {
args[2] = buffer_index;
input = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.vs.load.input", ctx->v4f32, args, 3,
- AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
+ AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND |
+ AC_FUNC_ATTR_LEGACY);
for (unsigned chan = 0; chan < 4; chan++) {
LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
}
}
+static void
+handle_gs_input_decl(struct nir_to_llvm_context *ctx,
+ struct nir_variable *variable)
+{
+ int idx = variable->data.location;
+
+ if (idx == VARYING_SLOT_CLIP_DIST0 ||
+ idx == VARYING_SLOT_CULL_DIST0) {
+ int length = glsl_get_length(glsl_get_array_element(variable->type));
+ if (idx == VARYING_SLOT_CLIP_DIST0)
+ ctx->num_input_clips = length;
+ else
+ ctx->num_input_culls = length;
+ }
+}
static void interp_fs_input(struct nir_to_llvm_context *ctx,
unsigned attr,
LLVMValueRef prim_mask,
LLVMValueRef result[4])
{
- const char *intr_name;
LLVMValueRef attr_number;
unsigned chan;
+ LLVMValueRef i, j;
+ bool interp = interp_param != NULL;
attr_number = LLVMConstInt(ctx->i32, attr, false);
* fs.interp cannot be used on integers, because they can be equal
* to NaN.
*/
- intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
+ if (interp) {
+ interp_param = LLVMBuildBitCast(ctx->builder, interp_param,
+ LLVMVectorType(ctx->f32, 2), "");
+
+ i = LLVMBuildExtractElement(ctx->builder, interp_param,
+ ctx->i32zero, "");
+ j = LLVMBuildExtractElement(ctx->builder, interp_param,
+ ctx->i32one, "");
+ }
for (chan = 0; chan < 4; chan++) {
- LLVMValueRef args[4];
LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
- args[0] = llvm_chan;
- args[1] = attr_number;
- args[2] = prim_mask;
- args[3] = interp_param;
- result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
- ctx->f32, args, args[3] ? 4 : 3,
- AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
+ if (interp) {
+ result[chan] = ac_build_fs_interp(&ctx->ac,
+ llvm_chan,
+ attr_number,
+ prim_mask, i, j);
+ } else {
+ result[chan] = ac_build_fs_interp_mov(&ctx->ac,
+ LLVMConstInt(ctx->i32, 2, false),
+ llvm_chan,
+ attr_number,
+ prim_mask);
+ }
}
}
case MESA_SHADER_FRAGMENT:
handle_fs_input_decl(ctx, variable);
break;
+ case MESA_SHADER_GEOMETRY:
+ handle_gs_input_decl(ctx, variable);
+ break;
default:
break;
}
if (idx == VARYING_SLOT_CLIP_DIST0 ||
idx == VARYING_SLOT_CULL_DIST0) {
int length = glsl_get_length(variable->type);
- if (ctx->stage == MESA_SHADER_VERTEX) {
- if (idx == VARYING_SLOT_CLIP_DIST0) {
+ if (idx == VARYING_SLOT_CLIP_DIST0) {
+ if (ctx->stage == MESA_SHADER_VERTEX)
ctx->shader_info->vs.clip_dist_mask = (1 << length) - 1;
- ctx->num_clips = length;
- } else if (idx == VARYING_SLOT_CULL_DIST0) {
+ ctx->num_output_clips = length;
+ } else if (idx == VARYING_SLOT_CULL_DIST0) {
+ if (ctx->stage == MESA_SHADER_VERTEX)
ctx->shader_info->vs.cull_dist_mask = (1 << length) - 1;
- ctx->num_culls = length;
- }
+ ctx->num_output_culls = length;
}
if (length > 4)
attrib_count = 2;
LLVMValueRef packed;
packed = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.packf16",
- ctx->i32, pack_args, 2,
- AC_FUNC_ATTR_READNONE);
+ ctx->i32, pack_args, 2,
+ AC_FUNC_ATTR_READNONE |
+ AC_FUNC_ATTR_LEGACY);
args[chan + 5] = packed;
}
break;
unsigned j;
if (ctx->shader_info->vs.cull_dist_mask)
- ctx->shader_info->vs.cull_dist_mask <<= ctx->num_clips;
+ ctx->shader_info->vs.cull_dist_mask <<= ctx->num_output_clips;
i = VARYING_SLOT_CLIP_DIST0;
- for (j = 0; j < ctx->num_clips; j++)
+ for (j = 0; j < ctx->num_output_clips; j++)
slots[j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
i = VARYING_SLOT_CULL_DIST0;
- for (j = 0; j < ctx->num_culls; j++)
- slots[ctx->num_clips + j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
+ for (j = 0; j < ctx->num_output_culls; j++)
+ slots[ctx->num_output_clips + j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
- for (i = ctx->num_clips + ctx->num_culls; i < 8; i++)
+ for (i = ctx->num_output_clips + ctx->num_output_culls; i < 8; i++)
slots[i] = LLVMGetUndef(ctx->f32);
- if (ctx->num_clips + ctx->num_culls > 4) {
+ if (ctx->num_output_clips + ctx->num_output_culls > 4) {
target = V_008DFC_SQ_EXP_POS + 3;
si_llvm_init_export_args(ctx, &slots[4], target, args);
memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.export",
ctx->voidt,
- args, 9, 0);
+ args, 9,
+ AC_FUNC_ATTR_LEGACY);
}
}
ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.export",
ctx->voidt,
- pos_args[i], 9, 0);
+ pos_args[i], 9,
+ AC_FUNC_ATTR_LEGACY);
}
ctx->shader_info->vs.pos_exports = num_pos_exports;
for (unsigned i = 0; i < RADEON_LLVM_MAX_OUTPUTS; ++i) {
LLVMValueRef *out_ptr = &ctx->outputs[i * 4];
int param_index;
+ int length = 4;
+ int start = 0;
if (!(ctx->output_mask & (1ull << i)))
continue;
+ if (i == VARYING_SLOT_CLIP_DIST0) {
+ length = ctx->num_output_clips;
+ } else if (i == VARYING_SLOT_CULL_DIST0) {
+ start = ctx->num_output_clips;
+ length = ctx->num_output_culls;
+ }
param_index = shader_io_get_unique_index(i);
if (param_index > max_output_written)
max_output_written = param_index;
- for (j = 0; j < 4; j++) {
+ for (j = 0; j < length; j++) {
LLVMValueRef out_val = LLVMBuildLoad(ctx->builder, out_ptr[j], "");
out_val = LLVMBuildBitCast(ctx->builder, out_val, ctx->i32, "");
ctx->esgs_ring,
out_val, 1,
LLVMGetUndef(ctx->i32), ctx->es2gs_offset,
- (4 * param_index + j) * 4,
+ (4 * param_index + j + start) * 4,
V_008F0C_BUF_DATA_FORMAT_32,
V_008F0C_BUF_NUM_FORMAT_UINT,
0, 0, 1, 1, 0);
return; /* unnecessary NULL export */
ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
- ctx->voidt, args, 9, 0);
+ ctx->voidt, args, 9,
+ AC_FUNC_ATTR_LEGACY);
}
static void
args[0] = LLVMConstInt(ctx->i32, mask, false);
ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
- ctx->voidt, args, 9, 0);
+ ctx->voidt, args, 9,
+ AC_FUNC_ATTR_LEGACY);
}
static void
ctx->shader_info->fs.writes_stencil = true;
stencil = to_float(ctx, LLVMBuildLoad(ctx->builder,
ctx->outputs[radeon_llvm_reg_index_soa(i, 0)], ""));
+ } else if (i == FRAG_RESULT_SAMPLE_MASK) {
+ ctx->shader_info->fs.writes_sample_mask = true;
+ samplemask = to_float(ctx, LLVMBuildLoad(ctx->builder,
+ ctx->outputs[radeon_llvm_reg_index_soa(i, 0)], ""));
} else {
bool last = false;
for (unsigned j = 0; j < 4; j++)
values[j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
- if (!ctx->shader_info->fs.writes_z && !ctx->shader_info->fs.writes_stencil)
+ if (!ctx->shader_info->fs.writes_z && !ctx->shader_info->fs.writes_stencil && !ctx->shader_info->fs.writes_sample_mask)
last = ctx->output_mask <= ((1ull << (i + 1)) - 1);
si_export_mrt_color(ctx, values, V_008DFC_SQ_EXP_MRT + index, last);
}
}
- if (depth || stencil)
+ if (depth || stencil || samplemask)
si_export_mrt_z(ctx, depth, stencil, samplemask);
else if (!index)
si_export_mrt_color(ctx, NULL, V_008DFC_SQ_EXP_NULL, true);
static void
emit_gs_epilogue(struct nir_to_llvm_context *ctx)
{
- LLVMValueRef args[2];
-
- args[0] = LLVMConstInt(ctx->i32, SENDMSG_GS_OP_NOP | SENDMSG_GS_DONE, false);
- args[1] = ctx->gs_wave_id;
- ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.sendmsg",
- ctx->voidt, args, 2, 0);
+ ac_emit_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE, ctx->gs_wave_id);
}
static void
idx++;
}
- shared_size *= 4;
+ shared_size *= 16;
var = LLVMAddGlobalInAddressSpace(ctx.module,
LLVMArrayType(ctx.i8, shared_size),
"compute_lds",
args[8] = ctx->i32zero; /* TFE */
int idx = 0;
+ int clip_cull_slot = -1;
for (unsigned i = 0; i < RADEON_LLVM_MAX_OUTPUTS; ++i) {
+ int length = 4;
+ int start = 0;
+ int slot = idx;
+ int slot_inc = 1;
if (!(ctx->output_mask & (1ull << i)))
continue;
- for (unsigned j = 0; j < 4; j++) {
+ if (i == VARYING_SLOT_CLIP_DIST1 ||
+ i == VARYING_SLOT_CULL_DIST1)
+ continue;
+
+ if (i == VARYING_SLOT_CLIP_DIST0 ||
+ i == VARYING_SLOT_CULL_DIST0) {
+ /* unpack clip and cull from a single set of slots */
+ if (clip_cull_slot == -1) {
+ clip_cull_slot = idx;
+ if (ctx->num_output_clips + ctx->num_output_culls > 4)
+ slot_inc = 2;
+ } else {
+ slot = clip_cull_slot;
+ slot_inc = 0;
+ }
+ if (i == VARYING_SLOT_CLIP_DIST0)
+ length = ctx->num_output_clips;
+ if (i == VARYING_SLOT_CULL_DIST0) {
+ start = ctx->num_output_clips;
+ length = ctx->num_output_culls;
+ }
+ }
+
+ for (unsigned j = 0; j < length; j++) {
LLVMValueRef value;
args[2] = LLVMConstInt(ctx->i32,
- (idx * 4 + j) *
+ (slot * 4 + j + start) *
ctx->gs_max_out_vertices * 16 * 4, false);
value = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.buffer.load.dword.i32.i32",
ctx->i32, args, 9,
- AC_FUNC_ATTR_READONLY);
+ AC_FUNC_ATTR_READONLY |
+ AC_FUNC_ATTR_LEGACY);
LLVMBuildStore(ctx->builder,
to_float(ctx, value), ctx->outputs[radeon_llvm_reg_index_soa(i, j)]);
}
- idx++;
+ idx += slot_inc;
}
handle_vs_outputs_post(ctx);
}
projects / mesa.git / blobdiff