struct ac_shader_abi *abi;
gl_shader_stage stage;
+ shader_info *info;
LLVMValueRef *ssa_defs;
+ LLVMValueRef scratch;
+
struct hash_table *defs;
struct hash_table *phis;
struct hash_table *vars;
{
switch (dim) {
case GLSL_SAMPLER_DIM_1D:
- if (ctx->chip_class >= GFX9)
+ if (ctx->chip_class == GFX9)
return is_array ? ac_image_2darray : ac_image_2d;
return is_array ? ac_image_1darray : ac_image_1d;
case GLSL_SAMPLER_DIM_2D:
ac_to_float(ctx, src0),
};
- MAYBE_UNUSED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
+ ASSERTED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
ac_get_elem_bits(ctx, result_type));
assert(length < sizeof(name));
return ac_build_intrinsic(ctx, name, result_type, params, 1, AC_FUNC_ATTR_READNONE);
ac_to_float(ctx, src1),
};
- MAYBE_UNUSED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
+ ASSERTED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
ac_get_elem_bits(ctx, result_type));
assert(length < sizeof(name));
return ac_build_intrinsic(ctx, name, result_type, params, 2, AC_FUNC_ATTR_READNONE);
ac_to_float(ctx, src2),
};
- MAYBE_UNUSED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
+ ASSERTED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
ac_get_elem_bits(ctx, result_type));
assert(length < sizeof(name));
return ac_build_intrinsic(ctx, name, result_type, params, 3, AC_FUNC_ATTR_READNONE);
return result;
}
-static LLVMValueRef emit_bitfield_extract(struct ac_llvm_context *ctx,
- bool is_signed,
- const LLVMValueRef srcs[3])
+static LLVMValueRef emit_bfm(struct ac_llvm_context *ctx,
+ LLVMValueRef bits, LLVMValueRef offset)
{
- LLVMValueRef result;
-
- if (HAVE_LLVM >= 0x0800) {
- LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2], LLVMConstInt(ctx->i32, 32, false), "");
- result = ac_build_bfe(ctx, srcs[0], srcs[1], srcs[2], is_signed);
- result = LLVMBuildSelect(ctx->builder, icond, srcs[0], result, "");
- } else {
- /* FIXME: LLVM 7+ returns incorrect result when count is 0.
- * https://bugs.freedesktop.org/show_bug.cgi?id=107276
- */
- LLVMValueRef zero = ctx->i32_0;
- LLVMValueRef icond1 = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2], LLVMConstInt(ctx->i32, 32, false), "");
- LLVMValueRef icond2 = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2], zero, "");
-
- result = ac_build_bfe(ctx, srcs[0], srcs[1], srcs[2], is_signed);
- result = LLVMBuildSelect(ctx->builder, icond1, srcs[0], result, "");
- result = LLVMBuildSelect(ctx->builder, icond2, zero, result, "");
- }
-
- return result;
+ /* mask = ((1 << bits) - 1) << offset */
+ return LLVMBuildShl(ctx->builder,
+ LLVMBuildSub(ctx->builder,
+ LLVMBuildShl(ctx->builder,
+ ctx->i32_1,
+ bits, ""),
+ ctx->i32_1, ""),
+ offset, "");
}
-static LLVMValueRef emit_bitfield_insert(struct ac_llvm_context *ctx,
- LLVMValueRef src0, LLVMValueRef src1,
- LLVMValueRef src2, LLVMValueRef src3)
+static LLVMValueRef emit_bitfield_select(struct ac_llvm_context *ctx,
+ LLVMValueRef mask, LLVMValueRef insert,
+ LLVMValueRef base)
{
- LLVMValueRef bfi_args[3], result;
-
- bfi_args[0] = LLVMBuildShl(ctx->builder,
- LLVMBuildSub(ctx->builder,
- LLVMBuildShl(ctx->builder,
- ctx->i32_1,
- src3, ""),
- ctx->i32_1, ""),
- src2, "");
- bfi_args[1] = LLVMBuildShl(ctx->builder, src1, src2, "");
- bfi_args[2] = src0;
-
- LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, src3, LLVMConstInt(ctx->i32, 32, false), "");
-
/* Calculate:
- * (arg0 & arg1) | (~arg0 & arg2) = arg2 ^ (arg0 & (arg1 ^ arg2)
+ * (mask & insert) | (~mask & base) = base ^ (mask & (insert ^ base))
* Use the right-hand side, which the LLVM backend can convert to V_BFI.
*/
- result = LLVMBuildXor(ctx->builder, bfi_args[2],
- LLVMBuildAnd(ctx->builder, bfi_args[0],
- LLVMBuildXor(ctx->builder, bfi_args[1], bfi_args[2], ""), ""), "");
-
- result = LLVMBuildSelect(ctx->builder, icond, src1, result, "");
- return result;
+ return LLVMBuildXor(ctx->builder, base,
+ LLVMBuildAnd(ctx->builder, mask,
+ LLVMBuildXor(ctx->builder, insert, base, ""), ""), "");
}
-static LLVMValueRef emit_pack_half_2x16(struct ac_llvm_context *ctx,
- LLVMValueRef src0)
+static LLVMValueRef emit_pack_2x16(struct ac_llvm_context *ctx,
+ LLVMValueRef src0,
+ LLVMValueRef (*pack)(struct ac_llvm_context *ctx,
+ LLVMValueRef args[2]))
{
LLVMValueRef comp[2];
comp[0] = LLVMBuildExtractElement(ctx->builder, src0, ctx->i32_0, "");
comp[1] = LLVMBuildExtractElement(ctx->builder, src0, ctx->i32_1, "");
- return LLVMBuildBitCast(ctx->builder, ac_build_cvt_pkrtz_f16(ctx, comp),
- ctx->i32, "");
+ return LLVMBuildBitCast(ctx->builder, pack(ctx, comp), ctx->i32, "");
}
static LLVMValueRef emit_unpack_half_2x16(struct ac_llvm_context *ctx,
src_components = 1;
break;
case nir_op_pack_half_2x16:
+ case nir_op_pack_snorm_2x16:
+ case nir_op_pack_unorm_2x16:
src_components = 2;
break;
case nir_op_unpack_half_2x16:
else
result = ac_build_intrinsic(&ctx->ac, "llvm.amdgcn.ldexp.f64", ctx->ac.f64, src, 2, AC_FUNC_ATTR_READNONE);
break;
- case nir_op_ibitfield_extract:
- result = emit_bitfield_extract(&ctx->ac, true, src);
+ case nir_op_bfm:
+ result = emit_bfm(&ctx->ac, src[0], src[1]);
break;
- case nir_op_ubitfield_extract:
- result = emit_bitfield_extract(&ctx->ac, false, src);
+ case nir_op_bitfield_select:
+ result = emit_bitfield_select(&ctx->ac, src[0], src[1], src[2]);
break;
- case nir_op_bitfield_insert:
- result = emit_bitfield_insert(&ctx->ac, src[0], src[1], src[2], src[3]);
+ case nir_op_ubfe:
+ result = ac_build_bfe(&ctx->ac, src[0], src[1], src[2], false);
+ break;
+ case nir_op_ibfe:
+ result = ac_build_bfe(&ctx->ac, src[0], src[1], src[2], true);
break;
case nir_op_bitfield_reverse:
result = ac_build_bitfield_reverse(&ctx->ac, src[0]);
result = emit_imul_high(&ctx->ac, src[0], src[1]);
break;
case nir_op_pack_half_2x16:
- result = emit_pack_half_2x16(&ctx->ac, src[0]);
+ result = emit_pack_2x16(&ctx->ac, src[0], ac_build_cvt_pkrtz_f16);
+ break;
+ case nir_op_pack_snorm_2x16:
+ result = emit_pack_2x16(&ctx->ac, src[0], ac_build_cvt_pknorm_i16);
+ break;
+ case nir_op_pack_unorm_2x16:
+ result = emit_pack_2x16(&ctx->ac, src[0], ac_build_cvt_pknorm_u16);
break;
case nir_op_unpack_half_2x16:
result = emit_unpack_half_2x16(&ctx->ac, src[0]);
args->coords[0],
ctx->ac.i32_0,
util_last_bit(mask),
- false, true);
+ 0, true);
} else {
return ac_build_buffer_load_format(&ctx->ac,
args->resource,
args->coords[0],
ctx->ac.i32_0,
util_last_bit(mask),
- false, true);
+ 0, true);
}
}
}
/* Fixup for GFX9 which allocates 1D textures as 2D. */
- if (instr->op == nir_texop_lod && ctx->ac.chip_class >= GFX9) {
+ if (instr->op == nir_texop_lod && ctx->ac.chip_class == GFX9) {
if ((args->dim == ac_image_2darray ||
args->dim == ac_image_2d) && !args->coords[1]) {
args->coords[1] = ctx->ac.i32_0;
}
args->attributes = AC_FUNC_ATTR_READNONE;
+ bool cs_derivs = ctx->stage == MESA_SHADER_COMPUTE &&
+ ctx->info->cs.derivative_group != DERIVATIVE_GROUP_NONE;
+ if (ctx->stage == MESA_SHADER_FRAGMENT || cs_derivs) {
+ /* Prevent texture instructions with implicit derivatives from being
+ * sinked into branches. */
+ switch (instr->op) {
+ case nir_texop_tex:
+ case nir_texop_txb:
+ case nir_texop_lod:
+ args->attributes |= AC_FUNC_ATTR_CONVERGENT;
+ break;
+ default:
+ break;
+ }
+ }
+
return ac_build_image_opcode(&ctx->ac, args);
}
if (num_bytes == 1) {
ac_build_tbuffer_store_byte(&ctx->ac, rsrc, data,
offset, ctx->ac.i32_0,
- cache_policy & ac_glc,
- writeonly_memory);
+ cache_policy);
} else if (num_bytes == 2) {
ac_build_tbuffer_store_short(&ctx->ac, rsrc, data,
offset, ctx->ac.i32_0,
- cache_policy & ac_glc,
- writeonly_memory);
+ cache_policy);
} else {
int num_channels = num_bytes / 4;
ac_build_buffer_store_dword(&ctx->ac, rsrc, data,
num_channels, offset,
ctx->ac.i32_0, 0,
- cache_policy & ac_glc,
- false, writeonly_memory,
- false);
+ cache_policy, false);
}
}
}
+static LLVMValueRef emit_ssbo_comp_swap_64(struct ac_nir_context *ctx,
+ LLVMValueRef descriptor,
+ LLVMValueRef offset,
+ LLVMValueRef compare,
+ LLVMValueRef exchange)
+{
+ LLVMBasicBlockRef start_block = NULL, then_block = NULL;
+ if (ctx->abi->robust_buffer_access) {
+ LLVMValueRef size = ac_llvm_extract_elem(&ctx->ac, descriptor, 2);
+
+ LLVMValueRef cond = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, offset, size, "");
+ start_block = LLVMGetInsertBlock(ctx->ac.builder);
+
+ ac_build_ifcc(&ctx->ac, cond, -1);
+
+ then_block = LLVMGetInsertBlock(ctx->ac.builder);
+ }
+
+ LLVMValueRef ptr_parts[2] = {
+ ac_llvm_extract_elem(&ctx->ac, descriptor, 0),
+ LLVMBuildAnd(ctx->ac.builder,
+ ac_llvm_extract_elem(&ctx->ac, descriptor, 1),
+ LLVMConstInt(ctx->ac.i32, 65535, 0), "")
+ };
+
+ ptr_parts[1] = LLVMBuildTrunc(ctx->ac.builder, ptr_parts[1], ctx->ac.i16, "");
+ ptr_parts[1] = LLVMBuildSExt(ctx->ac.builder, ptr_parts[1], ctx->ac.i32, "");
+
+ offset = LLVMBuildZExt(ctx->ac.builder, offset, ctx->ac.i64, "");
+
+ LLVMValueRef ptr = ac_build_gather_values(&ctx->ac, ptr_parts, 2);
+ ptr = LLVMBuildBitCast(ctx->ac.builder, ptr, ctx->ac.i64, "");
+ ptr = LLVMBuildAdd(ctx->ac.builder, ptr, offset, "");
+ ptr = LLVMBuildIntToPtr(ctx->ac.builder, ptr, LLVMPointerType(ctx->ac.i64, AC_ADDR_SPACE_GLOBAL), "");
+
+ LLVMValueRef result = ac_build_atomic_cmp_xchg(&ctx->ac, ptr, compare, exchange, "singlethread-one-as");
+ result = LLVMBuildExtractValue(ctx->ac.builder, result, 0, "");
+
+ if (ctx->abi->robust_buffer_access) {
+ ac_build_endif(&ctx->ac, -1);
+
+ LLVMBasicBlockRef incoming_blocks[2] = {
+ start_block,
+ then_block,
+ };
+
+ LLVMValueRef incoming_values[2] = {
+ LLVMConstInt(ctx->ac.i64, 0, 0),
+ result,
+ };
+ LLVMValueRef ret = LLVMBuildPhi(ctx->ac.builder, ctx->ac.i64, "");
+ LLVMAddIncoming(ret, incoming_values, incoming_blocks, 2);
+ return ret;
+ } else {
+ return result;
+ }
+}
+
static LLVMValueRef visit_atomic_ssbo(struct ac_nir_context *ctx,
const nir_intrinsic_instr *instr)
{
LLVMTypeRef return_type = LLVMTypeOf(get_src(ctx, instr->src[2]));
const char *op;
char name[64], type[8];
- LLVMValueRef params[6];
+ LLVMValueRef params[6], descriptor;
int arg_count = 0;
switch (instr->intrinsic) {
abort();
}
+ descriptor = ctx->abi->load_ssbo(ctx->abi,
+ get_src(ctx, instr->src[0]),
+ true);
+
+ if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap &&
+ return_type == ctx->ac.i64) {
+ return emit_ssbo_comp_swap_64(ctx, descriptor,
+ get_src(ctx, instr->src[1]),
+ get_src(ctx, instr->src[2]),
+ get_src(ctx, instr->src[3]));
+ }
if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) {
params[arg_count++] = ac_llvm_extract_elem(&ctx->ac, get_src(ctx, instr->src[3]), 0);
}
params[arg_count++] = ac_llvm_extract_elem(&ctx->ac, get_src(ctx, instr->src[2]), 0);
- params[arg_count++] = ctx->abi->load_ssbo(ctx->abi,
- get_src(ctx, instr->src[0]),
- true);
+ params[arg_count++] = descriptor;
if (HAVE_LLVM >= 0x900) {
/* XXX: The new raw/struct atomic intrinsics are buggy with
offset,
ctx->ac.i32_0,
immoffset,
- cache_policy & ac_glc);
+ cache_policy);
} else if (load_bytes == 2) {
ret = ac_build_tbuffer_load_short(&ctx->ac,
rsrc,
offset,
ctx->ac.i32_0,
immoffset,
- cache_policy & ac_glc);
+ cache_policy);
} else {
int num_channels = util_next_power_of_two(load_bytes) / 4;
+ bool can_speculate = access & ACCESS_CAN_REORDER;
ret = ac_build_buffer_load(&ctx->ac, rsrc, num_channels,
vindex, offset, immoffset, 0,
- cache_policy & ac_glc, 0,
- false, false);
+ cache_policy, can_speculate, false);
}
LLVMTypeRef byte_vec = LLVMVectorType(ctx->ac.i8, ac_get_type_size(LLVMTypeOf(ret)));
offset,
ctx->ac.i32_0,
immoffset,
- false);
+ 0);
} else {
assert(load_bytes == 2);
results[i] = ac_build_tbuffer_load_short(&ctx->ac,
offset,
ctx->ac.i32_0,
immoffset,
- false);
+ 0);
}
}
ret = ac_build_gather_values(&ctx->ac, results, num_components);
} else {
ret = ac_build_buffer_load(&ctx->ac, rsrc, num_components, NULL, offset,
- NULL, 0, false, false, true, true);
+ NULL, 0, 0, true, true);
ret = ac_trim_vector(&ctx->ac, ret, num_components);
}
return load_tess_varyings(ctx, instr, false);
}
+ if (ctx->stage == MESA_SHADER_FRAGMENT &&
+ var->data.fb_fetch_output &&
+ ctx->abi->emit_fbfetch)
+ return ctx->abi->emit_fbfetch(ctx->abi);
+
for (unsigned chan = comp; chan < ve + comp; chan++) {
if (indir_index) {
unsigned count = glsl_count_attribute_slots(
LLVMValueRef sample_index = ac_llvm_extract_elem(&ctx->ac, get_src(ctx, instr->src[2]), 0);
int count;
- bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS ||
- dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
+ ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS ||
+ dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
bool is_ms = (dim == GLSL_SAMPLER_DIM_MS ||
dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
- bool gfx9_1d = ctx->ac.chip_class >= GFX9 && dim == GLSL_SAMPLER_DIM_1D;
+ bool gfx9_1d = ctx->ac.chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
+ assert(!add_frag_pos && "Input attachments should be lowered by this point.");
count = image_type_to_components_count(dim, is_array);
if (is_ms && (instr->intrinsic == nir_intrinsic_image_deref_load ||
instr->intrinsic == nir_intrinsic_bindless_image_load)) {
LLVMValueRef fmask_load_address[3];
- int chan;
fmask_load_address[0] = LLVMBuildExtractElement(ctx->ac.builder, src0, masks[0], "");
fmask_load_address[1] = LLVMBuildExtractElement(ctx->ac.builder, src0, masks[1], "");
fmask_load_address[2] = LLVMBuildExtractElement(ctx->ac.builder, src0, masks[2], "");
else
fmask_load_address[2] = NULL;
- if (add_frag_pos) {
- for (chan = 0; chan < 2; ++chan)
- fmask_load_address[chan] =
- LLVMBuildAdd(ctx->ac.builder, fmask_load_address[chan],
- LLVMBuildFPToUI(ctx->ac.builder, ctx->abi->frag_pos[chan],
- ctx->ac.i32, ""), "");
- fmask_load_address[2] = ac_to_integer(&ctx->ac, ctx->abi->inputs[ac_llvm_reg_index_soa(VARYING_SLOT_LAYER, 0)]);
- }
+
sample_index = adjust_sample_index_using_fmask(&ctx->ac,
fmask_load_address[0],
fmask_load_address[1],
for (chan = 0; chan < count; ++chan) {
args->coords[chan] = ac_llvm_extract_elem(&ctx->ac, src0, chan);
}
- if (add_frag_pos) {
- for (chan = 0; chan < 2; ++chan) {
- args->coords[chan] = LLVMBuildAdd(
- ctx->ac.builder, args->coords[chan],
- LLVMBuildFPToUI(
- ctx->ac.builder, ctx->abi->frag_pos[chan],
- ctx->ac.i32, ""), "");
- }
- args->coords[2] = ac_to_integer(&ctx->ac,
- ctx->abi->inputs[ac_llvm_reg_index_soa(VARYING_SLOT_LAYER, 0)]);
- count++;
- }
if (gfx9_1d) {
if (is_array) {
vindex = LLVMBuildExtractElement(ctx->ac.builder, get_src(ctx, instr->src[1]),
ctx->ac.i32_0, "");
- /* TODO: set "can_speculate" when OpenGL needs it. */
+ bool can_speculate = access & ACCESS_CAN_REORDER;
res = ac_build_buffer_load_format(&ctx->ac, rsrc, vindex,
ctx->ac.i32_0, num_channels,
- !!(args.cache_policy & ac_glc),
- false);
+ args.cache_policy,
+ can_speculate);
res = ac_build_expand_to_vec4(&ctx->ac, res, num_channels);
res = ac_trim_vector(&ctx->ac, res, instr->dest.ssa.num_components);
ac_build_buffer_store_format(&ctx->ac, rsrc, src, vindex,
ctx->ac.i32_0, src_channels,
- args.cache_policy & ac_glc,
- writeonly_memory);
+ args.cache_policy);
} else {
args.opcode = ac_image_store;
args.data[0] = ac_to_float(&ctx->ac, get_src(ctx, instr->src[3]));
const char *atomic_name;
char intrinsic_name[64];
enum ac_atomic_op atomic_subop;
- MAYBE_UNUSED int length;
+ ASSERTED int length;
enum glsl_sampler_dim dim;
bool is_unsigned = false;
z = LLVMBuildSDiv(ctx->ac.builder, z, six, "");
res = LLVMBuildInsertElement(ctx->ac.builder, res, z, two, "");
}
- if (ctx->ac.chip_class >= GFX9 && dim == GLSL_SAMPLER_DIM_1D && is_array) {
+ if (ctx->ac.chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D && is_array) {
LLVMValueRef layers = LLVMBuildExtractElement(ctx->ac.builder, res, two, "");
res = LLVMBuildInsertElement(ctx->ac.builder, res, layers,
ctx->ac.i32_1, "");
static void emit_membar(struct ac_llvm_context *ac,
const nir_intrinsic_instr *instr)
{
- unsigned waitcnt = NOOP_WAITCNT;
+ unsigned wait_flags = 0;
switch (instr->intrinsic) {
case nir_intrinsic_memory_barrier:
case nir_intrinsic_group_memory_barrier:
- waitcnt &= VM_CNT & LGKM_CNT;
+ wait_flags = AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE;
break;
case nir_intrinsic_memory_barrier_atomic_counter:
case nir_intrinsic_memory_barrier_buffer:
case nir_intrinsic_memory_barrier_image:
- waitcnt &= VM_CNT;
+ wait_flags = AC_WAIT_VLOAD | AC_WAIT_VSTORE;
break;
case nir_intrinsic_memory_barrier_shared:
- waitcnt &= LGKM_CNT;
+ wait_flags = AC_WAIT_LGKM;
break;
default:
break;
}
- if (waitcnt != NOOP_WAITCNT)
- ac_build_waitcnt(ac, waitcnt);
+
+ ac_build_waitcnt(ac, wait_flags);
}
void ac_emit_barrier(struct ac_llvm_context *ac, gl_shader_stage stage)
* always fits into a single wave.
*/
if (ac->chip_class == GFX6 && stage == MESA_SHADER_TESS_CTRL) {
- ac_build_waitcnt(ac, LGKM_CNT & VM_CNT);
+ ac_build_waitcnt(ac, AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE);
return;
}
ac_build_s_barrier(ac);
visit_first_invocation(struct ac_nir_context *ctx)
{
LLVMValueRef active_set = ac_build_ballot(&ctx->ac, ctx->ac.i32_1);
+ const char *intr = ctx->ac.wave_size == 32 ? "llvm.cttz.i32" : "llvm.cttz.i64";
/* The second argument is whether cttz(0) should be defined, but we do not care. */
LLVMValueRef args[] = {active_set, ctx->ac.i1false};
- LLVMValueRef result = ac_build_intrinsic(&ctx->ac,
- "llvm.cttz.i64",
- ctx->ac.i64, args, 2,
+ LLVMValueRef result = ac_build_intrinsic(&ctx->ac, intr,
+ ctx->ac.iN_wavemask, args, 2,
AC_FUNC_ATTR_NOUNWIND |
AC_FUNC_ATTR_READNONE);
return ac_build_gather_values(&ctx->ac, values, 2);
}
-static LLVMValueRef visit_interp(struct ac_nir_context *ctx,
- const nir_intrinsic_instr *instr)
+static LLVMValueRef barycentric_center(struct ac_nir_context *ctx,
+ unsigned mode)
{
- LLVMValueRef result[4];
- LLVMValueRef interp_param;
- unsigned location;
- unsigned chan;
- LLVMValueRef src_c0 = NULL;
- LLVMValueRef src_c1 = NULL;
- LLVMValueRef src0 = NULL;
-
- nir_deref_instr *deref_instr = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
- nir_variable *var = nir_deref_instr_get_variable(deref_instr);
- int input_base = ctx->abi->fs_input_attr_indices[var->data.location - VARYING_SLOT_VAR0];
- switch (instr->intrinsic) {
- case nir_intrinsic_interp_deref_at_centroid:
- location = INTERP_CENTROID;
- break;
- case nir_intrinsic_interp_deref_at_sample:
- case nir_intrinsic_interp_deref_at_offset:
- location = INTERP_CENTER;
- src0 = get_src(ctx, instr->src[1]);
- break;
- default:
- break;
- }
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_CENTER);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- if (instr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
- src_c0 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, src0, ctx->ac.i32_0, ""));
- src_c1 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, src0, ctx->ac.i32_1, ""));
- } else if (instr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
- LLVMValueRef sample_position;
- LLVMValueRef halfval = LLVMConstReal(ctx->ac.f32, 0.5f);
+static LLVMValueRef barycentric_offset(struct ac_nir_context *ctx,
+ unsigned mode,
+ LLVMValueRef offset)
+{
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_CENTER);
+ LLVMValueRef src_c0 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, offset, ctx->ac.i32_0, ""));
+ LLVMValueRef src_c1 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, offset, ctx->ac.i32_1, ""));
- /* fetch sample ID */
- sample_position = ctx->abi->load_sample_position(ctx->abi, src0);
+ LLVMValueRef ij_out[2];
+ LLVMValueRef ddxy_out = ac_build_ddxy_interp(&ctx->ac, interp_param);
- src_c0 = LLVMBuildExtractElement(ctx->ac.builder, sample_position, ctx->ac.i32_0, "");
- src_c0 = LLVMBuildFSub(ctx->ac.builder, src_c0, halfval, "");
- src_c1 = LLVMBuildExtractElement(ctx->ac.builder, sample_position, ctx->ac.i32_1, "");
- src_c1 = LLVMBuildFSub(ctx->ac.builder, src_c1, halfval, "");
- }
- interp_param = ctx->abi->lookup_interp_param(ctx->abi, var->data.interpolation, location);
+ /*
+ * take the I then J parameters, and the DDX/Y for it, and
+ * calculate the IJ inputs for the interpolator.
+ * temp1 = ddx * offset/sample.x + I;
+ * interp_param.I = ddy * offset/sample.y + temp1;
+ * temp1 = ddx * offset/sample.x + J;
+ * interp_param.J = ddy * offset/sample.y + temp1;
+ */
+ for (unsigned i = 0; i < 2; i++) {
+ LLVMValueRef ix_ll = LLVMConstInt(ctx->ac.i32, i, false);
+ LLVMValueRef iy_ll = LLVMConstInt(ctx->ac.i32, i + 2, false);
+ LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
+ ddxy_out, ix_ll, "");
+ LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
+ ddxy_out, iy_ll, "");
+ LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
+ interp_param, ix_ll, "");
+ LLVMValueRef temp1, temp2;
+
+ interp_el = LLVMBuildBitCast(ctx->ac.builder, interp_el,
+ ctx->ac.f32, "");
+
+ temp1 = ac_build_fmad(&ctx->ac, ddx_el, src_c0, interp_el);
+ temp2 = ac_build_fmad(&ctx->ac, ddy_el, src_c1, temp1);
+
+ ij_out[i] = LLVMBuildBitCast(ctx->ac.builder,
+ temp2, ctx->ac.i32, "");
+ }
+ interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- if (location == INTERP_CENTER) {
- LLVMValueRef ij_out[2];
- LLVMValueRef ddxy_out = ac_build_ddxy_interp(&ctx->ac, interp_param);
+static LLVMValueRef barycentric_centroid(struct ac_nir_context *ctx,
+ unsigned mode)
+{
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_CENTROID);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- /*
- * take the I then J parameters, and the DDX/Y for it, and
- * calculate the IJ inputs for the interpolator.
- * temp1 = ddx * offset/sample.x + I;
- * interp_param.I = ddy * offset/sample.y + temp1;
- * temp1 = ddx * offset/sample.x + J;
- * interp_param.J = ddy * offset/sample.y + temp1;
- */
- for (unsigned i = 0; i < 2; i++) {
- LLVMValueRef ix_ll = LLVMConstInt(ctx->ac.i32, i, false);
- LLVMValueRef iy_ll = LLVMConstInt(ctx->ac.i32, i + 2, false);
- LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
- ddxy_out, ix_ll, "");
- LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
- ddxy_out, iy_ll, "");
- LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
- interp_param, ix_ll, "");
- LLVMValueRef temp1, temp2;
-
- interp_el = LLVMBuildBitCast(ctx->ac.builder, interp_el,
- ctx->ac.f32, "");
-
- temp1 = ac_build_fmad(&ctx->ac, ddx_el, src_c0, interp_el);
- temp2 = ac_build_fmad(&ctx->ac, ddy_el, src_c1, temp1);
-
- ij_out[i] = LLVMBuildBitCast(ctx->ac.builder,
- temp2, ctx->ac.i32, "");
- }
- interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
+static LLVMValueRef barycentric_at_sample(struct ac_nir_context *ctx,
+ unsigned mode,
+ LLVMValueRef sample_id)
+{
+ LLVMValueRef halfval = LLVMConstReal(ctx->ac.f32, 0.5f);
- }
+ /* fetch sample ID */
+ LLVMValueRef sample_pos = ctx->abi->load_sample_position(ctx->abi, sample_id);
- LLVMValueRef attrib_idx = ctx->ac.i32_0;
- while(deref_instr->deref_type != nir_deref_type_var) {
- if (deref_instr->deref_type == nir_deref_type_array) {
- unsigned array_size = glsl_count_attribute_slots(deref_instr->type, false);
+ LLVMValueRef src_c0 = LLVMBuildExtractElement(ctx->ac.builder, sample_pos, ctx->ac.i32_0, "");
+ src_c0 = LLVMBuildFSub(ctx->ac.builder, src_c0, halfval, "");
+ LLVMValueRef src_c1 = LLVMBuildExtractElement(ctx->ac.builder, sample_pos, ctx->ac.i32_1, "");
+ src_c1 = LLVMBuildFSub(ctx->ac.builder, src_c1, halfval, "");
+ LLVMValueRef coords[] = { src_c0, src_c1 };
+ LLVMValueRef offset = ac_build_gather_values(&ctx->ac, coords, 2);
- LLVMValueRef offset;
- if (nir_src_is_const(deref_instr->arr.index)) {
- offset = LLVMConstInt(ctx->ac.i32, array_size * nir_src_as_uint(deref_instr->arr.index), false);
- } else {
- LLVMValueRef indirect = get_src(ctx, deref_instr->arr.index);
+ return barycentric_offset(ctx, mode, offset);
+}
- offset = LLVMBuildMul(ctx->ac.builder, indirect,
- LLVMConstInt(ctx->ac.i32, array_size, false), "");
- }
- attrib_idx = LLVMBuildAdd(ctx->ac.builder, attrib_idx, offset, "");
- deref_instr = nir_src_as_deref(deref_instr->parent);
- } else if (deref_instr->deref_type == nir_deref_type_struct) {
- LLVMValueRef offset;
- unsigned sidx = deref_instr->strct.index;
- deref_instr = nir_src_as_deref(deref_instr->parent);
- offset = LLVMConstInt(ctx->ac.i32, glsl_get_struct_location_offset(deref_instr->type, sidx), false);
- attrib_idx = LLVMBuildAdd(ctx->ac.builder, attrib_idx, offset, "");
+static LLVMValueRef barycentric_sample(struct ac_nir_context *ctx,
+ unsigned mode)
+{
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_SAMPLE);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
+
+static LLVMValueRef load_interpolated_input(struct ac_nir_context *ctx,
+ LLVMValueRef interp_param,
+ unsigned index, unsigned comp_start,
+ unsigned num_components,
+ unsigned bitsize)
+{
+ LLVMValueRef attr_number = LLVMConstInt(ctx->ac.i32, index, false);
+
+ interp_param = LLVMBuildBitCast(ctx->ac.builder,
+ interp_param, ctx->ac.v2f32, "");
+ LLVMValueRef i = LLVMBuildExtractElement(
+ ctx->ac.builder, interp_param, ctx->ac.i32_0, "");
+ LLVMValueRef j = LLVMBuildExtractElement(
+ ctx->ac.builder, interp_param, ctx->ac.i32_1, "");
+
+ LLVMValueRef values[4];
+ assert(bitsize == 16 || bitsize == 32);
+ for (unsigned comp = 0; comp < num_components; comp++) {
+ LLVMValueRef llvm_chan = LLVMConstInt(ctx->ac.i32, comp_start + comp, false);
+ if (bitsize == 16) {
+ values[comp] = ac_build_fs_interp_f16(&ctx->ac, llvm_chan, attr_number,
+ ctx->abi->prim_mask, i, j);
} else {
- unreachable("Unsupported deref type");
+ values[comp] = ac_build_fs_interp(&ctx->ac, llvm_chan, attr_number,
+ ctx->abi->prim_mask, i, j);
}
-
}
- unsigned attrib_size = glsl_count_attribute_slots(var->type, false);
- for (chan = 0; chan < 4; chan++) {
- LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->ac.f32, attrib_size));
- LLVMValueRef llvm_chan = LLVMConstInt(ctx->ac.i32, chan, false);
-
- for (unsigned idx = 0; idx < attrib_size; ++idx) {
- LLVMValueRef v, attr_number;
-
- attr_number = LLVMConstInt(ctx->ac.i32, input_base + idx, false);
- if (interp_param) {
- interp_param = LLVMBuildBitCast(ctx->ac.builder,
- interp_param, ctx->ac.v2f32, "");
- LLVMValueRef i = LLVMBuildExtractElement(
- ctx->ac.builder, interp_param, ctx->ac.i32_0, "");
- LLVMValueRef j = LLVMBuildExtractElement(
- ctx->ac.builder, interp_param, ctx->ac.i32_1, "");
-
- v = ac_build_fs_interp(&ctx->ac, llvm_chan, attr_number,
- ctx->abi->prim_mask, i, j);
- } else {
- v = ac_build_fs_interp_mov(&ctx->ac, LLVMConstInt(ctx->ac.i32, 2, false),
- llvm_chan, attr_number, ctx->abi->prim_mask);
- }
+ return ac_to_integer(&ctx->ac, ac_build_gather_values(&ctx->ac, values, num_components));
+}
- gather = LLVMBuildInsertElement(ctx->ac.builder, gather, v,
- LLVMConstInt(ctx->ac.i32, idx, false), "");
- }
+static LLVMValueRef load_flat_input(struct ac_nir_context *ctx,
+ unsigned index, unsigned comp_start,
+ unsigned num_components,
+ unsigned bit_size)
+{
+ LLVMValueRef attr_number = LLVMConstInt(ctx->ac.i32, index, false);
- result[chan] = LLVMBuildExtractElement(ctx->ac.builder, gather, attrib_idx, "");
+ LLVMValueRef values[8];
+ /* Each component of a 64-bit value takes up two GL-level channels. */
+ unsigned channels =
+ bit_size == 64 ? num_components * 2 : num_components;
+
+ for (unsigned chan = 0; chan < channels; chan++) {
+ if (comp_start + chan > 4)
+ attr_number = LLVMConstInt(ctx->ac.i32, index + 1, false);
+ LLVMValueRef llvm_chan = LLVMConstInt(ctx->ac.i32, (comp_start + chan) % 4, false);
+ values[chan] = ac_build_fs_interp_mov(&ctx->ac,
+ LLVMConstInt(ctx->ac.i32, 2, false),
+ llvm_chan,
+ attr_number,
+ ctx->abi->prim_mask);
+ values[chan] = LLVMBuildBitCast(ctx->ac.builder, values[chan], ctx->ac.i32, "");
+ values[chan] = LLVMBuildTruncOrBitCast(ctx->ac.builder, values[chan],
+ bit_size == 16 ? ctx->ac.i16 : ctx->ac.i32, "");
+ }
+
+ LLVMValueRef result = ac_build_gather_values(&ctx->ac, values, channels);
+ if (bit_size == 64) {
+ LLVMTypeRef type = num_components == 1 ? ctx->ac.i64 :
+ LLVMVectorType(ctx->ac.i64, num_components);
+ result = LLVMBuildBitCast(ctx->ac.builder, result, type, "");
}
- return ac_build_varying_gather_values(&ctx->ac, result, instr->num_components,
- var->data.location_frac);
+ return result;
}
static void visit_intrinsic(struct ac_nir_context *ctx,
result = ctx->abi->view_index;
break;
case nir_intrinsic_load_invocation_id:
- if (ctx->stage == MESA_SHADER_TESS_CTRL)
+ if (ctx->stage == MESA_SHADER_TESS_CTRL) {
result = ac_unpack_param(&ctx->ac, ctx->abi->tcs_rel_ids, 8, 5);
- else
- result = ctx->abi->gs_invocation_id;
+ } else {
+ if (ctx->ac.chip_class >= GFX10) {
+ result = LLVMBuildAnd(ctx->ac.builder,
+ ctx->abi->gs_invocation_id,
+ LLVMConstInt(ctx->ac.i32, 127, 0), "");
+ } else {
+ result = ctx->abi->gs_invocation_id;
+ }
+ }
break;
case nir_intrinsic_load_primitive_id:
if (ctx->stage == MESA_SHADER_GEOMETRY) {
ac_build_gather_values(&ctx->ac, values, 4));
break;
}
+ case nir_intrinsic_load_layer_id:
+ result = ctx->abi->inputs[ac_llvm_reg_index_soa(VARYING_SLOT_LAYER, 0)];
+ break;
case nir_intrinsic_load_front_face:
result = ctx->abi->front_face;
break;
case nir_intrinsic_load_helper_invocation:
result = ac_build_load_helper_invocation(&ctx->ac);
break;
+ case nir_intrinsic_load_color0:
+ result = ctx->abi->color0;
+ break;
+ case nir_intrinsic_load_color1:
+ result = ctx->abi->color1;
+ break;
case nir_intrinsic_load_instance_id:
result = ctx->abi->instance_id;
break;
result = visit_var_atomic(ctx, instr, ptr, 1);
break;
}
- case nir_intrinsic_interp_deref_at_centroid:
- case nir_intrinsic_interp_deref_at_sample:
- case nir_intrinsic_interp_deref_at_offset:
- result = visit_interp(ctx, instr);
+ case nir_intrinsic_load_barycentric_pixel:
+ result = barycentric_center(ctx, nir_intrinsic_interp_mode(instr));
+ break;
+ case nir_intrinsic_load_barycentric_centroid:
+ result = barycentric_centroid(ctx, nir_intrinsic_interp_mode(instr));
+ break;
+ case nir_intrinsic_load_barycentric_sample:
+ result = barycentric_sample(ctx, nir_intrinsic_interp_mode(instr));
+ break;
+ case nir_intrinsic_load_barycentric_at_offset: {
+ LLVMValueRef offset = ac_to_float(&ctx->ac, get_src(ctx, instr->src[0]));
+ result = barycentric_offset(ctx, nir_intrinsic_interp_mode(instr), offset);
+ break;
+ }
+ case nir_intrinsic_load_barycentric_at_sample: {
+ LLVMValueRef sample_id = get_src(ctx, instr->src[0]);
+ result = barycentric_at_sample(ctx, nir_intrinsic_interp_mode(instr), sample_id);
+ break;
+ }
+ case nir_intrinsic_load_interpolated_input: {
+ /* We assume any indirect loads have been lowered away */
+ ASSERTED nir_const_value *offset = nir_src_as_const_value(instr->src[1]);
+ assert(offset);
+ assert(offset[0].i32 == 0);
+
+ LLVMValueRef interp_param = get_src(ctx, instr->src[0]);
+ unsigned index = nir_intrinsic_base(instr);
+ unsigned component = nir_intrinsic_component(instr);
+ result = load_interpolated_input(ctx, interp_param, index,
+ component,
+ instr->dest.ssa.num_components,
+ instr->dest.ssa.bit_size);
+ break;
+ }
+ case nir_intrinsic_load_input: {
+ /* We only lower inputs for fragment shaders ATM */
+ ASSERTED nir_const_value *offset = nir_src_as_const_value(instr->src[0]);
+ assert(offset);
+ assert(offset[0].i32 == 0);
+
+ unsigned index = nir_intrinsic_base(instr);
+ unsigned component = nir_intrinsic_component(instr);
+ result = load_flat_input(ctx, index, component,
+ instr->dest.ssa.num_components,
+ instr->dest.ssa.bit_size);
break;
+ }
case nir_intrinsic_emit_vertex:
ctx->abi->emit_vertex(ctx->abi, nir_intrinsic_stream_id(instr), ctx->abi->outputs);
break;
case nir_intrinsic_quad_swap_diagonal:
result = ac_build_quad_swizzle(&ctx->ac, get_src(ctx, instr->src[0]), 3, 2, 1 ,0);
break;
+ case nir_intrinsic_quad_swizzle_amd: {
+ uint32_t mask = nir_intrinsic_swizzle_mask(instr);
+ result = ac_build_quad_swizzle(&ctx->ac, get_src(ctx, instr->src[0]),
+ mask & 0x3, (mask >> 2) & 0x3,
+ (mask >> 4) & 0x3, (mask >> 6) & 0x3);
+ break;
+ }
+ case nir_intrinsic_masked_swizzle_amd: {
+ uint32_t mask = nir_intrinsic_swizzle_mask(instr);
+ result = ac_build_ds_swizzle(&ctx->ac, get_src(ctx, instr->src[0]), mask);
+ break;
+ }
+ case nir_intrinsic_write_invocation_amd:
+ result = ac_build_writelane(&ctx->ac, get_src(ctx, instr->src[0]),
+ get_src(ctx, instr->src[1]),
+ get_src(ctx, instr->src[2]));
+ break;
+ case nir_intrinsic_mbcnt_amd:
+ result = ac_build_mbcnt(&ctx->ac, get_src(ctx, instr->src[0]));
+ break;
+ case nir_intrinsic_load_scratch: {
+ LLVMValueRef offset = get_src(ctx, instr->src[0]);
+ LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->scratch,
+ offset);
+ LLVMTypeRef comp_type =
+ LLVMIntTypeInContext(ctx->ac.context, instr->dest.ssa.bit_size);
+ LLVMTypeRef vec_type =
+ instr->dest.ssa.num_components == 1 ? comp_type :
+ LLVMVectorType(comp_type, instr->dest.ssa.num_components);
+ unsigned addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
+ ptr = LLVMBuildBitCast(ctx->ac.builder, ptr,
+ LLVMPointerType(vec_type, addr_space), "");
+ result = LLVMBuildLoad(ctx->ac.builder, ptr, "");
+ break;
+ }
+ case nir_intrinsic_store_scratch: {
+ LLVMValueRef offset = get_src(ctx, instr->src[1]);
+ LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->scratch,
+ offset);
+ LLVMTypeRef comp_type =
+ LLVMIntTypeInContext(ctx->ac.context, instr->src[0].ssa->bit_size);
+ LLVMTypeRef vec_type =
+ instr->src[0].ssa->num_components == 1 ? comp_type :
+ LLVMVectorType(comp_type, instr->src[0].ssa->num_components);
+ unsigned addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
+ ptr = LLVMBuildBitCast(ctx->ac.builder, ptr,
+ LLVMPointerType(vec_type, addr_space), "");
+ LLVMBuildStore(ctx->ac.builder, get_src(ctx, instr->src[0]), ptr);
+ break;
+ }
default:
fprintf(stderr, "Unknown intrinsic: ");
nir_print_instr(&instr->instr, stderr);
LLVMValueRef ubo_index = ctx->abi->load_ubo(ctx->abi, ctx->ac.i32_0);
LLVMValueRef ret = ac_build_buffer_load(&ctx->ac, ubo_index, 1, NULL, offset,
- NULL, 0, false, false, true, true);
+ NULL, 0, 0, true, true);
return LLVMBuildBitCast(ctx->ac.builder, ret, ctx->ac.i32, "");
}
goto write_result;
}
- if (args.offset && instr->op != nir_texop_txf) {
+ if (args.offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
LLVMValueRef offset[3], pack;
for (unsigned chan = 0; chan < 3; ++chan)
offset[chan] = ctx->ac.i32_0;
/* TC-compatible HTILE on radeonsi promotes Z16 and Z24 to Z32_FLOAT,
* so the depth comparison value isn't clamped for Z16 and
- * Z24 anymore. Do it manually here.
+ * Z24 anymore. Do it manually here for GFX8-9; GFX10 has an explicitly
+ * clamped 32-bit float format.
*
* It's unnecessary if the original texture format was
* Z32_FLOAT, but we don't know that here.
*/
- if (args.compare && ctx->ac.chip_class >= GFX8 && ctx->abi->clamp_shadow_reference)
+ if (args.compare &&
+ ctx->ac.chip_class >= GFX8 &&
+ ctx->ac.chip_class <= GFX9 &&
+ ctx->abi->clamp_shadow_reference)
args.compare = ac_build_clamp(&ctx->ac, ac_to_float(&ctx->ac, args.compare));
/* pack derivatives */
break;
case GLSL_SAMPLER_DIM_1D:
num_src_deriv_channels = 1;
- if (ctx->ac.chip_class >= GFX9) {
+ if (ctx->ac.chip_class == GFX9) {
num_dest_deriv_channels = 2;
} else {
num_dest_deriv_channels = 1;
args.coords[2] = apply_round_slice(&ctx->ac, args.coords[2]);
}
- if (ctx->ac.chip_class >= GFX9 &&
+ if (ctx->ac.chip_class == GFX9 &&
instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
instr->op != nir_texop_lod) {
LLVMValueRef filler;
goto write_result;
}
- if (instr->sampler_dim == GLSL_SAMPLER_DIM_MS &&
+ if ((instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ||
+ instr->sampler_dim == GLSL_SAMPLER_DIM_MS) &&
instr->op != nir_texop_txs) {
unsigned sample_chan = instr->is_array ? 3 : 2;
args.coords[sample_chan] = adjust_sample_index_using_fmask(
args.coords[sample_chan], fmask_ptr);
}
- if (args.offset && instr->op == nir_texop_txf) {
+ if (args.offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
int num_offsets = instr->src[offset_src].src.ssa->num_components;
num_offsets = MIN2(num_offsets, instr->coord_components);
for (unsigned i = 0; i < num_offsets; ++i) {
args.offset = NULL;
}
- /* TODO TG4 support */
+ /* DMASK was repurposed for GATHER4. 4 components are always
+ * returned and DMASK works like a swizzle - it selects
+ * the component to fetch. The only valid DMASK values are
+ * 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
+ * (red,red,red,red) etc.) The ISA document doesn't mention
+ * this.
+ */
args.dmask = 0xf;
if (instr->op == nir_texop_tg4) {
if (instr->is_shadow)
LLVMValueRef z = LLVMBuildExtractElement(ctx->ac.builder, result, two, "");
z = LLVMBuildSDiv(ctx->ac.builder, z, six, "");
result = LLVMBuildInsertElement(ctx->ac.builder, result, z, two, "");
- } else if (ctx->ac.chip_class >= GFX9 &&
+ } else if (ctx->ac.chip_class == GFX9 &&
instr->op == nir_texop_txs &&
instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
instr->is_array) {
static void visit_block(struct ac_nir_context *ctx, nir_block *block)
{
- LLVMBasicBlockRef llvm_block = LLVMGetInsertBlock(ctx->ac.builder);
nir_foreach_instr(instr, block)
{
switch (instr->type) {
}
}
- _mesa_hash_table_insert(ctx->defs, block, llvm_block);
+ _mesa_hash_table_insert(ctx->defs, block,
+ LLVMGetInsertBlock(ctx->ac.builder));
}
static void visit_if(struct ac_nir_context *ctx, nir_if *if_stmt)
}
}
+static void
+setup_scratch(struct ac_nir_context *ctx,
+ struct nir_shader *shader)
+{
+ if (shader->scratch_size == 0)
+ return;
+
+ ctx->scratch = ac_build_alloca_undef(&ctx->ac,
+ LLVMArrayType(ctx->ac.i8, shader->scratch_size),
+ "scratch");
+}
+
static void
setup_shared(struct ac_nir_context *ctx,
struct nir_shader *nir)
ctx.abi = abi;
ctx.stage = nir->info.stage;
+ ctx.info = &nir->info;
ctx.main_function = LLVMGetBasicBlockParent(LLVMGetInsertBlock(ctx.ac.builder));
ctx.ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
setup_locals(&ctx, func);
+ setup_scratch(&ctx, nir);
if (gl_shader_stage_is_compute(nir->info.stage))
setup_shared(&ctx, nir);
void
ac_lower_indirect_derefs(struct nir_shader *nir, enum chip_class chip_class)
{
+ /* Lower large variables to scratch first so that we won't bloat the
+ * shader by generating large if ladders for them. We later lower
+ * scratch to alloca's, assuming LLVM won't generate VGPR indexing.
+ */
+ NIR_PASS_V(nir, nir_lower_vars_to_scratch,
+ nir_var_function_temp,
+ 256,
+ glsl_get_natural_size_align_bytes);
+
/* While it would be nice not to have this flag, we are constrained
- * by the reality that LLVM 5.0 doesn't have working VGPR indexing
- * on GFX9.
+ * by the reality that LLVM 9.0 has buggy VGPR indexing on GFX9.
*/
- bool llvm_has_working_vgpr_indexing = chip_class <= GFX8;
+ bool llvm_has_working_vgpr_indexing = chip_class != GFX9;
/* TODO: Indirect indexing of GS inputs is unimplemented.
*
projects / mesa.git / blobdiff