LLVMValueRef *ssa_defs;
+ LLVMValueRef scratch;
+
struct hash_table *defs;
struct hash_table *phis;
struct hash_table *vars;
{
switch (dim) {
case GLSL_SAMPLER_DIM_1D:
- if (ctx->chip_class >= GFX9)
+ if (ctx->chip_class == GFX9)
return is_array ? ac_image_2darray : ac_image_2d;
return is_array ? ac_image_1darray : ac_image_1d;
case GLSL_SAMPLER_DIM_2D:
ac_to_float(ctx, src0),
};
- MAYBE_UNUSED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
+ ASSERTED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
ac_get_elem_bits(ctx, result_type));
assert(length < sizeof(name));
return ac_build_intrinsic(ctx, name, result_type, params, 1, AC_FUNC_ATTR_READNONE);
ac_to_float(ctx, src1),
};
- MAYBE_UNUSED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
+ ASSERTED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
ac_get_elem_bits(ctx, result_type));
assert(length < sizeof(name));
return ac_build_intrinsic(ctx, name, result_type, params, 2, AC_FUNC_ATTR_READNONE);
ac_to_float(ctx, src2),
};
- MAYBE_UNUSED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
+ ASSERTED const int length = snprintf(name, sizeof(name), "%s.f%d", intrin,
ac_get_elem_bits(ctx, result_type));
assert(length < sizeof(name));
return ac_build_intrinsic(ctx, name, result_type, params, 3, AC_FUNC_ATTR_READNONE);
LLVMBuildXor(ctx->builder, insert, base, ""), ""), "");
}
-static LLVMValueRef emit_pack_half_2x16(struct ac_llvm_context *ctx,
- LLVMValueRef src0)
+static LLVMValueRef emit_pack_2x16(struct ac_llvm_context *ctx,
+ LLVMValueRef src0,
+ LLVMValueRef (*pack)(struct ac_llvm_context *ctx,
+ LLVMValueRef args[2]))
{
LLVMValueRef comp[2];
comp[0] = LLVMBuildExtractElement(ctx->builder, src0, ctx->i32_0, "");
comp[1] = LLVMBuildExtractElement(ctx->builder, src0, ctx->i32_1, "");
- return LLVMBuildBitCast(ctx->builder, ac_build_cvt_pkrtz_f16(ctx, comp),
- ctx->i32, "");
+ return LLVMBuildBitCast(ctx->builder, pack(ctx, comp), ctx->i32, "");
}
static LLVMValueRef emit_unpack_half_2x16(struct ac_llvm_context *ctx,
src_components = 1;
break;
case nir_op_pack_half_2x16:
+ case nir_op_pack_snorm_2x16:
+ case nir_op_pack_unorm_2x16:
src_components = 2;
break;
case nir_op_unpack_half_2x16:
result = emit_imul_high(&ctx->ac, src[0], src[1]);
break;
case nir_op_pack_half_2x16:
- result = emit_pack_half_2x16(&ctx->ac, src[0]);
+ result = emit_pack_2x16(&ctx->ac, src[0], ac_build_cvt_pkrtz_f16);
+ break;
+ case nir_op_pack_snorm_2x16:
+ result = emit_pack_2x16(&ctx->ac, src[0], ac_build_cvt_pknorm_i16);
+ break;
+ case nir_op_pack_unorm_2x16:
+ result = emit_pack_2x16(&ctx->ac, src[0], ac_build_cvt_pknorm_u16);
break;
case nir_op_unpack_half_2x16:
result = emit_unpack_half_2x16(&ctx->ac, src[0]);
args->coords[0],
ctx->ac.i32_0,
util_last_bit(mask),
- false, true);
+ 0, true);
} else {
return ac_build_buffer_load_format(&ctx->ac,
args->resource,
args->coords[0],
ctx->ac.i32_0,
util_last_bit(mask),
- false, true);
+ 0, true);
}
}
}
/* Fixup for GFX9 which allocates 1D textures as 2D. */
- if (instr->op == nir_texop_lod && ctx->ac.chip_class >= GFX9) {
+ if (instr->op == nir_texop_lod && ctx->ac.chip_class == GFX9) {
if ((args->dim == ac_image_2darray ||
args->dim == ac_image_2d) && !args->coords[1]) {
args->coords[1] = ctx->ac.i32_0;
cache_policy |= ac_glc;
}
+ if (access & ACCESS_STREAM_CACHE_POLICY)
+ cache_policy |= ac_slc;
+
return cache_policy;
}
if (num_bytes == 1) {
ac_build_tbuffer_store_byte(&ctx->ac, rsrc, data,
offset, ctx->ac.i32_0,
- cache_policy & ac_glc);
+ cache_policy);
} else if (num_bytes == 2) {
ac_build_tbuffer_store_short(&ctx->ac, rsrc, data,
offset, ctx->ac.i32_0,
- cache_policy & ac_glc);
+ cache_policy);
} else {
int num_channels = num_bytes / 4;
ac_build_buffer_store_dword(&ctx->ac, rsrc, data,
num_channels, offset,
ctx->ac.i32_0, 0,
- cache_policy & ac_glc,
- false, false);
+ cache_policy, false);
}
}
}
+static LLVMValueRef emit_ssbo_comp_swap_64(struct ac_nir_context *ctx,
+ LLVMValueRef descriptor,
+ LLVMValueRef offset,
+ LLVMValueRef compare,
+ LLVMValueRef exchange)
+{
+ LLVMBasicBlockRef start_block = NULL, then_block = NULL;
+ if (ctx->abi->robust_buffer_access) {
+ LLVMValueRef size = ac_llvm_extract_elem(&ctx->ac, descriptor, 2);
+
+ LLVMValueRef cond = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, offset, size, "");
+ start_block = LLVMGetInsertBlock(ctx->ac.builder);
+
+ ac_build_ifcc(&ctx->ac, cond, -1);
+
+ then_block = LLVMGetInsertBlock(ctx->ac.builder);
+ }
+
+ LLVMValueRef ptr_parts[2] = {
+ ac_llvm_extract_elem(&ctx->ac, descriptor, 0),
+ LLVMBuildAnd(ctx->ac.builder,
+ ac_llvm_extract_elem(&ctx->ac, descriptor, 1),
+ LLVMConstInt(ctx->ac.i32, 65535, 0), "")
+ };
+
+ ptr_parts[1] = LLVMBuildTrunc(ctx->ac.builder, ptr_parts[1], ctx->ac.i16, "");
+ ptr_parts[1] = LLVMBuildSExt(ctx->ac.builder, ptr_parts[1], ctx->ac.i32, "");
+
+ offset = LLVMBuildZExt(ctx->ac.builder, offset, ctx->ac.i64, "");
+
+ LLVMValueRef ptr = ac_build_gather_values(&ctx->ac, ptr_parts, 2);
+ ptr = LLVMBuildBitCast(ctx->ac.builder, ptr, ctx->ac.i64, "");
+ ptr = LLVMBuildAdd(ctx->ac.builder, ptr, offset, "");
+ ptr = LLVMBuildIntToPtr(ctx->ac.builder, ptr, LLVMPointerType(ctx->ac.i64, AC_ADDR_SPACE_GLOBAL), "");
+
+ LLVMValueRef result = ac_build_atomic_cmp_xchg(&ctx->ac, ptr, compare, exchange, "singlethread-one-as");
+ result = LLVMBuildExtractValue(ctx->ac.builder, result, 0, "");
+
+ if (ctx->abi->robust_buffer_access) {
+ ac_build_endif(&ctx->ac, -1);
+
+ LLVMBasicBlockRef incoming_blocks[2] = {
+ start_block,
+ then_block,
+ };
+
+ LLVMValueRef incoming_values[2] = {
+ LLVMConstInt(ctx->ac.i64, 0, 0),
+ result,
+ };
+ LLVMValueRef ret = LLVMBuildPhi(ctx->ac.builder, ctx->ac.i64, "");
+ LLVMAddIncoming(ret, incoming_values, incoming_blocks, 2);
+ return ret;
+ } else {
+ return result;
+ }
+}
+
static LLVMValueRef visit_atomic_ssbo(struct ac_nir_context *ctx,
const nir_intrinsic_instr *instr)
{
LLVMTypeRef return_type = LLVMTypeOf(get_src(ctx, instr->src[2]));
const char *op;
char name[64], type[8];
- LLVMValueRef params[6];
+ LLVMValueRef params[6], descriptor;
int arg_count = 0;
switch (instr->intrinsic) {
abort();
}
+ descriptor = ctx->abi->load_ssbo(ctx->abi,
+ get_src(ctx, instr->src[0]),
+ true);
+
+ if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap &&
+ return_type == ctx->ac.i64) {
+ return emit_ssbo_comp_swap_64(ctx, descriptor,
+ get_src(ctx, instr->src[1]),
+ get_src(ctx, instr->src[2]),
+ get_src(ctx, instr->src[3]));
+ }
if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) {
params[arg_count++] = ac_llvm_extract_elem(&ctx->ac, get_src(ctx, instr->src[3]), 0);
}
params[arg_count++] = ac_llvm_extract_elem(&ctx->ac, get_src(ctx, instr->src[2]), 0);
- params[arg_count++] = ctx->abi->load_ssbo(ctx->abi,
- get_src(ctx, instr->src[0]),
- true);
+ params[arg_count++] = descriptor;
if (HAVE_LLVM >= 0x900) {
/* XXX: The new raw/struct atomic intrinsics are buggy with
offset,
ctx->ac.i32_0,
immoffset,
- cache_policy & ac_glc);
+ cache_policy);
} else if (load_bytes == 2) {
ret = ac_build_tbuffer_load_short(&ctx->ac,
rsrc,
offset,
ctx->ac.i32_0,
immoffset,
- cache_policy & ac_glc);
+ cache_policy);
} else {
int num_channels = util_next_power_of_two(load_bytes) / 4;
bool can_speculate = access & ACCESS_CAN_REORDER;
ret = ac_build_buffer_load(&ctx->ac, rsrc, num_channels,
vindex, offset, immoffset, 0,
- cache_policy & ac_glc, 0,
- can_speculate, false);
+ cache_policy, can_speculate, false);
}
LLVMTypeRef byte_vec = LLVMVectorType(ctx->ac.i8, ac_get_type_size(LLVMTypeOf(ret)));
offset,
ctx->ac.i32_0,
immoffset,
- false);
+ 0);
} else {
assert(load_bytes == 2);
results[i] = ac_build_tbuffer_load_short(&ctx->ac,
offset,
ctx->ac.i32_0,
immoffset,
- false);
+ 0);
}
}
ret = ac_build_gather_values(&ctx->ac, results, num_components);
} else {
ret = ac_build_buffer_load(&ctx->ac, rsrc, num_components, NULL, offset,
- NULL, 0, false, false, true, true);
+ NULL, 0, 0, true, true);
ret = ac_trim_vector(&ctx->ac, ret, num_components);
}
return load_tess_varyings(ctx, instr, false);
}
+ if (ctx->stage == MESA_SHADER_FRAGMENT &&
+ var->data.fb_fetch_output &&
+ ctx->abi->emit_fbfetch)
+ return ctx->abi->emit_fbfetch(ctx->abi);
+
for (unsigned chan = comp; chan < ve + comp; chan++) {
if (indir_index) {
unsigned count = glsl_count_attribute_slots(
LLVMValueRef sample_index = ac_llvm_extract_elem(&ctx->ac, get_src(ctx, instr->src[2]), 0);
int count;
- bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS ||
- dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
+ ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS ||
+ dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
bool is_ms = (dim == GLSL_SAMPLER_DIM_MS ||
dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
- bool gfx9_1d = ctx->ac.chip_class >= GFX9 && dim == GLSL_SAMPLER_DIM_1D;
+ bool gfx9_1d = ctx->ac.chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
+ assert(!add_frag_pos && "Input attachments should be lowered by this point.");
count = image_type_to_components_count(dim, is_array);
if (is_ms && (instr->intrinsic == nir_intrinsic_image_deref_load ||
instr->intrinsic == nir_intrinsic_bindless_image_load)) {
LLVMValueRef fmask_load_address[3];
- int chan;
fmask_load_address[0] = LLVMBuildExtractElement(ctx->ac.builder, src0, masks[0], "");
fmask_load_address[1] = LLVMBuildExtractElement(ctx->ac.builder, src0, masks[1], "");
fmask_load_address[2] = LLVMBuildExtractElement(ctx->ac.builder, src0, masks[2], "");
else
fmask_load_address[2] = NULL;
- if (add_frag_pos) {
- for (chan = 0; chan < 2; ++chan)
- fmask_load_address[chan] =
- LLVMBuildAdd(ctx->ac.builder, fmask_load_address[chan],
- LLVMBuildFPToUI(ctx->ac.builder, ctx->abi->frag_pos[chan],
- ctx->ac.i32, ""), "");
- fmask_load_address[2] = ac_to_integer(&ctx->ac, ctx->abi->inputs[ac_llvm_reg_index_soa(VARYING_SLOT_LAYER, 0)]);
- }
+
sample_index = adjust_sample_index_using_fmask(&ctx->ac,
fmask_load_address[0],
fmask_load_address[1],
fmask_load_address[2],
sample_index,
get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr),
- AC_DESC_FMASK, &instr->instr, false, false));
+ AC_DESC_FMASK, &instr->instr, true, false));
}
if (count == 1 && !gfx9_1d) {
if (instr->src[1].ssa->num_components)
for (chan = 0; chan < count; ++chan) {
args->coords[chan] = ac_llvm_extract_elem(&ctx->ac, src0, chan);
}
- if (add_frag_pos) {
- for (chan = 0; chan < 2; ++chan) {
- args->coords[chan] = LLVMBuildAdd(
- ctx->ac.builder, args->coords[chan],
- LLVMBuildFPToUI(
- ctx->ac.builder, ctx->abi->frag_pos[chan],
- ctx->ac.i32, ""), "");
- }
- args->coords[2] = ac_to_integer(&ctx->ac,
- ctx->abi->inputs[ac_llvm_reg_index_soa(VARYING_SLOT_LAYER, 0)]);
- count++;
- }
if (gfx9_1d) {
if (is_array) {
bool can_speculate = access & ACCESS_CAN_REORDER;
res = ac_build_buffer_load_format(&ctx->ac, rsrc, vindex,
ctx->ac.i32_0, num_channels,
- !!(args.cache_policy & ac_glc),
+ args.cache_policy,
can_speculate);
res = ac_build_expand_to_vec4(&ctx->ac, res, num_channels);
ac_build_buffer_store_format(&ctx->ac, rsrc, src, vindex,
ctx->ac.i32_0, src_channels,
- args.cache_policy & ac_glc, false);
+ args.cache_policy);
} else {
args.opcode = ac_image_store;
args.data[0] = ac_to_float(&ctx->ac, get_src(ctx, instr->src[3]));
const char *atomic_name;
char intrinsic_name[64];
enum ac_atomic_op atomic_subop;
- MAYBE_UNUSED int length;
+ ASSERTED int length;
enum glsl_sampler_dim dim;
bool is_unsigned = false;
atomic_name = "cmpswap";
atomic_subop = 0; /* not used */
break;
+ case nir_intrinsic_bindless_image_atomic_inc_wrap:
+ case nir_intrinsic_image_deref_atomic_inc_wrap: {
+ atomic_name = "inc";
+ atomic_subop = ac_atomic_inc_wrap;
+ /* ATOMIC_INC instruction does:
+ * value = (value + 1) % (data + 1)
+ * but we want:
+ * value = (value + 1) % data
+ * So replace 'data' by 'data - 1'.
+ */
+ ctx->ssa_defs[instr->src[3].ssa->index] =
+ LLVMBuildSub(ctx->ac.builder,
+ ctx->ssa_defs[instr->src[3].ssa->index],
+ ctx->ac.i32_1, "");
+ break;
+ }
+ case nir_intrinsic_bindless_image_atomic_dec_wrap:
+ case nir_intrinsic_image_deref_atomic_dec_wrap:
+ atomic_name = "dec";
+ atomic_subop = ac_atomic_dec_wrap;
+ break;
default:
abort();
}
z = LLVMBuildSDiv(ctx->ac.builder, z, six, "");
res = LLVMBuildInsertElement(ctx->ac.builder, res, z, two, "");
}
- if (ctx->ac.chip_class >= GFX9 && dim == GLSL_SAMPLER_DIM_1D && is_array) {
+ if (ctx->ac.chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D && is_array) {
LLVMValueRef layers = LLVMBuildExtractElement(ctx->ac.builder, res, two, "");
res = LLVMBuildInsertElement(ctx->ac.builder, res, layers,
ctx->ac.i32_1, "");
static void emit_membar(struct ac_llvm_context *ac,
const nir_intrinsic_instr *instr)
{
- unsigned waitcnt = NOOP_WAITCNT;
+ unsigned wait_flags = 0;
switch (instr->intrinsic) {
case nir_intrinsic_memory_barrier:
case nir_intrinsic_group_memory_barrier:
- waitcnt &= VM_CNT & LGKM_CNT;
+ wait_flags = AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE;
break;
case nir_intrinsic_memory_barrier_atomic_counter:
case nir_intrinsic_memory_barrier_buffer:
case nir_intrinsic_memory_barrier_image:
- waitcnt &= VM_CNT;
+ wait_flags = AC_WAIT_VLOAD | AC_WAIT_VSTORE;
break;
case nir_intrinsic_memory_barrier_shared:
- waitcnt &= LGKM_CNT;
+ wait_flags = AC_WAIT_LGKM;
break;
default:
break;
}
- if (waitcnt != NOOP_WAITCNT)
- ac_build_waitcnt(ac, waitcnt);
+
+ ac_build_waitcnt(ac, wait_flags);
}
void ac_emit_barrier(struct ac_llvm_context *ac, gl_shader_stage stage)
* always fits into a single wave.
*/
if (ac->chip_class == GFX6 && stage == MESA_SHADER_TESS_CTRL) {
- ac_build_waitcnt(ac, LGKM_CNT & VM_CNT);
+ ac_build_waitcnt(ac, AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE);
return;
}
ac_build_s_barrier(ac);
visit_first_invocation(struct ac_nir_context *ctx)
{
LLVMValueRef active_set = ac_build_ballot(&ctx->ac, ctx->ac.i32_1);
+ const char *intr = ctx->ac.wave_size == 32 ? "llvm.cttz.i32" : "llvm.cttz.i64";
/* The second argument is whether cttz(0) should be defined, but we do not care. */
LLVMValueRef args[] = {active_set, ctx->ac.i1false};
- LLVMValueRef result = ac_build_intrinsic(&ctx->ac,
- "llvm.cttz.i64",
- ctx->ac.i64, args, 2,
+ LLVMValueRef result = ac_build_intrinsic(&ctx->ac, intr,
+ ctx->ac.iN_wavemask, args, 2,
AC_FUNC_ATTR_NOUNWIND |
AC_FUNC_ATTR_READNONE);
return ac_build_gather_values(&ctx->ac, values, 2);
}
-static LLVMValueRef visit_interp(struct ac_nir_context *ctx,
- const nir_intrinsic_instr *instr)
+static LLVMValueRef barycentric_center(struct ac_nir_context *ctx,
+ unsigned mode)
{
- LLVMValueRef result[4];
- LLVMValueRef interp_param;
- unsigned location;
- unsigned chan;
- LLVMValueRef src_c0 = NULL;
- LLVMValueRef src_c1 = NULL;
- LLVMValueRef src0 = NULL;
-
- nir_deref_instr *deref_instr = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
- nir_variable *var = nir_deref_instr_get_variable(deref_instr);
- int input_base = ctx->abi->fs_input_attr_indices[var->data.location - VARYING_SLOT_VAR0];
- switch (instr->intrinsic) {
- case nir_intrinsic_interp_deref_at_centroid:
- location = INTERP_CENTROID;
- break;
- case nir_intrinsic_interp_deref_at_sample:
- case nir_intrinsic_interp_deref_at_offset:
- location = INTERP_CENTER;
- src0 = get_src(ctx, instr->src[1]);
- break;
- default:
- break;
- }
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_CENTER);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- if (instr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
- src_c0 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, src0, ctx->ac.i32_0, ""));
- src_c1 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, src0, ctx->ac.i32_1, ""));
- } else if (instr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
- LLVMValueRef sample_position;
- LLVMValueRef halfval = LLVMConstReal(ctx->ac.f32, 0.5f);
+static LLVMValueRef barycentric_offset(struct ac_nir_context *ctx,
+ unsigned mode,
+ LLVMValueRef offset)
+{
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_CENTER);
+ LLVMValueRef src_c0 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, offset, ctx->ac.i32_0, ""));
+ LLVMValueRef src_c1 = ac_to_float(&ctx->ac, LLVMBuildExtractElement(ctx->ac.builder, offset, ctx->ac.i32_1, ""));
- /* fetch sample ID */
- sample_position = ctx->abi->load_sample_position(ctx->abi, src0);
+ LLVMValueRef ij_out[2];
+ LLVMValueRef ddxy_out = ac_build_ddxy_interp(&ctx->ac, interp_param);
- src_c0 = LLVMBuildExtractElement(ctx->ac.builder, sample_position, ctx->ac.i32_0, "");
- src_c0 = LLVMBuildFSub(ctx->ac.builder, src_c0, halfval, "");
- src_c1 = LLVMBuildExtractElement(ctx->ac.builder, sample_position, ctx->ac.i32_1, "");
- src_c1 = LLVMBuildFSub(ctx->ac.builder, src_c1, halfval, "");
- }
- interp_param = ctx->abi->lookup_interp_param(ctx->abi, var->data.interpolation, location);
+ /*
+ * take the I then J parameters, and the DDX/Y for it, and
+ * calculate the IJ inputs for the interpolator.
+ * temp1 = ddx * offset/sample.x + I;
+ * interp_param.I = ddy * offset/sample.y + temp1;
+ * temp1 = ddx * offset/sample.x + J;
+ * interp_param.J = ddy * offset/sample.y + temp1;
+ */
+ for (unsigned i = 0; i < 2; i++) {
+ LLVMValueRef ix_ll = LLVMConstInt(ctx->ac.i32, i, false);
+ LLVMValueRef iy_ll = LLVMConstInt(ctx->ac.i32, i + 2, false);
+ LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
+ ddxy_out, ix_ll, "");
+ LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
+ ddxy_out, iy_ll, "");
+ LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
+ interp_param, ix_ll, "");
+ LLVMValueRef temp1, temp2;
+
+ interp_el = LLVMBuildBitCast(ctx->ac.builder, interp_el,
+ ctx->ac.f32, "");
+
+ temp1 = ac_build_fmad(&ctx->ac, ddx_el, src_c0, interp_el);
+ temp2 = ac_build_fmad(&ctx->ac, ddy_el, src_c1, temp1);
+
+ ij_out[i] = LLVMBuildBitCast(ctx->ac.builder,
+ temp2, ctx->ac.i32, "");
+ }
+ interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- if (location == INTERP_CENTER) {
- LLVMValueRef ij_out[2];
- LLVMValueRef ddxy_out = ac_build_ddxy_interp(&ctx->ac, interp_param);
+static LLVMValueRef barycentric_centroid(struct ac_nir_context *ctx,
+ unsigned mode)
+{
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_CENTROID);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- /*
- * take the I then J parameters, and the DDX/Y for it, and
- * calculate the IJ inputs for the interpolator.
- * temp1 = ddx * offset/sample.x + I;
- * interp_param.I = ddy * offset/sample.y + temp1;
- * temp1 = ddx * offset/sample.x + J;
- * interp_param.J = ddy * offset/sample.y + temp1;
- */
- for (unsigned i = 0; i < 2; i++) {
- LLVMValueRef ix_ll = LLVMConstInt(ctx->ac.i32, i, false);
- LLVMValueRef iy_ll = LLVMConstInt(ctx->ac.i32, i + 2, false);
- LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
- ddxy_out, ix_ll, "");
- LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
- ddxy_out, iy_ll, "");
- LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
- interp_param, ix_ll, "");
- LLVMValueRef temp1, temp2;
-
- interp_el = LLVMBuildBitCast(ctx->ac.builder, interp_el,
- ctx->ac.f32, "");
-
- temp1 = ac_build_fmad(&ctx->ac, ddx_el, src_c0, interp_el);
- temp2 = ac_build_fmad(&ctx->ac, ddy_el, src_c1, temp1);
-
- ij_out[i] = LLVMBuildBitCast(ctx->ac.builder,
- temp2, ctx->ac.i32, "");
- }
- interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
+static LLVMValueRef barycentric_at_sample(struct ac_nir_context *ctx,
+ unsigned mode,
+ LLVMValueRef sample_id)
+{
+ if (ctx->abi->interp_at_sample_force_center)
+ return barycentric_center(ctx, mode);
- }
+ LLVMValueRef halfval = LLVMConstReal(ctx->ac.f32, 0.5f);
- LLVMValueRef attrib_idx = ctx->ac.i32_0;
- while(deref_instr->deref_type != nir_deref_type_var) {
- if (deref_instr->deref_type == nir_deref_type_array) {
- unsigned array_size = glsl_count_attribute_slots(deref_instr->type, false);
+ /* fetch sample ID */
+ LLVMValueRef sample_pos = ctx->abi->load_sample_position(ctx->abi, sample_id);
- LLVMValueRef offset;
- if (nir_src_is_const(deref_instr->arr.index)) {
- offset = LLVMConstInt(ctx->ac.i32, array_size * nir_src_as_uint(deref_instr->arr.index), false);
- } else {
- LLVMValueRef indirect = get_src(ctx, deref_instr->arr.index);
+ LLVMValueRef src_c0 = LLVMBuildExtractElement(ctx->ac.builder, sample_pos, ctx->ac.i32_0, "");
+ src_c0 = LLVMBuildFSub(ctx->ac.builder, src_c0, halfval, "");
+ LLVMValueRef src_c1 = LLVMBuildExtractElement(ctx->ac.builder, sample_pos, ctx->ac.i32_1, "");
+ src_c1 = LLVMBuildFSub(ctx->ac.builder, src_c1, halfval, "");
+ LLVMValueRef coords[] = { src_c0, src_c1 };
+ LLVMValueRef offset = ac_build_gather_values(&ctx->ac, coords, 2);
- offset = LLVMBuildMul(ctx->ac.builder, indirect,
- LLVMConstInt(ctx->ac.i32, array_size, false), "");
- }
+ return barycentric_offset(ctx, mode, offset);
+}
+
+
+static LLVMValueRef barycentric_sample(struct ac_nir_context *ctx,
+ unsigned mode)
+{
+ LLVMValueRef interp_param = ctx->abi->lookup_interp_param(ctx->abi, mode, INTERP_SAMPLE);
+ return LLVMBuildBitCast(ctx->ac.builder, interp_param, ctx->ac.v2i32, "");
+}
- attrib_idx = LLVMBuildAdd(ctx->ac.builder, attrib_idx, offset, "");
- deref_instr = nir_src_as_deref(deref_instr->parent);
- } else if (deref_instr->deref_type == nir_deref_type_struct) {
- LLVMValueRef offset;
- unsigned sidx = deref_instr->strct.index;
- deref_instr = nir_src_as_deref(deref_instr->parent);
- offset = LLVMConstInt(ctx->ac.i32, glsl_get_struct_location_offset(deref_instr->type, sidx), false);
- attrib_idx = LLVMBuildAdd(ctx->ac.builder, attrib_idx, offset, "");
+static LLVMValueRef load_interpolated_input(struct ac_nir_context *ctx,
+ LLVMValueRef interp_param,
+ unsigned index, unsigned comp_start,
+ unsigned num_components,
+ unsigned bitsize)
+{
+ LLVMValueRef attr_number = LLVMConstInt(ctx->ac.i32, index, false);
+
+ interp_param = LLVMBuildBitCast(ctx->ac.builder,
+ interp_param, ctx->ac.v2f32, "");
+ LLVMValueRef i = LLVMBuildExtractElement(
+ ctx->ac.builder, interp_param, ctx->ac.i32_0, "");
+ LLVMValueRef j = LLVMBuildExtractElement(
+ ctx->ac.builder, interp_param, ctx->ac.i32_1, "");
+
+ LLVMValueRef values[4];
+ assert(bitsize == 16 || bitsize == 32);
+ for (unsigned comp = 0; comp < num_components; comp++) {
+ LLVMValueRef llvm_chan = LLVMConstInt(ctx->ac.i32, comp_start + comp, false);
+ if (bitsize == 16) {
+ values[comp] = ac_build_fs_interp_f16(&ctx->ac, llvm_chan, attr_number,
+ ctx->abi->prim_mask, i, j);
} else {
- unreachable("Unsupported deref type");
+ values[comp] = ac_build_fs_interp(&ctx->ac, llvm_chan, attr_number,
+ ctx->abi->prim_mask, i, j);
}
-
}
- unsigned attrib_size = glsl_count_attribute_slots(var->type, false);
- for (chan = 0; chan < 4; chan++) {
- LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->ac.f32, attrib_size));
- LLVMValueRef llvm_chan = LLVMConstInt(ctx->ac.i32, chan, false);
-
- for (unsigned idx = 0; idx < attrib_size; ++idx) {
- LLVMValueRef v, attr_number;
-
- attr_number = LLVMConstInt(ctx->ac.i32, input_base + idx, false);
- if (interp_param) {
- interp_param = LLVMBuildBitCast(ctx->ac.builder,
- interp_param, ctx->ac.v2f32, "");
- LLVMValueRef i = LLVMBuildExtractElement(
- ctx->ac.builder, interp_param, ctx->ac.i32_0, "");
- LLVMValueRef j = LLVMBuildExtractElement(
- ctx->ac.builder, interp_param, ctx->ac.i32_1, "");
-
- v = ac_build_fs_interp(&ctx->ac, llvm_chan, attr_number,
- ctx->abi->prim_mask, i, j);
- } else {
- v = ac_build_fs_interp_mov(&ctx->ac, LLVMConstInt(ctx->ac.i32, 2, false),
- llvm_chan, attr_number, ctx->abi->prim_mask);
- }
+ return ac_to_integer(&ctx->ac, ac_build_gather_values(&ctx->ac, values, num_components));
+}
- gather = LLVMBuildInsertElement(ctx->ac.builder, gather, v,
- LLVMConstInt(ctx->ac.i32, idx, false), "");
- }
+static LLVMValueRef load_flat_input(struct ac_nir_context *ctx,
+ unsigned index, unsigned comp_start,
+ unsigned num_components,
+ unsigned bit_size)
+{
+ LLVMValueRef attr_number = LLVMConstInt(ctx->ac.i32, index, false);
- result[chan] = LLVMBuildExtractElement(ctx->ac.builder, gather, attrib_idx, "");
+ LLVMValueRef values[8];
+ /* Each component of a 64-bit value takes up two GL-level channels. */
+ unsigned channels =
+ bit_size == 64 ? num_components * 2 : num_components;
+
+ for (unsigned chan = 0; chan < channels; chan++) {
+ if (comp_start + chan > 4)
+ attr_number = LLVMConstInt(ctx->ac.i32, index + 1, false);
+ LLVMValueRef llvm_chan = LLVMConstInt(ctx->ac.i32, (comp_start + chan) % 4, false);
+ values[chan] = ac_build_fs_interp_mov(&ctx->ac,
+ LLVMConstInt(ctx->ac.i32, 2, false),
+ llvm_chan,
+ attr_number,
+ ctx->abi->prim_mask);
+ values[chan] = LLVMBuildBitCast(ctx->ac.builder, values[chan], ctx->ac.i32, "");
+ values[chan] = LLVMBuildTruncOrBitCast(ctx->ac.builder, values[chan],
+ bit_size == 16 ? ctx->ac.i16 : ctx->ac.i32, "");
+ }
+
+ LLVMValueRef result = ac_build_gather_values(&ctx->ac, values, channels);
+ if (bit_size == 64) {
+ LLVMTypeRef type = num_components == 1 ? ctx->ac.i64 :
+ LLVMVectorType(ctx->ac.i64, num_components);
+ result = LLVMBuildBitCast(ctx->ac.builder, result, type, "");
}
- return ac_build_varying_gather_values(&ctx->ac, result, instr->num_components,
- var->data.location_frac);
+ return result;
}
static void visit_intrinsic(struct ac_nir_context *ctx,
switch (instr->intrinsic) {
case nir_intrinsic_ballot:
result = ac_build_ballot(&ctx->ac, get_src(ctx, instr->src[0]));
+ if (ctx->ac.ballot_mask_bits > ctx->ac.wave_size)
+ result = LLVMBuildZExt(ctx->ac.builder, result, ctx->ac.iN_ballotmask, "");
break;
case nir_intrinsic_read_invocation:
result = ac_build_readlane(&ctx->ac, get_src(ctx, instr->src[0]),
result = ctx->abi->view_index;
break;
case nir_intrinsic_load_invocation_id:
- if (ctx->stage == MESA_SHADER_TESS_CTRL)
+ if (ctx->stage == MESA_SHADER_TESS_CTRL) {
result = ac_unpack_param(&ctx->ac, ctx->abi->tcs_rel_ids, 8, 5);
- else
- result = ctx->abi->gs_invocation_id;
+ } else {
+ if (ctx->ac.chip_class >= GFX10) {
+ result = LLVMBuildAnd(ctx->ac.builder,
+ ctx->abi->gs_invocation_id,
+ LLVMConstInt(ctx->ac.i32, 127, 0), "");
+ } else {
+ result = ctx->abi->gs_invocation_id;
+ }
+ }
break;
case nir_intrinsic_load_primitive_id:
if (ctx->stage == MESA_SHADER_GEOMETRY) {
ac_build_gather_values(&ctx->ac, values, 4));
break;
}
+ case nir_intrinsic_load_layer_id:
+ result = ctx->abi->inputs[ac_llvm_reg_index_soa(VARYING_SLOT_LAYER, 0)];
+ break;
case nir_intrinsic_load_front_face:
result = ctx->abi->front_face;
break;
case nir_intrinsic_load_helper_invocation:
result = ac_build_load_helper_invocation(&ctx->ac);
break;
+ case nir_intrinsic_load_color0:
+ result = ctx->abi->color0;
+ break;
+ case nir_intrinsic_load_color1:
+ result = ctx->abi->color1;
+ break;
+ case nir_intrinsic_load_user_data_amd:
+ assert(LLVMTypeOf(ctx->abi->user_data) == ctx->ac.v4i32);
+ result = ctx->abi->user_data;
+ break;
case nir_intrinsic_load_instance_id:
result = ctx->abi->instance_id;
break;
case nir_intrinsic_bindless_image_atomic_xor:
case nir_intrinsic_bindless_image_atomic_exchange:
case nir_intrinsic_bindless_image_atomic_comp_swap:
+ case nir_intrinsic_bindless_image_atomic_inc_wrap:
+ case nir_intrinsic_bindless_image_atomic_dec_wrap:
result = visit_image_atomic(ctx, instr, true);
break;
case nir_intrinsic_image_deref_atomic_add:
case nir_intrinsic_image_deref_atomic_xor:
case nir_intrinsic_image_deref_atomic_exchange:
case nir_intrinsic_image_deref_atomic_comp_swap:
+ case nir_intrinsic_image_deref_atomic_inc_wrap:
+ case nir_intrinsic_image_deref_atomic_dec_wrap:
result = visit_image_atomic(ctx, instr, false);
break;
case nir_intrinsic_bindless_image_size:
result = visit_var_atomic(ctx, instr, ptr, 1);
break;
}
- case nir_intrinsic_interp_deref_at_centroid:
- case nir_intrinsic_interp_deref_at_sample:
- case nir_intrinsic_interp_deref_at_offset:
- result = visit_interp(ctx, instr);
+ case nir_intrinsic_load_barycentric_pixel:
+ result = barycentric_center(ctx, nir_intrinsic_interp_mode(instr));
+ break;
+ case nir_intrinsic_load_barycentric_centroid:
+ result = barycentric_centroid(ctx, nir_intrinsic_interp_mode(instr));
+ break;
+ case nir_intrinsic_load_barycentric_sample:
+ result = barycentric_sample(ctx, nir_intrinsic_interp_mode(instr));
+ break;
+ case nir_intrinsic_load_barycentric_at_offset: {
+ LLVMValueRef offset = ac_to_float(&ctx->ac, get_src(ctx, instr->src[0]));
+ result = barycentric_offset(ctx, nir_intrinsic_interp_mode(instr), offset);
+ break;
+ }
+ case nir_intrinsic_load_barycentric_at_sample: {
+ LLVMValueRef sample_id = get_src(ctx, instr->src[0]);
+ result = barycentric_at_sample(ctx, nir_intrinsic_interp_mode(instr), sample_id);
break;
+ }
+ case nir_intrinsic_load_interpolated_input: {
+ /* We assume any indirect loads have been lowered away */
+ ASSERTED nir_const_value *offset = nir_src_as_const_value(instr->src[1]);
+ assert(offset);
+ assert(offset[0].i32 == 0);
+
+ LLVMValueRef interp_param = get_src(ctx, instr->src[0]);
+ unsigned index = nir_intrinsic_base(instr);
+ unsigned component = nir_intrinsic_component(instr);
+ result = load_interpolated_input(ctx, interp_param, index,
+ component,
+ instr->dest.ssa.num_components,
+ instr->dest.ssa.bit_size);
+ break;
+ }
+ case nir_intrinsic_load_input: {
+ /* We only lower inputs for fragment shaders ATM */
+ ASSERTED nir_const_value *offset = nir_src_as_const_value(instr->src[0]);
+ assert(offset);
+ assert(offset[0].i32 == 0);
+
+ unsigned index = nir_intrinsic_base(instr);
+ unsigned component = nir_intrinsic_component(instr);
+ result = load_flat_input(ctx, index, component,
+ instr->dest.ssa.num_components,
+ instr->dest.ssa.bit_size);
+ break;
+ }
case nir_intrinsic_emit_vertex:
ctx->abi->emit_vertex(ctx->abi, nir_intrinsic_stream_id(instr), ctx->abi->outputs);
break;
result = ctx->abi->load_tess_coord(ctx->abi);
break;
case nir_intrinsic_load_tess_level_outer:
- result = ctx->abi->load_tess_level(ctx->abi, VARYING_SLOT_TESS_LEVEL_OUTER);
+ result = ctx->abi->load_tess_level(ctx->abi, VARYING_SLOT_TESS_LEVEL_OUTER, false);
break;
case nir_intrinsic_load_tess_level_inner:
- result = ctx->abi->load_tess_level(ctx->abi, VARYING_SLOT_TESS_LEVEL_INNER);
+ result = ctx->abi->load_tess_level(ctx->abi, VARYING_SLOT_TESS_LEVEL_INNER, false);
+ break;
+ case nir_intrinsic_load_tess_level_outer_default:
+ result = ctx->abi->load_tess_level(ctx->abi, VARYING_SLOT_TESS_LEVEL_OUTER, true);
+ break;
+ case nir_intrinsic_load_tess_level_inner_default:
+ result = ctx->abi->load_tess_level(ctx->abi, VARYING_SLOT_TESS_LEVEL_INNER, true);
break;
case nir_intrinsic_load_patch_vertices_in:
result = ctx->abi->load_patch_vertices_in(ctx->abi);
case nir_intrinsic_mbcnt_amd:
result = ac_build_mbcnt(&ctx->ac, get_src(ctx, instr->src[0]));
break;
+ case nir_intrinsic_load_scratch: {
+ LLVMValueRef offset = get_src(ctx, instr->src[0]);
+ LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->scratch,
+ offset);
+ LLVMTypeRef comp_type =
+ LLVMIntTypeInContext(ctx->ac.context, instr->dest.ssa.bit_size);
+ LLVMTypeRef vec_type =
+ instr->dest.ssa.num_components == 1 ? comp_type :
+ LLVMVectorType(comp_type, instr->dest.ssa.num_components);
+ unsigned addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
+ ptr = LLVMBuildBitCast(ctx->ac.builder, ptr,
+ LLVMPointerType(vec_type, addr_space), "");
+ result = LLVMBuildLoad(ctx->ac.builder, ptr, "");
+ break;
+ }
+ case nir_intrinsic_store_scratch: {
+ LLVMValueRef offset = get_src(ctx, instr->src[1]);
+ LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->scratch,
+ offset);
+ LLVMTypeRef comp_type =
+ LLVMIntTypeInContext(ctx->ac.context, instr->src[0].ssa->bit_size);
+ unsigned addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
+ ptr = LLVMBuildBitCast(ctx->ac.builder, ptr,
+ LLVMPointerType(comp_type, addr_space), "");
+ LLVMValueRef src = get_src(ctx, instr->src[0]);
+ unsigned wrmask = nir_intrinsic_write_mask(instr);
+ while (wrmask) {
+ int start, count;
+ u_bit_scan_consecutive_range(&wrmask, &start, &count);
+
+ LLVMValueRef offset = LLVMConstInt(ctx->ac.i32, start, false);
+ LLVMValueRef offset_ptr = LLVMBuildGEP(ctx->ac.builder, ptr, &offset, 1, "");
+ LLVMTypeRef vec_type =
+ count == 1 ? comp_type : LLVMVectorType(comp_type, count);
+ offset_ptr = LLVMBuildBitCast(ctx->ac.builder,
+ offset_ptr,
+ LLVMPointerType(vec_type, addr_space),
+ "");
+ LLVMValueRef offset_src =
+ ac_extract_components(&ctx->ac, src, start, count);
+ LLVMBuildStore(ctx->ac.builder, offset_src, offset_ptr);
+ }
+ break;
+ }
default:
fprintf(stderr, "Unknown intrinsic: ");
nir_print_instr(&instr->instr, stderr);
LLVMValueRef ubo_index = ctx->abi->load_ubo(ctx->abi, ctx->ac.i32_0);
LLVMValueRef ret = ac_build_buffer_load(&ctx->ac, ubo_index, 1, NULL, offset,
- NULL, 0, false, false, true, true);
+ NULL, 0, 0, true, true);
return LLVMBuildBitCast(ctx->ac.builder, ret, ctx->ac.i32, "");
}
goto write_result;
}
- if (args.offset && instr->op != nir_texop_txf) {
+ if (args.offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
LLVMValueRef offset[3], pack;
for (unsigned chan = 0; chan < 3; ++chan)
offset[chan] = ctx->ac.i32_0;
/* TC-compatible HTILE on radeonsi promotes Z16 and Z24 to Z32_FLOAT,
* so the depth comparison value isn't clamped for Z16 and
- * Z24 anymore. Do it manually here.
+ * Z24 anymore. Do it manually here for GFX8-9; GFX10 has an explicitly
+ * clamped 32-bit float format.
*
* It's unnecessary if the original texture format was
* Z32_FLOAT, but we don't know that here.
*/
- if (args.compare && ctx->ac.chip_class >= GFX8 && ctx->abi->clamp_shadow_reference)
+ if (args.compare &&
+ ctx->ac.chip_class >= GFX8 &&
+ ctx->ac.chip_class <= GFX9 &&
+ ctx->abi->clamp_shadow_reference)
args.compare = ac_build_clamp(&ctx->ac, ac_to_float(&ctx->ac, args.compare));
/* pack derivatives */
break;
case GLSL_SAMPLER_DIM_1D:
num_src_deriv_channels = 1;
- if (ctx->ac.chip_class >= GFX9) {
+ if (ctx->ac.chip_class == GFX9) {
num_dest_deriv_channels = 2;
} else {
num_dest_deriv_channels = 1;
args.coords[2] = apply_round_slice(&ctx->ac, args.coords[2]);
}
- if (ctx->ac.chip_class >= GFX9 &&
+ if (ctx->ac.chip_class == GFX9 &&
instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
instr->op != nir_texop_lod) {
LLVMValueRef filler;
goto write_result;
}
- if (instr->sampler_dim == GLSL_SAMPLER_DIM_MS &&
+ if ((instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ||
+ instr->sampler_dim == GLSL_SAMPLER_DIM_MS) &&
instr->op != nir_texop_txs) {
unsigned sample_chan = instr->is_array ? 3 : 2;
args.coords[sample_chan] = adjust_sample_index_using_fmask(
args.coords[sample_chan], fmask_ptr);
}
- if (args.offset && instr->op == nir_texop_txf) {
+ if (args.offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
int num_offsets = instr->src[offset_src].src.ssa->num_components;
num_offsets = MIN2(num_offsets, instr->coord_components);
for (unsigned i = 0; i < num_offsets; ++i) {
LLVMValueRef z = LLVMBuildExtractElement(ctx->ac.builder, result, two, "");
z = LLVMBuildSDiv(ctx->ac.builder, z, six, "");
result = LLVMBuildInsertElement(ctx->ac.builder, result, z, two, "");
- } else if (ctx->ac.chip_class >= GFX9 &&
+ } else if (ctx->ac.chip_class == GFX9 &&
instr->op == nir_texop_txs &&
instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
instr->is_array) {
static void visit_block(struct ac_nir_context *ctx, nir_block *block)
{
- LLVMBasicBlockRef llvm_block = LLVMGetInsertBlock(ctx->ac.builder);
nir_foreach_instr(instr, block)
{
switch (instr->type) {
}
}
- _mesa_hash_table_insert(ctx->defs, block, llvm_block);
+ _mesa_hash_table_insert(ctx->defs, block,
+ LLVMGetInsertBlock(ctx->ac.builder));
}
static void visit_if(struct ac_nir_context *ctx, nir_if *if_stmt)
}
}
+static void
+setup_scratch(struct ac_nir_context *ctx,
+ struct nir_shader *shader)
+{
+ if (shader->scratch_size == 0)
+ return;
+
+ ctx->scratch = ac_build_alloca_undef(&ctx->ac,
+ LLVMArrayType(ctx->ac.i8, shader->scratch_size),
+ "scratch");
+}
+
static void
setup_shared(struct ac_nir_context *ctx,
struct nir_shader *nir)
ctx.ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
setup_locals(&ctx, func);
+ setup_scratch(&ctx, nir);
if (gl_shader_stage_is_compute(nir->info.stage))
setup_shared(&ctx, nir);
void
ac_lower_indirect_derefs(struct nir_shader *nir, enum chip_class chip_class)
{
+ /* Lower large variables to scratch first so that we won't bloat the
+ * shader by generating large if ladders for them. We later lower
+ * scratch to alloca's, assuming LLVM won't generate VGPR indexing.
+ */
+ NIR_PASS_V(nir, nir_lower_vars_to_scratch,
+ nir_var_function_temp,
+ 256,
+ glsl_get_natural_size_align_bytes);
+
/* While it would be nice not to have this flag, we are constrained
- * by the reality that LLVM 5.0 doesn't have working VGPR indexing
- * on GFX9.
+ * by the reality that LLVM 9.0 has buggy VGPR indexing on GFX9.
*/
- bool llvm_has_working_vgpr_indexing = chip_class <= GFX8;
+ bool llvm_has_working_vgpr_indexing = chip_class != GFX9;
/* TODO: Indirect indexing of GS inputs is unimplemented.
*