dst[i] = ir3_MOV(b, src[i], TYPE_U32);
}
- put_dst(ctx, &alu->dest.dest);
+ ir3_put_dst(ctx, &alu->dest.dest);
return;
}
}
}
- put_dst(ctx, &alu->dest.dest);
+ ir3_put_dst(ctx, &alu->dest.dest);
return;
}
dst[0] = ir3_SEL_B32(b, src[1], 0, cond, 0, src[2], 0);
break;
}
- case nir_op_bit_count:
- dst[0] = ir3_CBITS_B(b, src[0], 0);
+ case nir_op_bit_count: {
+ // TODO, we need to do this 16b at a time on a5xx+a6xx.. need to
+ // double check on earlier gen's. Once half-precision support is
+ // in place, this should probably move to a NIR lowering pass:
+ struct ir3_instruction *hi, *lo;
+
+ hi = ir3_COV(b, ir3_SHR_B(b, src[0], 0, create_immed(b, 16), 0),
+ TYPE_U32, TYPE_U16);
+ lo = ir3_COV(b, src[0], TYPE_U32, TYPE_U16);
+
+ hi = ir3_CBITS_B(b, hi, 0);
+ lo = ir3_CBITS_B(b, lo, 0);
+
+ // TODO maybe the builders should default to making dst half-precision
+ // if the src's were half precision, to make this less awkward.. otoh
+ // we should probably just do this lowering in NIR.
+ hi->regs[0]->flags |= IR3_REG_HALF;
+ lo->regs[0]->flags |= IR3_REG_HALF;
+
+ dst[0] = ir3_ADD_S(b, hi, 0, lo, 0);
+ dst[0]->regs[0]->flags |= IR3_REG_HALF;
+ dst[0] = ir3_COV(b, dst[0], TYPE_U16, TYPE_U32);
break;
+ }
case nir_op_ifind_msb: {
struct ir3_instruction *cmp;
dst[0] = ir3_CLZ_S(b, src[0], 0);
break;
}
- put_dst(ctx, &alu->dest.dest);
+ ir3_put_dst(ctx, &alu->dest.dest);
}
/* handles direct/indirect UBO reads: */
struct ir3_block *b = ctx->block;
struct ir3_instruction *base_lo, *base_hi, *addr, *src0, *src1;
nir_const_value *const_offset;
- /* UBO addresses are the first driver params: */
- unsigned ubo = regid(ctx->so->constbase.ubo, 0);
+ /* UBO addresses are the first driver params, but subtract 2 here to
+ * account for nir_lower_uniforms_to_ubo rebasing the UBOs such that UBO 0
+ * is the uniforms: */
+ unsigned ubo = regid(ctx->so->constbase.ubo, 0) - 2;
const unsigned ptrsz = ir3_pointer_size(ctx);
int off = 0;
base_lo = create_uniform(b, ubo + (src0->regs[1]->iim_val * ptrsz));
base_hi = create_uniform(b, ubo + (src0->regs[1]->iim_val * ptrsz) + 1);
} else {
- base_lo = create_uniform_indirect(b, ubo, ir3_get_addr(ctx, src0, 4));
- base_hi = create_uniform_indirect(b, ubo + 1, ir3_get_addr(ctx, src0, 4));
+ base_lo = create_uniform_indirect(b, ubo, ir3_get_addr(ctx, src0, ptrsz));
+ base_hi = create_uniform_indirect(b, ubo + 1, ir3_get_addr(ctx, src0, ptrsz));
}
/* note: on 32bit gpu's base_hi is ignored and DCE'd */
return atomic;
}
+/* TODO handle actual indirect/dynamic case.. which is going to be weird
+ * to handle with the image_mapping table..
+ */
+static struct ir3_instruction *
+get_image_samp_tex_src(struct ir3_context *ctx, nir_intrinsic_instr *intr)
+{
+ unsigned slot = ir3_get_image_slot(nir_src_as_deref(intr->src[0]));
+ unsigned tex_idx = ir3_image_to_tex(&ctx->so->image_mapping, slot);
+ struct ir3_instruction *texture, *sampler;
+
+ texture = create_immed_typed(ctx->block, tex_idx, TYPE_U16);
+ sampler = create_immed_typed(ctx->block, tex_idx, TYPE_U16);
+
+ return ir3_create_collect(ctx, (struct ir3_instruction*[]){
+ sampler,
+ texture,
+ }, 2);
+}
+
/* src[] = { deref, coord, sample_index }. const_index[] = {} */
static void
emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
{
struct ir3_block *b = ctx->block;
const nir_variable *var = nir_intrinsic_get_var(intr, 0);
+ struct ir3_instruction *samp_tex = get_image_samp_tex_src(ctx, intr);
struct ir3_instruction *sam;
struct ir3_instruction * const *src0 = ir3_get_src(ctx, &intr->src[1]);
struct ir3_instruction *coords[4];
unsigned flags, ncoords = ir3_get_image_coords(var, &flags);
- unsigned slot = ir3_get_image_slot(nir_src_as_deref(intr->src[0]));
- unsigned tex_idx = ir3_image_to_tex(&ctx->so->image_mapping, slot);
type_t type = ir3_get_image_type(var);
/* hmm, this seems a bit odd, but it is what blob does and (at least
coords[ncoords++] = create_immed(b, 0);
sam = ir3_SAM(b, OPC_ISAM, type, 0b1111, flags,
- tex_idx, tex_idx, ir3_create_collect(ctx, coords, ncoords), NULL);
+ samp_tex, ir3_create_collect(ctx, coords, ncoords), NULL);
sam->barrier_class = IR3_BARRIER_IMAGE_R;
sam->barrier_conflict = IR3_BARRIER_IMAGE_W;
{
struct ir3_block *b = ctx->block;
const nir_variable *var = nir_intrinsic_get_var(intr, 0);
- unsigned slot = ir3_get_image_slot(nir_src_as_deref(intr->src[0]));
- unsigned tex_idx = ir3_image_to_tex(&ctx->so->image_mapping, slot);
+ struct ir3_instruction *samp_tex = get_image_samp_tex_src(ctx, intr);
struct ir3_instruction *sam, *lod;
unsigned flags, ncoords = ir3_get_image_coords(var, &flags);
lod = create_immed(b, 0);
sam = ir3_SAM(b, OPC_GETSIZE, TYPE_U32, 0b1111, flags,
- tex_idx, tex_idx, lod, NULL);
+ samp_tex, lod, NULL);
/* Array size actually ends up in .w rather than .z. This doesn't
* matter for miplevel 0, but for higher mips the value in z is
if (const_offset) {
idx += const_offset->u32[0];
for (int i = 0; i < intr->num_components; i++) {
- unsigned n = idx * 4 + i;
- dst[i] = create_uniform(b, n);
+ dst[i] = create_uniform(b, idx + i);
}
} else {
src = ir3_get_src(ctx, &intr->src[0]);
for (int i = 0; i < intr->num_components; i++) {
- int n = idx * 4 + i;
- dst[i] = create_uniform_indirect(b, n,
- ir3_get_addr(ctx, src[0], 4));
+ dst[i] = create_uniform_indirect(b, idx + i,
+ ir3_get_addr(ctx, src[0], 1));
}
/* NOTE: if relative addressing is used, we set
* constlen in the compiler (to worst-case value)
}
}
break;
- case nir_intrinsic_load_ssbo:
+ /* All SSBO intrinsics should have been lowered by 'lower_io_offsets'
+ * pass and replaced by an ir3-specifc version that adds the
+ * dword-offset in the last source.
+ */
+ case nir_intrinsic_load_ssbo_ir3:
ctx->funcs->emit_intrinsic_load_ssbo(ctx, intr, dst);
break;
- case nir_intrinsic_store_ssbo:
+ case nir_intrinsic_store_ssbo_ir3:
+ if ((ctx->so->type == MESA_SHADER_FRAGMENT) &&
+ !ctx->s->info.fs.early_fragment_tests)
+ ctx->so->no_earlyz = true;
ctx->funcs->emit_intrinsic_store_ssbo(ctx, intr);
break;
case nir_intrinsic_get_buffer_size:
emit_intrinsic_ssbo_size(ctx, intr, dst);
break;
- case nir_intrinsic_ssbo_atomic_add:
- case nir_intrinsic_ssbo_atomic_imin:
- case nir_intrinsic_ssbo_atomic_umin:
- case nir_intrinsic_ssbo_atomic_imax:
- case nir_intrinsic_ssbo_atomic_umax:
- case nir_intrinsic_ssbo_atomic_and:
- case nir_intrinsic_ssbo_atomic_or:
- case nir_intrinsic_ssbo_atomic_xor:
- case nir_intrinsic_ssbo_atomic_exchange:
- case nir_intrinsic_ssbo_atomic_comp_swap:
+ case nir_intrinsic_ssbo_atomic_add_ir3:
+ case nir_intrinsic_ssbo_atomic_imin_ir3:
+ case nir_intrinsic_ssbo_atomic_umin_ir3:
+ case nir_intrinsic_ssbo_atomic_imax_ir3:
+ case nir_intrinsic_ssbo_atomic_umax_ir3:
+ case nir_intrinsic_ssbo_atomic_and_ir3:
+ case nir_intrinsic_ssbo_atomic_or_ir3:
+ case nir_intrinsic_ssbo_atomic_xor_ir3:
+ case nir_intrinsic_ssbo_atomic_exchange_ir3:
+ case nir_intrinsic_ssbo_atomic_comp_swap_ir3:
+ if ((ctx->so->type == MESA_SHADER_FRAGMENT) &&
+ !ctx->s->info.fs.early_fragment_tests)
+ ctx->so->no_earlyz = true;
dst[0] = ctx->funcs->emit_intrinsic_atomic_ssbo(ctx, intr);
break;
case nir_intrinsic_load_shared:
emit_intrinsic_load_image(ctx, intr, dst);
break;
case nir_intrinsic_image_deref_store:
+ if ((ctx->so->type == MESA_SHADER_FRAGMENT) &&
+ !ctx->s->info.fs.early_fragment_tests)
+ ctx->so->no_earlyz = true;
ctx->funcs->emit_intrinsic_store_image(ctx, intr);
break;
case nir_intrinsic_image_deref_size:
case nir_intrinsic_image_deref_atomic_xor:
case nir_intrinsic_image_deref_atomic_exchange:
case nir_intrinsic_image_deref_atomic_comp_swap:
+ if ((ctx->so->type == MESA_SHADER_FRAGMENT) &&
+ !ctx->s->info.fs.early_fragment_tests)
+ ctx->so->no_earlyz = true;
dst[0] = ctx->funcs->emit_intrinsic_atomic_image(ctx, intr);
break;
case nir_intrinsic_barrier:
array_insert(ctx->ir, ctx->ir->predicates, kill);
array_insert(b, b->keeps, kill);
- ctx->so->has_kill = true;
+ ctx->so->no_earlyz = true;
break;
}
}
if (info->has_dest)
- put_dst(ctx, &intr->dest);
+ ir3_put_dst(ctx, &intr->dest);
}
static void
*coordsp = coords;
}
+/* Gets the sampler/texture idx as a hvec2. Which could either be dynamic
+ * or immediate (in which case it will get lowered later to a non .s2en
+ * version of the tex instruction which encode tex/samp as immediates:
+ */
+static struct ir3_instruction *
+get_tex_samp_tex_src(struct ir3_context *ctx, nir_tex_instr *tex)
+{
+ int texture_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_offset);
+ int sampler_idx = nir_tex_instr_src_index(tex, nir_tex_src_sampler_offset);
+ struct ir3_instruction *texture, *sampler;
+
+ if (texture_idx >= 0) {
+ texture = ir3_get_src(ctx, &tex->src[texture_idx].src)[0];
+ texture = ir3_COV(ctx->block, texture, TYPE_U32, TYPE_U16);
+ } else {
+ /* TODO what to do for dynamic case? I guess we only need the
+ * max index for astc srgb workaround so maybe not a problem
+ * to worry about if we don't enable indirect samplers for
+ * a4xx?
+ */
+ ctx->max_texture_index = MAX2(ctx->max_texture_index, tex->texture_index);
+ texture = create_immed_typed(ctx->block, tex->texture_index, TYPE_U16);
+ }
+
+ if (sampler_idx >= 0) {
+ sampler = ir3_get_src(ctx, &tex->src[sampler_idx].src)[0];
+ sampler = ir3_COV(ctx->block, sampler, TYPE_U32, TYPE_U16);
+ } else {
+ sampler = create_immed_typed(ctx->block, tex->sampler_index, TYPE_U16);
+ }
+
+ return ir3_create_collect(ctx, (struct ir3_instruction*[]){
+ sampler,
+ texture,
+ }, 2);
+}
+
static void
emit_tex(struct ir3_context *ctx, nir_tex_instr *tex)
{
struct ir3_instruction * const *coord, * const *off, * const *ddx, * const *ddy;
struct ir3_instruction *lod, *compare, *proj, *sample_index;
bool has_bias = false, has_lod = false, has_proj = false, has_off = false;
- unsigned i, coords, flags;
+ unsigned i, coords, flags, ncomp;
unsigned nsrc0 = 0, nsrc1 = 0;
type_t type;
opc_t opc = 0;
+ ncomp = nir_dest_num_components(tex->dest);
+
coord = off = ddx = ddy = NULL;
lod = proj = compare = sample_index = NULL;
- /* TODO: might just be one component for gathers? */
- dst = ir3_get_dst(ctx, &tex->dest, 4);
+ dst = ir3_get_dst(ctx, &tex->dest, ncomp);
for (unsigned i = 0; i < tex->num_srcs; i++) {
switch (tex->src[i].src_type) {
case nir_tex_src_ms_index:
sample_index = ir3_get_src(ctx, &tex->src[i].src)[0];
break;
+ case nir_tex_src_texture_offset:
+ case nir_tex_src_sampler_offset:
+ /* handled in get_tex_samp_src() */
+ break;
default:
ir3_context_error(ctx, "Unhandled NIR tex src type: %d\n",
tex->src[i].src_type);
nsrc0 = i;
- /* NOTE a3xx (and possibly a4xx?) might be different, using isaml
- * with scaled x coord according to requested sample:
- */
- if (tex->op == nir_texop_txf_ms) {
- if (ctx->compiler->txf_ms_with_isaml) {
- /* the samples are laid out in x dimension as
- * 0 1 2 3
- * x_ms = (x << ms) + sample_index;
- */
- struct ir3_instruction *ms;
- ms = create_immed(b, (ctx->samples >> (2 * tex->texture_index)) & 3);
-
- src0[0] = ir3_SHL_B(b, src0[0], 0, ms, 0);
- src0[0] = ir3_ADD_U(b, src0[0], 0, sample_index, 0);
-
- opc = OPC_ISAML;
- } else {
- src0[nsrc0++] = sample_index;
- }
- }
-
/* scale up integer coords for TXF based on the LOD */
if (ctx->compiler->unminify_coords && (opc == OPC_ISAML)) {
assert(has_lod);
if (coords == 1) {
/* hw doesn't do 1d, so we treat it as 2d with
* height of 1, and patch up the y coord.
- * TODO: y coord should be (int)0 in some cases..
*/
- src0[nsrc0++] = create_immed(b, fui(0.5));
+ if (is_isam(opc)) {
+ src0[nsrc0++] = create_immed(b, 0);
+ } else {
+ src0[nsrc0++] = create_immed(b, fui(0.5));
+ }
}
if (tex->is_shadow && tex->op != nir_texop_lod)
struct ir3_instruction *idx = coord[coords];
/* the array coord for cube arrays needs 0.5 added to it */
- if (ctx->compiler->array_index_add_half && (opc != OPC_ISAML))
+ if (ctx->compiler->array_index_add_half && !is_isam(opc))
idx = ir3_ADD_F(b, idx, 0, create_immed(b, fui(0.5)), 0);
src0[nsrc0++] = idx;
src0[nsrc0++] = create_immed(b, fui(0.0));
}
+ /* NOTE a3xx (and possibly a4xx?) might be different, using isaml
+ * with scaled x coord according to requested sample:
+ */
+ if (tex->op == nir_texop_txf_ms) {
+ if (ctx->compiler->txf_ms_with_isaml) {
+ /* the samples are laid out in x dimension as
+ * 0 1 2 3
+ * x_ms = (x << ms) + sample_index;
+ */
+ struct ir3_instruction *ms;
+ ms = create_immed(b, (ctx->samples >> (2 * tex->texture_index)) & 3);
+
+ src0[0] = ir3_SHL_B(b, src0[0], 0, ms, 0);
+ src0[0] = ir3_ADD_U(b, src0[0], 0, sample_index, 0);
+
+ opc = OPC_ISAML;
+ } else {
+ src0[nsrc0++] = sample_index;
+ }
+ }
+
/*
* second argument (if applicable):
* - offsets
if (opc == OPC_GETLOD)
type = TYPE_U32;
- unsigned tex_idx = tex->texture_index;
-
- ctx->max_texture_index = MAX2(ctx->max_texture_index, tex_idx);
-
+ struct ir3_instruction *samp_tex = get_tex_samp_tex_src(ctx, tex);
struct ir3_instruction *col0 = ir3_create_collect(ctx, src0, nsrc0);
struct ir3_instruction *col1 = ir3_create_collect(ctx, src1, nsrc1);
- sam = ir3_SAM(b, opc, type, 0b1111, flags,
- tex_idx, tex_idx, col0, col1);
+ sam = ir3_SAM(b, opc, type, MASK(ncomp), flags,
+ samp_tex, col0, col1);
- if ((ctx->astc_srgb & (1 << tex_idx)) && !nir_tex_instr_is_query(tex)) {
+ if ((ctx->astc_srgb & (1 << tex->texture_index)) && !nir_tex_instr_is_query(tex)) {
/* only need first 3 components: */
sam->regs[0]->wrmask = 0x7;
ir3_split_dest(b, dst, sam, 0, 3);
* texture state:
*/
sam = ir3_SAM(b, opc, type, 0b1000, flags,
- tex_idx, tex_idx, col0, col1);
+ samp_tex, col0, col1);
array_insert(ctx->ir, ctx->ir->astc_srgb, sam);
ir3_split_dest(b, &dst[3], sam, 3, 1);
} else {
/* normal (non-workaround) case: */
- ir3_split_dest(b, dst, sam, 0, 4);
+ ir3_split_dest(b, dst, sam, 0, ncomp);
}
/* GETLOD returns results in 4.8 fixed point */
}
}
- put_dst(ctx, &tex->dest);
+ ir3_put_dst(ctx, &tex->dest);
}
static void
dst = ir3_get_dst(ctx, &tex->dest, 1);
sam = ir3_SAM(b, OPC_GETINFO, TYPE_U32, 0b0100, 0,
- tex->texture_index, tex->texture_index, NULL, NULL);
+ get_tex_samp_tex_src(ctx, tex), NULL, NULL);
/* even though there is only one component, since it ends
* up in .z rather than .x, we need a split_dest()
if (ctx->compiler->levels_add_one)
dst[0] = ir3_ADD_U(b, dst[0], 0, create_immed(b, 1), 0);
- put_dst(ctx, &tex->dest);
+ ir3_put_dst(ctx, &tex->dest);
}
static void
lod = ir3_get_src(ctx, &tex->src[0].src)[0];
sam = ir3_SAM(b, OPC_GETSIZE, TYPE_U32, 0b1111, flags,
- tex->texture_index, tex->texture_index, lod, NULL);
+ get_tex_samp_tex_src(ctx, tex), lod, NULL);
ir3_split_dest(b, dst, sam, 0, 4);
}
}
- put_dst(ctx, &tex->dest);
+ ir3_put_dst(ctx, &tex->dest);
}
static void
*/
ninputs += max_sysvals[ctx->so->type];
- ctx->ir = ir3_create(ctx->compiler, ninputs, noutputs);
+ ctx->ir = ir3_create(ctx->compiler, ctx->so->type, ninputs, noutputs);
/* Create inputs in first block: */
ctx->block = get_block(ctx, nir_start_block(fxn));
setup_output(ctx, var);
}
+ /* Find # of samplers: */
+ nir_foreach_variable(var, &ctx->s->uniforms) {
+ ctx->so->num_samp += glsl_type_get_sampler_count(var->type);
+ /* just assume that we'll be reading from images.. if it
+ * is write-only we don't have to count it, but not sure
+ * if there is a good way to know?
+ */
+ ctx->so->num_samp += glsl_type_get_image_count(var->type);
+ }
+
/* Setup registers (which should only be arrays): */
nir_foreach_register(reg, &ctx->s->registers) {
ir3_declare_array(ctx, reg);
ir3_print(ir);
}
+ /* do Sethi–Ullman numbering before scheduling: */
+ ir3_sun(ir);
+
ret = ir3_sched(ir);
if (ret) {
DBG("SCHED failed!");
goto out;
}
+ if (compiler->gpu_id >= 600) {
+ ir3_a6xx_fixup_atomic_dests(ir, so);
+ }
+
if (ir3_shader_debug & IR3_DBG_OPTMSGS) {
printf("AFTER SCHED:\n");
ir3_print(ir);
/* We need to do legalize after (for frag shader's) the "bary.f"
* offsets (inloc) have been assigned.
*/
- ir3_legalize(ir, &so->num_samp, &so->has_ssbo, &max_bary);
+ ir3_legalize(ir, &so->has_ssbo, &so->need_pixlod, &max_bary);
if (ir3_shader_debug & IR3_DBG_OPTMSGS) {
printf("AFTER LEGALIZE:\n");
else
so->total_in = max_bary + 1;
+ so->max_sun = ir->max_sun;
+
out:
if (ret) {
if (so->ir)
projects / mesa.git / blobdiff