#include "compiler/glsl/ir.h"
#include "brw_fs.h"
-#include "brw_fs_surface_builder.h"
#include "brw_nir.h"
+#include "brw_eu.h"
+#include "nir_search_helpers.h"
+#include "util/u_math.h"
+#include "util/bitscan.h"
using namespace brw;
-using namespace brw::surface_access;
void
fs_visitor::emit_nir_code()
{
+ emit_shader_float_controls_execution_mode();
+
/* emit the arrays used for inputs and outputs - load/store intrinsics will
* be converted to reads/writes of these arrays
*/
nir_setup_uniforms();
nir_emit_system_values();
- /* get the main function and emit it */
- nir_foreach_function(function, nir) {
- assert(strcmp(function->name, "main") == 0);
- assert(function->impl);
- nir_emit_impl(function->impl);
- }
+ nir_emit_impl(nir_shader_get_entrypoint((nir_shader *)nir));
}
void
const int loc = var->data.driver_location;
const unsigned var_vec4s =
var->data.compact ? DIV_ROUND_UP(glsl_get_length(var->type), 4)
- : type_size_vec4(var->type);
+ : type_size_vec4(var->type, true);
vec4s[loc] = MAX2(vec4s[loc], var_vec4s);
}
- nir_foreach_variable(var, &nir->outputs) {
- const int loc = var->data.driver_location;
- if (outputs[loc].file == BAD_FILE) {
- fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_F, 4 * vec4s[loc]);
- for (unsigned i = 0; i < vec4s[loc]; i++) {
- outputs[loc + i] = offset(reg, bld, 4 * i);
- }
+ for (unsigned loc = 0; loc < ARRAY_SIZE(vec4s);) {
+ if (vec4s[loc] == 0) {
+ loc++;
+ continue;
}
+
+ unsigned reg_size = vec4s[loc];
+
+ /* Check if there are any ranges that start within this range and extend
+ * past it. If so, include them in this allocation.
+ */
+ for (unsigned i = 1; i < reg_size; i++)
+ reg_size = MAX2(vec4s[i + loc] + i, reg_size);
+
+ fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_F, 4 * reg_size);
+ for (unsigned i = 0; i < reg_size; i++)
+ outputs[loc + i] = offset(reg, bld, 4 * i);
+
+ loc += reg_size;
}
}
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_vertex_id:
- unreachable("should be lowered by lower_vertex_id().");
+ case nir_intrinsic_load_base_vertex:
+ unreachable("should be lowered by nir_lower_system_values().");
case nir_intrinsic_load_vertex_id_zero_base:
- case nir_intrinsic_load_base_vertex:
+ case nir_intrinsic_load_is_indexed_draw:
+ case nir_intrinsic_load_first_vertex:
case nir_intrinsic_load_instance_id:
case nir_intrinsic_load_base_instance:
case nir_intrinsic_load_draw_id:
* masks for 2 and 3) in SIMD16.
*/
fs_reg shifted = abld.vgrf(BRW_REGISTER_TYPE_UW, 1);
- abld.SHR(shifted,
- stride(byte_offset(retype(brw_vec1_grf(1, 0),
- BRW_REGISTER_TYPE_UB), 28),
- 1, 8, 0),
- brw_imm_v(0x76543210));
+
+ for (unsigned i = 0; i < DIV_ROUND_UP(v->dispatch_width, 16); i++) {
+ const fs_builder hbld = abld.group(MIN2(16, v->dispatch_width), i);
+ hbld.SHR(offset(shifted, hbld, i),
+ stride(retype(brw_vec1_grf(1 + i, 7),
+ BRW_REGISTER_TYPE_UB),
+ 1, 8, 0),
+ brw_imm_v(0x76543210));
+ }
/* A set bit in the pixel mask means the channel is enabled, but
* that is the opposite of gl_HelperInvocation so we need to invert
}
}
- nir_foreach_function(function, nir) {
- assert(strcmp(function->name, "main") == 0);
- assert(function->impl);
- nir_foreach_block(block, function->impl) {
- emit_system_values_block(block, this);
- }
- }
+ nir_function_impl *impl = nir_shader_get_entrypoint((nir_shader *)nir);
+ nir_foreach_block(block, impl)
+ emit_system_values_block(block, this);
}
/*
default:
unreachable("Invalid bit size");
}
+ case BRW_REGISTER_TYPE_B:
case BRW_REGISTER_TYPE_W:
case BRW_REGISTER_TYPE_D:
case BRW_REGISTER_TYPE_Q:
switch(bit_size) {
+ case 8:
+ return BRW_REGISTER_TYPE_B;
case 16:
return BRW_REGISTER_TYPE_W;
case 32:
default:
unreachable("Invalid bit size");
}
+ case BRW_REGISTER_TYPE_UB:
case BRW_REGISTER_TYPE_UW:
case BRW_REGISTER_TYPE_UD:
case BRW_REGISTER_TYPE_UQ:
switch(bit_size) {
+ case 8:
+ return BRW_REGISTER_TYPE_UB;
case 16:
return BRW_REGISTER_TYPE_UW;
case 32:
unsigned array_elems =
reg->num_array_elems == 0 ? 1 : reg->num_array_elems;
unsigned size = array_elems * reg->num_components;
- const brw_reg_type reg_type =
+ const brw_reg_type reg_type = reg->bit_size == 8 ? BRW_REGISTER_TYPE_B :
brw_reg_type_from_bit_size(reg->bit_size, BRW_REGISTER_TYPE_F);
nir_locals[reg->index] = bld.vgrf(reg_type, size);
}
void
fs_visitor::nir_emit_if(nir_if *if_stmt)
{
+ bool invert;
+ fs_reg cond_reg;
+
+ /* If the condition has the form !other_condition, use other_condition as
+ * the source, but invert the predicate on the if instruction.
+ */
+ nir_alu_instr *cond = nir_src_as_alu_instr(if_stmt->condition);
+ if (cond != NULL && cond->op == nir_op_inot) {
+ assert(!cond->src[0].negate);
+ assert(!cond->src[0].abs);
+
+ invert = true;
+ cond_reg = get_nir_src(cond->src[0].src);
+ } else {
+ invert = false;
+ cond_reg = get_nir_src(if_stmt->condition);
+ }
+
/* first, put the condition into f0 */
fs_inst *inst = bld.MOV(bld.null_reg_d(),
- retype(get_nir_src(if_stmt->condition),
- BRW_REGISTER_TYPE_D));
+ retype(cond_reg, BRW_REGISTER_TYPE_D));
inst->conditional_mod = BRW_CONDITIONAL_NZ;
- bld.IF(BRW_PREDICATE_NORMAL);
+ bld.IF(BRW_PREDICATE_NORMAL)->predicate_inverse = invert;
nir_emit_cf_list(&if_stmt->then_list);
- /* note: if the else is empty, dead CF elimination will remove it */
- bld.emit(BRW_OPCODE_ELSE);
-
- nir_emit_cf_list(&if_stmt->else_list);
+ if (!nir_cf_list_is_empty_block(&if_stmt->else_list)) {
+ bld.emit(BRW_OPCODE_ELSE);
+ nir_emit_cf_list(&if_stmt->else_list);
+ }
bld.emit(BRW_OPCODE_ENDIF);
+
+ if (devinfo->gen < 7)
+ limit_dispatch_width(16, "Non-uniform control flow unsupported "
+ "in SIMD32 mode.");
}
void
nir_emit_cf_list(&loop->body);
bld.emit(BRW_OPCODE_WHILE);
+
+ if (devinfo->gen < 7)
+ limit_dispatch_width(16, "Non-uniform control flow unsupported "
+ "in SIMD32 mode.");
}
void
switch (instr->type) {
case nir_instr_type_alu:
- nir_emit_alu(abld, nir_instr_as_alu(instr));
+ nir_emit_alu(abld, nir_instr_as_alu(instr), true);
+ break;
+
+ case nir_instr_type_deref:
+ unreachable("All derefs should've been lowered");
break;
case nir_instr_type_intrinsic:
src0->op != nir_op_extract_i8 && src0->op != nir_op_extract_i16)
return false;
- nir_const_value *element = nir_src_as_const_value(src0->src[1].src);
- assert(element != NULL);
+ /* If either opcode has source modifiers, bail.
+ *
+ * TODO: We can potentially handle source modifiers if both of the opcodes
+ * we're combining are signed integers.
+ */
+ if (instr->src[0].abs || instr->src[0].negate ||
+ src0->src[0].abs || src0->src[0].negate)
+ return false;
+
+ unsigned element = nir_src_as_uint(src0->src[1].src);
/* Element type to extract.*/
const brw_reg_type type = brw_int_type(
op0 = offset(op0, bld, src0->src[0].swizzle[0]);
set_saturate(instr->dest.saturate,
- bld.MOV(result, subscript(op0, type, element->u32[0])));
+ bld.MOV(result, subscript(op0, type, element)));
return true;
}
fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr,
const fs_reg &result)
{
- if (!instr->src[0].src.is_ssa ||
- instr->src[0].src.ssa->parent_instr->type != nir_instr_type_intrinsic)
+ nir_intrinsic_instr *src0 = nir_src_as_intrinsic(instr->src[0].src);
+ if (src0 == NULL || src0->intrinsic != nir_intrinsic_load_front_face)
return false;
- nir_intrinsic_instr *src0 =
- nir_instr_as_intrinsic(instr->src[0].src.ssa->parent_instr);
-
- if (src0->intrinsic != nir_intrinsic_load_front_face)
+ if (!nir_src_is_const(instr->src[1].src) ||
+ !nir_src_is_const(instr->src[2].src))
return false;
- nir_const_value *value1 = nir_src_as_const_value(instr->src[1].src);
- if (!value1 || fabsf(value1->f32[0]) != 1.0f)
+ const float value1 = nir_src_as_float(instr->src[1].src);
+ const float value2 = nir_src_as_float(instr->src[2].src);
+ if (fabsf(value1) != 1.0f || fabsf(value2) != 1.0f)
return false;
- nir_const_value *value2 = nir_src_as_const_value(instr->src[2].src);
- if (!value2 || fabsf(value2->f32[0]) != 1.0f)
- return false;
+ /* nir_opt_algebraic should have gotten rid of bcsel(b, a, a) */
+ assert(value1 == -value2);
fs_reg tmp = vgrf(glsl_type::int_type);
- if (devinfo->gen >= 6) {
+ if (devinfo->gen >= 12) {
+ /* Bit 15 of g1.1 is 0 if the polygon is front facing. */
+ fs_reg g1 = fs_reg(retype(brw_vec1_grf(1, 1), BRW_REGISTER_TYPE_W));
+
+ /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
+ *
+ * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
+ * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
+ *
+ * and negate the result for (gl_FrontFacing ? -1.0 : 1.0).
+ */
+ bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
+ g1, brw_imm_uw(0x3f80));
+
+ if (value1 == -1.0f)
+ bld.MOV(tmp, negate(tmp));
+
+ } else if (devinfo->gen >= 6) {
/* Bit 15 of g0.0 is 0 if the polygon is front facing. */
fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
* surely be TRIANGLES
*/
- if (value1->f32[0] == -1.0f) {
+ if (value1 == -1.0f) {
g0.negate = true;
}
* surely be TRIANGLES
*/
- if (value1->f32[0] == -1.0f) {
+ if (value1 == -1.0f) {
g1_6.negate = true;
}
}
}
-void
-fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr)
+static brw_rnd_mode
+brw_rnd_mode_from_execution_mode(unsigned execution_mode)
{
- struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key;
- fs_inst *inst;
+ if (nir_has_any_rounding_mode_rtne(execution_mode))
+ return BRW_RND_MODE_RTNE;
+ if (nir_has_any_rounding_mode_rtz(execution_mode))
+ return BRW_RND_MODE_RTZ;
+ return BRW_RND_MODE_UNSPECIFIED;
+}
+
+fs_reg
+fs_visitor::prepare_alu_destination_and_sources(const fs_builder &bld,
+ nir_alu_instr *instr,
+ fs_reg *op,
+ bool need_dest)
+{
+ fs_reg result =
+ need_dest ? get_nir_dest(instr->dest.dest) : bld.null_reg_ud();
- fs_reg result = get_nir_dest(instr->dest.dest);
result.type = brw_type_for_nir_type(devinfo,
(nir_alu_type)(nir_op_infos[instr->op].output_type |
nir_dest_bit_size(instr->dest.dest)));
- fs_reg op[4];
for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
op[i] = get_nir_src(instr->src[i].src);
op[i].type = brw_type_for_nir_type(devinfo,
op[i].negate = instr->src[i].negate;
}
- /* We get a bunch of mov's out of the from_ssa pass and they may still
- * be vectorized. We'll handle them as a special-case. We'll also
- * handle vecN here because it's basically the same thing.
+ /* Move and vecN instrutions may still be vectored. Return the raw,
+ * vectored source and destination so that fs_visitor::nir_emit_alu can
+ * handle it. Other callers should not have to handle these kinds of
+ * instructions.
+ */
+ switch (instr->op) {
+ case nir_op_mov:
+ case nir_op_vec2:
+ case nir_op_vec3:
+ case nir_op_vec4:
+ return result;
+ default:
+ break;
+ }
+
+ /* At this point, we have dealt with any instruction that operates on
+ * more than a single channel. Therefore, we can just adjust the source
+ * and destination registers for that channel and emit the instruction.
+ */
+ unsigned channel = 0;
+ if (nir_op_infos[instr->op].output_size == 0) {
+ /* Since NIR is doing the scalarizing for us, we should only ever see
+ * vectorized operations with a single channel.
+ */
+ assert(util_bitcount(instr->dest.write_mask) == 1);
+ channel = ffs(instr->dest.write_mask) - 1;
+
+ result = offset(result, bld, channel);
+ }
+
+ for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
+ assert(nir_op_infos[instr->op].input_sizes[i] < 2);
+ op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
+ }
+
+ return result;
+}
+
+void
+fs_visitor::resolve_inot_sources(const fs_builder &bld, nir_alu_instr *instr,
+ fs_reg *op)
+{
+ for (unsigned i = 0; i < 2; i++) {
+ nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[i].src);
+
+ if (inot_instr != NULL && inot_instr->op == nir_op_inot &&
+ !inot_instr->src[0].abs && !inot_instr->src[0].negate) {
+ /* The source of the inot is now the source of instr. */
+ prepare_alu_destination_and_sources(bld, inot_instr, &op[i], false);
+
+ assert(!op[i].negate);
+ op[i].negate = true;
+ } else {
+ op[i] = resolve_source_modifiers(op[i]);
+ }
+ }
+}
+
+bool
+fs_visitor::try_emit_b2fi_of_inot(const fs_builder &bld,
+ fs_reg result,
+ nir_alu_instr *instr)
+{
+ if (devinfo->gen < 6 || devinfo->gen >= 12)
+ return false;
+
+ nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[0].src);
+
+ if (inot_instr == NULL || inot_instr->op != nir_op_inot)
+ return false;
+
+ /* HF is also possible as a destination on BDW+. For nir_op_b2i, the set
+ * of valid size-changing combinations is a bit more complex.
+ *
+ * The source restriction is just because I was lazy about generating the
+ * constant below.
+ */
+ if (nir_dest_bit_size(instr->dest.dest) != 32 ||
+ nir_src_bit_size(inot_instr->src[0].src) != 32)
+ return false;
+
+ /* b2[fi](inot(a)) maps a=0 => 1, a=-1 => 0. Since a can only be 0 or -1,
+ * this is float(1 + a).
+ */
+ fs_reg op;
+
+ prepare_alu_destination_and_sources(bld, inot_instr, &op, false);
+
+ /* Ignore the saturate modifier, if there is one. The result of the
+ * arithmetic can only be 0 or 1, so the clamping will do nothing anyway.
+ */
+ bld.ADD(result, op, brw_imm_d(1));
+
+ return true;
+}
+
+/**
+ * Emit code for nir_op_fsign possibly fused with a nir_op_fmul
+ *
+ * If \c instr is not the \c nir_op_fsign, then \c fsign_src is the index of
+ * the source of \c instr that is a \c nir_op_fsign.
+ */
+void
+fs_visitor::emit_fsign(const fs_builder &bld, const nir_alu_instr *instr,
+ fs_reg result, fs_reg *op, unsigned fsign_src)
+{
+ fs_inst *inst;
+
+ assert(instr->op == nir_op_fsign || instr->op == nir_op_fmul);
+ assert(fsign_src < nir_op_infos[instr->op].num_inputs);
+
+ if (instr->op != nir_op_fsign) {
+ const nir_alu_instr *const fsign_instr =
+ nir_src_as_alu_instr(instr->src[fsign_src].src);
+
+ assert(!fsign_instr->dest.saturate);
+
+ /* op[fsign_src] has the nominal result of the fsign, and op[1 -
+ * fsign_src] has the other multiply source. This must be rearranged so
+ * that op[0] is the source of the fsign op[1] is the other multiply
+ * source.
+ */
+ if (fsign_src != 0)
+ op[1] = op[0];
+
+ op[0] = get_nir_src(fsign_instr->src[0].src);
+
+ const nir_alu_type t =
+ (nir_alu_type)(nir_op_infos[instr->op].input_types[0] |
+ nir_src_bit_size(fsign_instr->src[0].src));
+
+ op[0].type = brw_type_for_nir_type(devinfo, t);
+ op[0].abs = fsign_instr->src[0].abs;
+ op[0].negate = fsign_instr->src[0].negate;
+
+ unsigned channel = 0;
+ if (nir_op_infos[instr->op].output_size == 0) {
+ /* Since NIR is doing the scalarizing for us, we should only ever see
+ * vectorized operations with a single channel.
+ */
+ assert(util_bitcount(instr->dest.write_mask) == 1);
+ channel = ffs(instr->dest.write_mask) - 1;
+ }
+
+ op[0] = offset(op[0], bld, fsign_instr->src[0].swizzle[channel]);
+ } else {
+ assert(!instr->dest.saturate);
+ }
+
+ if (op[0].abs) {
+ /* Straightforward since the source can be assumed to be either strictly
+ * >= 0 or strictly <= 0 depending on the setting of the negate flag.
+ */
+ set_condmod(BRW_CONDITIONAL_NZ, bld.MOV(result, op[0]));
+
+ if (instr->op == nir_op_fsign) {
+ inst = (op[0].negate)
+ ? bld.MOV(result, brw_imm_f(-1.0f))
+ : bld.MOV(result, brw_imm_f(1.0f));
+ } else {
+ op[1].negate = (op[0].negate != op[1].negate);
+ inst = bld.MOV(result, op[1]);
+ }
+
+ set_predicate(BRW_PREDICATE_NORMAL, inst);
+ } else if (type_sz(op[0].type) == 2) {
+ /* AND(val, 0x8000) gives the sign bit.
+ *
+ * Predicated OR ORs 1.0 (0x3c00) with the sign bit if val is not zero.
+ */
+ fs_reg zero = retype(brw_imm_uw(0), BRW_REGISTER_TYPE_HF);
+ bld.CMP(bld.null_reg_f(), op[0], zero, BRW_CONDITIONAL_NZ);
+
+ op[0].type = BRW_REGISTER_TYPE_UW;
+ result.type = BRW_REGISTER_TYPE_UW;
+ bld.AND(result, op[0], brw_imm_uw(0x8000u));
+
+ if (instr->op == nir_op_fsign)
+ inst = bld.OR(result, result, brw_imm_uw(0x3c00u));
+ else {
+ /* Use XOR here to get the result sign correct. */
+ inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UW));
+ }
+
+ inst->predicate = BRW_PREDICATE_NORMAL;
+ } else if (type_sz(op[0].type) == 4) {
+ /* AND(val, 0x80000000) gives the sign bit.
+ *
+ * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
+ * zero.
+ */
+ bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
+
+ op[0].type = BRW_REGISTER_TYPE_UD;
+ result.type = BRW_REGISTER_TYPE_UD;
+ bld.AND(result, op[0], brw_imm_ud(0x80000000u));
+
+ if (instr->op == nir_op_fsign)
+ inst = bld.OR(result, result, brw_imm_ud(0x3f800000u));
+ else {
+ /* Use XOR here to get the result sign correct. */
+ inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UD));
+ }
+
+ inst->predicate = BRW_PREDICATE_NORMAL;
+ } else {
+ /* For doubles we do the same but we need to consider:
+ *
+ * - 2-src instructions can't operate with 64-bit immediates
+ * - The sign is encoded in the high 32-bit of each DF
+ * - We need to produce a DF result.
+ */
+
+ fs_reg zero = vgrf(glsl_type::double_type);
+ bld.MOV(zero, setup_imm_df(bld, 0.0));
+ bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ);
+
+ bld.MOV(result, zero);
+
+ fs_reg r = subscript(result, BRW_REGISTER_TYPE_UD, 1);
+ bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1),
+ brw_imm_ud(0x80000000u));
+
+ if (instr->op == nir_op_fsign) {
+ set_predicate(BRW_PREDICATE_NORMAL,
+ bld.OR(r, r, brw_imm_ud(0x3ff00000u)));
+ } else {
+ /* This could be done better in some cases. If the scale is an
+ * immediate with the low 32-bits all 0, emitting a separate XOR and
+ * OR would allow an algebraic optimization to remove the OR. There
+ * are currently zero instances of fsign(double(x))*IMM in shader-db
+ * or any test suite, so it is hard to care at this time.
+ */
+ fs_reg result_int64 = retype(result, BRW_REGISTER_TYPE_UQ);
+ inst = bld.XOR(result_int64, result_int64,
+ retype(op[1], BRW_REGISTER_TYPE_UQ));
+ }
+ }
+}
+
+/**
+ * Deteremine whether sources of a nir_op_fmul can be fused with a nir_op_fsign
+ *
+ * Checks the operands of a \c nir_op_fmul to determine whether or not
+ * \c emit_fsign could fuse the multiplication with the \c sign() calculation.
+ *
+ * \param instr The multiplication instruction
+ *
+ * \param fsign_src The source of \c instr that may or may not be a
+ * \c nir_op_fsign
+ */
+static bool
+can_fuse_fmul_fsign(nir_alu_instr *instr, unsigned fsign_src)
+{
+ assert(instr->op == nir_op_fmul);
+
+ nir_alu_instr *const fsign_instr =
+ nir_src_as_alu_instr(instr->src[fsign_src].src);
+
+ /* Rules:
+ *
+ * 1. instr->src[fsign_src] must be a nir_op_fsign.
+ * 2. The nir_op_fsign can only be used by this multiplication.
+ * 3. The source that is the nir_op_fsign does not have source modifiers.
+ * \c emit_fsign only examines the source modifiers of the source of the
+ * \c nir_op_fsign.
+ *
+ * The nir_op_fsign must also not have the saturate modifier, but steps
+ * have already been taken (in nir_opt_algebraic) to ensure that.
*/
+ return fsign_instr != NULL && fsign_instr->op == nir_op_fsign &&
+ is_used_once(fsign_instr) &&
+ !instr->src[fsign_src].abs && !instr->src[fsign_src].negate;
+}
+
+void
+fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr,
+ bool need_dest)
+{
+ struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key;
+ fs_inst *inst;
+ unsigned execution_mode =
+ bld.shader->nir->info.float_controls_execution_mode;
+
+ fs_reg op[4];
+ fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, need_dest);
+
switch (instr->op) {
- case nir_op_imov:
- case nir_op_fmov:
+ case nir_op_mov:
case nir_op_vec2:
case nir_op_vec3:
case nir_op_vec4: {
if (!(instr->dest.write_mask & (1 << i)))
continue;
- if (instr->op == nir_op_imov || instr->op == nir_op_fmov) {
+ if (instr->op == nir_op_mov) {
inst = bld.MOV(offset(temp, bld, i),
offset(op[0], bld, instr->src[0].swizzle[i]));
} else {
}
return;
}
- default:
- break;
- }
- /* At this point, we have dealt with any instruction that operates on
- * more than a single channel. Therefore, we can just adjust the source
- * and destination registers for that channel and emit the instruction.
- */
- unsigned channel = 0;
- if (nir_op_infos[instr->op].output_size == 0) {
- /* Since NIR is doing the scalarizing for us, we should only ever see
- * vectorized operations with a single channel.
- */
- assert(_mesa_bitcount(instr->dest.write_mask) == 1);
- channel = ffs(instr->dest.write_mask) - 1;
-
- result = offset(result, bld, channel);
- }
-
- for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
- assert(nir_op_infos[instr->op].input_sizes[i] < 2);
- op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
- }
-
- switch (instr->op) {
case nir_op_i2f32:
case nir_op_u2f32:
if (optimize_extract_to_float(instr, result))
case nir_op_f2f16_rtne:
case nir_op_f2f16_rtz:
- bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
- brw_imm_d(brw_rnd_mode_from_nir_op(instr->op)));
- /* fallthrough */
+ case nir_op_f2f16: {
+ brw_rnd_mode rnd = BRW_RND_MODE_UNSPECIFIED;
+
+ if (nir_op_f2f16 == instr->op)
+ rnd = brw_rnd_mode_from_execution_mode(execution_mode);
+ else
+ rnd = brw_rnd_mode_from_nir_op(instr->op);
+
+ if (BRW_RND_MODE_UNSPECIFIED != rnd)
+ bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd));
/* In theory, it would be better to use BRW_OPCODE_F32TO16. Depending
* on the HW gen, it is a special hw opcode or just a MOV, and
* only for gen8+, it will be better to use directly the MOV, and use
* BRW_OPCODE_F32TO16 when/if we work for HF support on gen7.
*/
-
- case nir_op_f2f16_undef:
- case nir_op_i2i16:
- case nir_op_u2u16: {
- /* TODO: Fixing aligment rules for conversions from 32-bits to
- * 16-bit types should be moved to lower_conversions
- */
- fs_reg tmp = bld.vgrf(op[0].type, 1);
- tmp = subscript(tmp, result.type, 0);
- inst = bld.MOV(tmp, op[0]);
- inst->saturate = instr->dest.saturate;
- inst = bld.MOV(result, tmp);
+ assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */
+ inst = bld.MOV(result, op[0]);
inst->saturate = instr->dest.saturate;
break;
}
- case nir_op_f2f64:
- case nir_op_f2i64:
- case nir_op_f2u64:
+ case nir_op_b2i8:
+ case nir_op_b2i16:
+ case nir_op_b2i32:
+ case nir_op_b2i64:
+ case nir_op_b2f16:
+ case nir_op_b2f32:
+ case nir_op_b2f64:
+ if (try_emit_b2fi_of_inot(bld, result, instr))
+ break;
+ op[0].type = BRW_REGISTER_TYPE_D;
+ op[0].negate = !op[0].negate;
+ /* fallthrough */
case nir_op_i2f64:
case nir_op_i2i64:
case nir_op_u2f64:
case nir_op_u2u64:
- /* CHV PRM, vol07, 3D Media GPGPU Engine, Register Region Restrictions:
- *
- * "When source or destination is 64b (...), regioning in Align1
- * must follow these rules:
- *
- * 1. Source and destination horizontal stride must be aligned to
- * the same qword.
- * (...)"
- *
- * This means that 32-bit to 64-bit conversions need to have the 32-bit
- * data elements aligned to 64-bit. This restriction does not apply to
- * BDW and later.
- */
- if (nir_dest_bit_size(instr->dest.dest) == 64 &&
- nir_src_bit_size(instr->src[0].src) == 32 &&
- (devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) {
- fs_reg tmp = bld.vgrf(result.type, 1);
- tmp = subscript(tmp, op[0].type, 0);
- inst = bld.MOV(tmp, op[0]);
- inst = bld.MOV(result, tmp);
- inst->saturate = instr->dest.saturate;
- break;
- }
- /* fallthrough */
- case nir_op_f2f32:
- case nir_op_f2i32:
- case nir_op_f2u32:
+ case nir_op_f2f64:
+ case nir_op_f2i64:
+ case nir_op_f2u64:
case nir_op_i2i32:
case nir_op_u2u32:
+ case nir_op_f2i32:
+ case nir_op_f2u32:
+ case nir_op_i2f16:
+ case nir_op_i2i16:
+ case nir_op_u2f16:
+ case nir_op_u2u16:
+ case nir_op_f2i16:
+ case nir_op_f2u16:
+ case nir_op_i2i8:
+ case nir_op_u2u8:
+ case nir_op_f2i8:
+ case nir_op_f2u8:
+ if (result.type == BRW_REGISTER_TYPE_B ||
+ result.type == BRW_REGISTER_TYPE_UB ||
+ result.type == BRW_REGISTER_TYPE_HF)
+ assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */
+
+ if (op[0].type == BRW_REGISTER_TYPE_B ||
+ op[0].type == BRW_REGISTER_TYPE_UB ||
+ op[0].type == BRW_REGISTER_TYPE_HF)
+ assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */
+
inst = bld.MOV(result, op[0]);
inst->saturate = instr->dest.saturate;
break;
- case nir_op_fsign: {
- if (op[0].abs) {
- /* Straightforward since the source can be assumed to be
- * non-negative.
- */
- set_condmod(BRW_CONDITIONAL_NZ, bld.MOV(result, op[0]));
- set_predicate(BRW_PREDICATE_NORMAL, bld.MOV(result, brw_imm_f(1.0f)));
+ case nir_op_fsat:
+ inst = bld.MOV(result, op[0]);
+ inst->saturate = true;
+ break;
- } else if (type_sz(op[0].type) < 8) {
- /* AND(val, 0x80000000) gives the sign bit.
- *
- * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
- * zero.
- */
- bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
+ case nir_op_fneg:
+ case nir_op_ineg:
+ op[0].negate = true;
+ inst = bld.MOV(result, op[0]);
+ if (instr->op == nir_op_fneg)
+ inst->saturate = instr->dest.saturate;
+ break;
- fs_reg result_int = retype(result, BRW_REGISTER_TYPE_UD);
- op[0].type = BRW_REGISTER_TYPE_UD;
- result.type = BRW_REGISTER_TYPE_UD;
- bld.AND(result_int, op[0], brw_imm_ud(0x80000000u));
+ case nir_op_fabs:
+ case nir_op_iabs:
+ op[0].negate = false;
+ op[0].abs = true;
+ inst = bld.MOV(result, op[0]);
+ if (instr->op == nir_op_fabs)
+ inst->saturate = instr->dest.saturate;
+ break;
- inst = bld.OR(result_int, result_int, brw_imm_ud(0x3f800000u));
- inst->predicate = BRW_PREDICATE_NORMAL;
- if (instr->dest.saturate) {
- inst = bld.MOV(result, result);
- inst->saturate = true;
- }
- } else {
- /* For doubles we do the same but we need to consider:
- *
- * - 2-src instructions can't operate with 64-bit immediates
- * - The sign is encoded in the high 32-bit of each DF
- * - We need to produce a DF result.
- */
-
- fs_reg zero = vgrf(glsl_type::double_type);
- bld.MOV(zero, setup_imm_df(bld, 0.0));
- bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ);
-
- bld.MOV(result, zero);
-
- fs_reg r = subscript(result, BRW_REGISTER_TYPE_UD, 1);
- bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1),
- brw_imm_ud(0x80000000u));
+ case nir_op_f2f32:
+ if (nir_has_any_rounding_mode_enabled(execution_mode)) {
+ brw_rnd_mode rnd =
+ brw_rnd_mode_from_execution_mode(execution_mode);
+ bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
+ brw_imm_d(rnd));
+ }
- set_predicate(BRW_PREDICATE_NORMAL,
- bld.OR(r, r, brw_imm_ud(0x3ff00000u)));
+ if (op[0].type == BRW_REGISTER_TYPE_HF)
+ assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */
- if (instr->dest.saturate) {
- inst = bld.MOV(result, result);
- inst->saturate = true;
- }
- }
+ inst = bld.MOV(result, op[0]);
+ inst->saturate = instr->dest.saturate;
break;
- }
- case nir_op_isign:
- /* ASR(val, 31) -> negative val generates 0xffffffff (signed -1).
- * -> non-negative val generates 0x00000000.
- * Predicated OR sets 1 if val is positive.
- */
- assert(nir_dest_bit_size(instr->dest.dest) < 64);
- bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_G);
- bld.ASR(result, op[0], brw_imm_d(31));
- inst = bld.OR(result, result, brw_imm_d(1));
- inst->predicate = BRW_PREDICATE_NORMAL;
+ case nir_op_fsign:
+ emit_fsign(bld, instr, result, op, 0);
break;
case nir_op_frcp:
inst->saturate = instr->dest.saturate;
break;
- case nir_op_iadd:
case nir_op_fadd:
+ if (nir_has_any_rounding_mode_enabled(execution_mode)) {
+ brw_rnd_mode rnd =
+ brw_rnd_mode_from_execution_mode(execution_mode);
+ bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
+ brw_imm_d(rnd));
+ }
+ /* fallthrough */
+ case nir_op_iadd:
inst = bld.ADD(result, op[0], op[1]);
inst->saturate = instr->dest.saturate;
break;
+ case nir_op_uadd_sat:
+ inst = bld.ADD(result, op[0], op[1]);
+ inst->saturate = true;
+ break;
+
case nir_op_fmul:
+ for (unsigned i = 0; i < 2; i++) {
+ if (can_fuse_fmul_fsign(instr, i)) {
+ emit_fsign(bld, instr, result, op, i);
+ return;
+ }
+ }
+
+ /* We emit the rounding mode after the previous fsign optimization since
+ * it won't result in a MUL, but will try to negate the value by other
+ * means.
+ */
+ if (nir_has_any_rounding_mode_enabled(execution_mode)) {
+ brw_rnd_mode rnd =
+ brw_rnd_mode_from_execution_mode(execution_mode);
+ bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
+ brw_imm_d(rnd));
+ }
+
inst = bld.MUL(result, op[0], op[1]);
inst->saturate = instr->dest.saturate;
break;
+ case nir_op_imul_2x32_64:
+ case nir_op_umul_2x32_64:
+ bld.MUL(result, op[0], op[1]);
+ break;
+
case nir_op_imul:
assert(nir_dest_bit_size(instr->dest.dest) < 64);
bld.MUL(result, op[0], op[1]);
break;
}
- case nir_op_flt:
- case nir_op_fge:
- case nir_op_feq:
- case nir_op_fne: {
+ case nir_op_flt32:
+ case nir_op_fge32:
+ case nir_op_feq32:
+ case nir_op_fne32: {
fs_reg dest = result;
- if (nir_src_bit_size(instr->src[0].src) > 32) {
- dest = bld.vgrf(BRW_REGISTER_TYPE_DF, 1);
- }
- brw_conditional_mod cond;
- switch (instr->op) {
- case nir_op_flt:
- cond = BRW_CONDITIONAL_L;
- break;
- case nir_op_fge:
- cond = BRW_CONDITIONAL_GE;
- break;
- case nir_op_feq:
- cond = BRW_CONDITIONAL_Z;
- break;
- case nir_op_fne:
- cond = BRW_CONDITIONAL_NZ;
- break;
- default:
- unreachable("bad opcode");
- }
- bld.CMP(dest, op[0], op[1], cond);
- if (nir_src_bit_size(instr->src[0].src) > 32) {
+
+ const uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
+ if (bit_size != 32)
+ dest = bld.vgrf(op[0].type, 1);
+
+ bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op));
+
+ if (bit_size > 32) {
bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
+ } else if(bit_size < 32) {
+ /* When we convert the result to 32-bit we need to be careful and do
+ * it as a signed conversion to get sign extension (for 32-bit true)
+ */
+ const brw_reg_type src_type =
+ brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D);
+
+ bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
}
break;
}
- case nir_op_ilt:
- case nir_op_ult:
- case nir_op_ige:
- case nir_op_uge:
- case nir_op_ieq:
- case nir_op_ine: {
+ case nir_op_ilt32:
+ case nir_op_ult32:
+ case nir_op_ige32:
+ case nir_op_uge32:
+ case nir_op_ieq32:
+ case nir_op_ine32: {
fs_reg dest = result;
- if (nir_src_bit_size(instr->src[0].src) > 32) {
- dest = bld.vgrf(BRW_REGISTER_TYPE_UQ, 1);
- }
- brw_conditional_mod cond;
- switch (instr->op) {
- case nir_op_ilt:
- case nir_op_ult:
- cond = BRW_CONDITIONAL_L;
- break;
- case nir_op_ige:
- case nir_op_uge:
- cond = BRW_CONDITIONAL_GE;
- break;
- case nir_op_ieq:
- cond = BRW_CONDITIONAL_Z;
- break;
- case nir_op_ine:
- cond = BRW_CONDITIONAL_NZ;
- break;
- default:
- unreachable("bad opcode");
- }
- bld.CMP(dest, op[0], op[1], cond);
- if (nir_src_bit_size(instr->src[0].src) > 32) {
+ /* On Gen11 we have an additional issue being that src1 cannot be a byte
+ * type. So we convert both operands for the comparison.
+ */
+ fs_reg temp_op[2];
+ temp_op[0] = bld.fix_byte_src(op[0]);
+ temp_op[1] = bld.fix_byte_src(op[1]);
+
+ const uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
+ if (bit_size != 32)
+ dest = bld.vgrf(temp_op[0].type, 1);
+
+ bld.CMP(dest, temp_op[0], temp_op[1],
+ brw_cmod_for_nir_comparison(instr->op));
+
+ if (bit_size > 32) {
bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
+ } else if (bit_size < 32) {
+ /* When we convert the result to 32-bit we need to be careful and do
+ * it as a signed conversion to get sign extension (for 32-bit true)
+ */
+ const brw_reg_type src_type =
+ brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D);
+
+ bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
}
break;
}
case nir_op_inot:
if (devinfo->gen >= 8) {
+ nir_alu_instr *inot_src_instr = nir_src_as_alu_instr(instr->src[0].src);
+
+ if (inot_src_instr != NULL &&
+ (inot_src_instr->op == nir_op_ior ||
+ inot_src_instr->op == nir_op_ixor ||
+ inot_src_instr->op == nir_op_iand) &&
+ !inot_src_instr->src[0].abs &&
+ !inot_src_instr->src[0].negate &&
+ !inot_src_instr->src[1].abs &&
+ !inot_src_instr->src[1].negate) {
+ /* The sources of the source logical instruction are now the
+ * sources of the instruction that will be generated.
+ */
+ prepare_alu_destination_and_sources(bld, inot_src_instr, op, false);
+ resolve_inot_sources(bld, inot_src_instr, op);
+
+ /* Smash all of the sources and destination to be signed. This
+ * doesn't matter for the operation of the instruction, but cmod
+ * propagation fails on unsigned sources with negation (due to
+ * fs_inst::can_do_cmod returning false).
+ */
+ result.type =
+ brw_type_for_nir_type(devinfo,
+ (nir_alu_type)(nir_type_int |
+ nir_dest_bit_size(instr->dest.dest)));
+ op[0].type =
+ brw_type_for_nir_type(devinfo,
+ (nir_alu_type)(nir_type_int |
+ nir_src_bit_size(inot_src_instr->src[0].src)));
+ op[1].type =
+ brw_type_for_nir_type(devinfo,
+ (nir_alu_type)(nir_type_int |
+ nir_src_bit_size(inot_src_instr->src[1].src)));
+
+ /* For XOR, only invert one of the sources. Arbitrarily choose
+ * the first source.
+ */
+ op[0].negate = !op[0].negate;
+ if (inot_src_instr->op != nir_op_ixor)
+ op[1].negate = !op[1].negate;
+
+ switch (inot_src_instr->op) {
+ case nir_op_ior:
+ bld.AND(result, op[0], op[1]);
+ return;
+
+ case nir_op_iand:
+ bld.OR(result, op[0], op[1]);
+ return;
+
+ case nir_op_ixor:
+ bld.XOR(result, op[0], op[1]);
+ return;
+
+ default:
+ unreachable("impossible opcode");
+ }
+ }
op[0] = resolve_source_modifiers(op[0]);
}
bld.NOT(result, op[0]);
break;
case nir_op_ixor:
if (devinfo->gen >= 8) {
- op[0] = resolve_source_modifiers(op[0]);
- op[1] = resolve_source_modifiers(op[1]);
+ resolve_inot_sources(bld, instr, op);
}
bld.XOR(result, op[0], op[1]);
break;
case nir_op_ior:
if (devinfo->gen >= 8) {
- op[0] = resolve_source_modifiers(op[0]);
- op[1] = resolve_source_modifiers(op[1]);
+ resolve_inot_sources(bld, instr, op);
}
bld.OR(result, op[0], op[1]);
break;
case nir_op_iand:
if (devinfo->gen >= 8) {
- op[0] = resolve_source_modifiers(op[0]);
- op[1] = resolve_source_modifiers(op[1]);
+ resolve_inot_sources(bld, instr, op);
}
bld.AND(result, op[0], op[1]);
break;
case nir_op_fdot2:
case nir_op_fdot3:
case nir_op_fdot4:
- case nir_op_ball_fequal2:
- case nir_op_ball_iequal2:
- case nir_op_ball_fequal3:
- case nir_op_ball_iequal3:
- case nir_op_ball_fequal4:
- case nir_op_ball_iequal4:
- case nir_op_bany_fnequal2:
- case nir_op_bany_inequal2:
- case nir_op_bany_fnequal3:
- case nir_op_bany_inequal3:
- case nir_op_bany_fnequal4:
- case nir_op_bany_inequal4:
+ case nir_op_b32all_fequal2:
+ case nir_op_b32all_iequal2:
+ case nir_op_b32all_fequal3:
+ case nir_op_b32all_iequal3:
+ case nir_op_b32all_fequal4:
+ case nir_op_b32all_iequal4:
+ case nir_op_b32any_fnequal2:
+ case nir_op_b32any_inequal2:
+ case nir_op_b32any_fnequal3:
+ case nir_op_b32any_inequal3:
+ case nir_op_b32any_fnequal4:
+ case nir_op_b32any_inequal4:
unreachable("Lowered by nir_lower_alu_reductions");
case nir_op_fnoise1_1:
inst->saturate = instr->dest.saturate;
break;
- case nir_op_b2i:
- case nir_op_b2f:
- bld.MOV(result, negate(op[0]));
- break;
-
- case nir_op_i2b:
- case nir_op_f2b:
- if (nir_src_bit_size(instr->src[0].src) == 64) {
+ case nir_op_i2b32:
+ case nir_op_f2b32: {
+ uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
+ if (bit_size == 64) {
/* two-argument instructions can't take 64-bit immediates */
fs_reg zero;
fs_reg tmp;
- if (instr->op == nir_op_f2b) {
+ if (instr->op == nir_op_f2b32) {
zero = vgrf(glsl_type::double_type);
tmp = vgrf(glsl_type::double_type);
bld.MOV(zero, setup_imm_df(bld, 0.0));
bld.CMP(tmp, op[0], zero, BRW_CONDITIONAL_NZ);
bld.MOV(result, subscript(tmp, BRW_REGISTER_TYPE_UD, 0));
} else {
- if (instr->op == nir_op_f2b) {
- bld.CMP(result, op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
+ fs_reg zero;
+ if (bit_size == 32) {
+ zero = instr->op == nir_op_f2b32 ? brw_imm_f(0.0f) : brw_imm_d(0);
} else {
- bld.CMP(result, op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ);
+ assert(bit_size == 16);
+ zero = instr->op == nir_op_f2b32 ?
+ retype(brw_imm_w(0), BRW_REGISTER_TYPE_HF) : brw_imm_w(0);
}
+ bld.CMP(result, op[0], zero, BRW_CONDITIONAL_NZ);
}
break;
+ }
case nir_op_ftrunc:
inst = bld.RNDZ(result, op[0]);
case nir_op_pack_half_2x16:
unreachable("not reached: should be handled by lower_packing_builtins");
+ case nir_op_unpack_half_2x16_split_x_flush_to_zero:
+ assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode);
+ /* Fall-through */
case nir_op_unpack_half_2x16_split_x:
- inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X, result, op[0]);
+ inst = bld.emit(BRW_OPCODE_F16TO32, result,
+ subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
inst->saturate = instr->dest.saturate;
break;
+
+ case nir_op_unpack_half_2x16_split_y_flush_to_zero:
+ assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode);
+ /* Fall-through */
case nir_op_unpack_half_2x16_split_y:
- inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y, result, op[0]);
+ inst = bld.emit(BRW_OPCODE_F16TO32, result,
+ subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
inst->saturate = instr->dest.saturate;
break;
case nir_op_pack_64_2x32_split:
+ case nir_op_pack_32_2x16_split:
bld.emit(FS_OPCODE_PACK, result, op[0], op[1]);
break;
break;
}
+ case nir_op_unpack_32_2x16_split_x:
+ case nir_op_unpack_32_2x16_split_y: {
+ if (instr->op == nir_op_unpack_32_2x16_split_x)
+ bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
+ else
+ bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
+ break;
+ }
+
case nir_op_fpow:
inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]);
inst->saturate = instr->dest.saturate;
unreachable("not reached: should have been lowered");
case nir_op_ishl:
+ bld.SHL(result, op[0], op[1]);
+ break;
case nir_op_ishr:
- case nir_op_ushr: {
- fs_reg shift_count = op[1];
-
- if (devinfo->is_cherryview || gen_device_info_is_9lp(devinfo)) {
- if (op[1].file == VGRF &&
- (result.type == BRW_REGISTER_TYPE_Q ||
- result.type == BRW_REGISTER_TYPE_UQ)) {
- shift_count = fs_reg(VGRF, alloc.allocate(dispatch_width / 4),
- BRW_REGISTER_TYPE_UD);
- shift_count.stride = 2;
- bld.MOV(shift_count, op[1]);
- }
- }
+ bld.ASR(result, op[0], op[1]);
+ break;
+ case nir_op_ushr:
+ bld.SHR(result, op[0], op[1]);
+ break;
- switch (instr->op) {
- case nir_op_ishl:
- bld.SHL(result, op[0], shift_count);
- break;
- case nir_op_ishr:
- bld.ASR(result, op[0], shift_count);
- break;
- case nir_op_ushr:
- bld.SHR(result, op[0], shift_count);
- break;
- default:
- unreachable("not reached");
- }
+ case nir_op_urol:
+ bld.ROL(result, op[0], op[1]);
+ break;
+ case nir_op_uror:
+ bld.ROR(result, op[0], op[1]);
break;
- }
case nir_op_pack_half_2x16_split:
bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]);
break;
case nir_op_ffma:
+ if (nir_has_any_rounding_mode_enabled(execution_mode)) {
+ brw_rnd_mode rnd =
+ brw_rnd_mode_from_execution_mode(execution_mode);
+ bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
+ brw_imm_d(rnd));
+ }
+
inst = bld.MAD(result, op[2], op[1], op[0]);
inst->saturate = instr->dest.saturate;
break;
case nir_op_flrp:
+ if (nir_has_any_rounding_mode_enabled(execution_mode)) {
+ brw_rnd_mode rnd =
+ brw_rnd_mode_from_execution_mode(execution_mode);
+ bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
+ brw_imm_d(rnd));
+ }
+
inst = bld.LRP(result, op[0], op[1], op[2]);
inst->saturate = instr->dest.saturate;
break;
- case nir_op_bcsel:
+ case nir_op_b32csel:
if (optimize_frontfacing_ternary(instr, result))
return;
case nir_op_extract_u8:
case nir_op_extract_i8: {
- nir_const_value *byte = nir_src_as_const_value(instr->src[1].src);
- assert(byte != NULL);
+ unsigned byte = nir_src_as_uint(instr->src[1].src);
/* The PRMs say:
*
* Use two instructions and a word or DWord intermediate integer type.
*/
if (nir_dest_bit_size(instr->dest.dest) == 64) {
- const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i8);
+ const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
if (instr->op == nir_op_extract_i8) {
/* If we need to sign extend, extract to a word first */
fs_reg w_temp = bld.vgrf(BRW_REGISTER_TYPE_W);
- bld.MOV(w_temp, subscript(op[0], type, byte->u32[0]));
+ bld.MOV(w_temp, subscript(op[0], type, byte));
bld.MOV(result, w_temp);
+ } else if (byte & 1) {
+ /* Extract the high byte from the word containing the desired byte
+ * offset.
+ */
+ bld.SHR(result,
+ subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2),
+ brw_imm_uw(8));
} else {
/* Otherwise use an AND with 0xff and a word type */
- bld.AND(result, subscript(op[0], type, byte->u32[0] / 2), brw_imm_uw(0xff));
+ bld.AND(result,
+ subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2),
+ brw_imm_uw(0xff));
}
} else {
const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
- bld.MOV(result, subscript(op[0], type, byte->u32[0]));
+ bld.MOV(result, subscript(op[0], type, byte));
}
break;
}
case nir_op_extract_u16:
case nir_op_extract_i16: {
const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i16);
- nir_const_value *word = nir_src_as_const_value(instr->src[1].src);
- assert(word != NULL);
- bld.MOV(result, subscript(op[0], type, word->u32[0]));
+ unsigned word = nir_src_as_uint(instr->src[1].src);
+ bld.MOV(result, subscript(op[0], type, word));
break;
}
* to sign extend the low bit to 0/~0
*/
if (devinfo->gen <= 5 &&
+ !result.is_null() &&
(instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
fs_reg masked = vgrf(glsl_type::int_type);
bld.AND(masked, result, brw_imm_d(1));
fs_reg reg = bld.vgrf(reg_type, instr->def.num_components);
switch (instr->def.bit_size) {
+ case 8:
+ for (unsigned i = 0; i < instr->def.num_components; i++)
+ bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value[i].i8));
+ break;
+
+ case 16:
+ for (unsigned i = 0; i < instr->def.num_components; i++)
+ bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value[i].i16));
+ break;
+
case 32:
for (unsigned i = 0; i < instr->def.num_components; i++)
- bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value.i32[i]));
+ bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value[i].i32));
break;
case 64:
/* We don't get 64-bit integer types until gen8 */
for (unsigned i = 0; i < instr->def.num_components; i++) {
bld.MOV(retype(offset(reg, bld, i), BRW_REGISTER_TYPE_DF),
- setup_imm_df(bld, instr->value.f64[i]));
+ setup_imm_df(bld, instr->value[i].f64));
}
} else {
for (unsigned i = 0; i < instr->def.num_components; i++)
- bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value.i64[i]));
+ bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value[i].i64));
}
break;
fs_reg
fs_visitor::get_nir_src_imm(const nir_src &src)
{
- nir_const_value *val = nir_src_as_const_value(src);
assert(nir_src_bit_size(src) == 32);
- return val ? fs_reg(brw_imm_d(val->i32[0])) : get_nir_src(src);
+ return nir_src_is_const(src) ?
+ fs_reg(brw_imm_d(nir_src_as_int(src))) : get_nir_src(src);
}
fs_reg
{
if (dest.is_ssa) {
const brw_reg_type reg_type =
- brw_reg_type_from_bit_size(dest.ssa.bit_size, BRW_REGISTER_TYPE_F);
+ brw_reg_type_from_bit_size(dest.ssa.bit_size,
+ dest.ssa.bit_size == 8 ?
+ BRW_REGISTER_TYPE_D :
+ BRW_REGISTER_TYPE_F);
nir_ssa_values[dest.ssa.index] =
bld.vgrf(reg_type, dest.ssa.num_components);
+ bld.UNDEF(nir_ssa_values[dest.ssa.index]);
return nir_ssa_values[dest.ssa.index];
} else {
/* We don't handle indirects on locals */
}
}
-fs_reg
-fs_visitor::get_nir_image_deref(const nir_deref_var *deref)
-{
- fs_reg image(UNIFORM, deref->var->data.driver_location / 4,
- BRW_REGISTER_TYPE_UD);
- fs_reg indirect;
- unsigned indirect_max = 0;
-
- for (const nir_deref *tail = &deref->deref; tail->child;
- tail = tail->child) {
- const nir_deref_array *deref_array = nir_deref_as_array(tail->child);
- assert(tail->child->deref_type == nir_deref_type_array);
- const unsigned size = glsl_get_length(tail->type);
- const unsigned element_size = type_size_scalar(deref_array->deref.type);
- const unsigned base = MIN2(deref_array->base_offset, size - 1);
- image = offset(image, bld, base * element_size);
-
- if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
- fs_reg tmp = vgrf(glsl_type::uint_type);
-
- /* Accessing an invalid surface index with the dataport can result
- * in a hang. According to the spec "if the index used to
- * select an individual element is negative or greater than or
- * equal to the size of the array, the results of the operation
- * are undefined but may not lead to termination" -- which is one
- * of the possible outcomes of the hang. Clamp the index to
- * prevent access outside of the array bounds.
- */
- bld.emit_minmax(tmp, retype(get_nir_src(deref_array->indirect),
- BRW_REGISTER_TYPE_UD),
- brw_imm_ud(size - base - 1), BRW_CONDITIONAL_L);
-
- indirect_max += element_size * (tail->type->length - 1);
-
- bld.MUL(tmp, tmp, brw_imm_ud(element_size * 4));
- if (indirect.file == BAD_FILE) {
- indirect = tmp;
- } else {
- bld.ADD(indirect, indirect, tmp);
- }
- }
- }
-
- if (indirect.file == BAD_FILE) {
- return image;
- } else {
- /* Emit a pile of MOVs to load the uniform into a temporary. The
- * dead-code elimination pass will get rid of what we don't use.
- */
- fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, BRW_IMAGE_PARAM_SIZE);
- for (unsigned j = 0; j < BRW_IMAGE_PARAM_SIZE; j++) {
- bld.emit(SHADER_OPCODE_MOV_INDIRECT,
- offset(tmp, bld, j), offset(image, bld, j),
- indirect, brw_imm_ud((indirect_max + 1) * 4));
- }
- return tmp;
- }
-}
-
void
fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst,
unsigned wr_mask)
}
}
-/**
- * Get the matching channel register datatype for an image intrinsic of the
- * specified GLSL image type.
- */
-static brw_reg_type
-get_image_base_type(const glsl_type *type)
-{
- switch ((glsl_base_type)type->sampled_type) {
- case GLSL_TYPE_UINT:
- return BRW_REGISTER_TYPE_UD;
- case GLSL_TYPE_INT:
- return BRW_REGISTER_TYPE_D;
- case GLSL_TYPE_FLOAT:
- return BRW_REGISTER_TYPE_F;
- default:
- unreachable("Not reached.");
- }
-}
-
-/**
- * Get the appropriate atomic op for an image atomic intrinsic.
- */
-static unsigned
-get_image_atomic_op(nir_intrinsic_op op, const glsl_type *type)
-{
- switch (op) {
- case nir_intrinsic_image_atomic_add:
- return BRW_AOP_ADD;
- case nir_intrinsic_image_atomic_min:
- return (get_image_base_type(type) == BRW_REGISTER_TYPE_D ?
- BRW_AOP_IMIN : BRW_AOP_UMIN);
- case nir_intrinsic_image_atomic_max:
- return (get_image_base_type(type) == BRW_REGISTER_TYPE_D ?
- BRW_AOP_IMAX : BRW_AOP_UMAX);
- case nir_intrinsic_image_atomic_and:
- return BRW_AOP_AND;
- case nir_intrinsic_image_atomic_or:
- return BRW_AOP_OR;
- case nir_intrinsic_image_atomic_xor:
- return BRW_AOP_XOR;
- case nir_intrinsic_image_atomic_exchange:
- return BRW_AOP_MOV;
- case nir_intrinsic_image_atomic_comp_swap:
- return BRW_AOP_CMPWR;
- default:
- unreachable("Not reachable.");
- }
-}
-
static fs_inst *
emit_pixel_interpolater_send(const fs_builder &bld,
enum opcode opcode,
{
struct brw_wm_prog_data *wm_prog_data =
brw_wm_prog_data(bld.shader->stage_prog_data);
- fs_inst *inst;
- fs_reg payload;
- int mlen;
-
- if (src.file == BAD_FILE) {
- /* Dummy payload */
- payload = bld.vgrf(BRW_REGISTER_TYPE_F, 1);
- mlen = 1;
- } else {
- payload = src;
- mlen = 2 * bld.dispatch_width() / 8;
- }
- inst = bld.emit(opcode, dst, payload, desc);
- inst->mlen = mlen;
+ fs_inst *inst = bld.emit(opcode, dst, src, desc);
/* 2 floats per slot returned */
inst->size_written = 2 * dst.component_size(inst->exec_size);
inst->pi_noperspective = interpolation == INTERP_MODE_NOPERSPECTIVE;
}
/* Store the control data bits in the message payload and send it. */
- int mlen = 2;
+ unsigned mlen = 2;
if (channel_mask.file != BAD_FILE)
mlen += 4; /* channel masks, plus 3 extra copies of the data */
if (per_slot_offset.file != BAD_FILE)
fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
fs_reg *sources = ralloc_array(mem_ctx, fs_reg, mlen);
- int i = 0;
+ unsigned i = 0;
sources[i++] = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
if (per_slot_offset.file != BAD_FILE)
sources[i++] = per_slot_offset;
unsigned num_components,
unsigned first_component)
{
+ assert(type_sz(dst.type) == 4);
struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
-
- nir_const_value *vertex_const = nir_src_as_const_value(vertex_src);
- nir_const_value *offset_const = nir_src_as_const_value(offset_src);
const unsigned push_reg_count = gs_prog_data->base.urb_read_length * 8;
/* TODO: figure out push input layout for invocations == 1 */
- /* TODO: make this work with 64-bit inputs */
if (gs_prog_data->invocations == 1 &&
- type_sz(dst.type) <= 4 &&
- offset_const != NULL && vertex_const != NULL &&
- 4 * (base_offset + offset_const->u32[0]) < push_reg_count) {
- int imm_offset = (base_offset + offset_const->u32[0]) * 4 +
- vertex_const->u32[0] * push_reg_count;
+ nir_src_is_const(offset_src) && nir_src_is_const(vertex_src) &&
+ 4 * (base_offset + nir_src_as_uint(offset_src)) < push_reg_count) {
+ int imm_offset = (base_offset + nir_src_as_uint(offset_src)) * 4 +
+ nir_src_as_uint(vertex_src) * push_reg_count;
for (unsigned i = 0; i < num_components; i++) {
bld.MOV(offset(dst, bld, i),
fs_reg(ATTR, imm_offset + i + first_component, dst.type));
fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
if (gs_prog_data->invocations == 1) {
- if (vertex_const) {
+ if (nir_src_is_const(vertex_src)) {
/* The vertex index is constant; just select the proper URB handle. */
icp_handle =
- retype(brw_vec8_grf(first_icp_handle + vertex_const->i32[0], 0),
+ retype(brw_vec8_grf(first_icp_handle + nir_src_as_uint(vertex_src), 0),
BRW_REGISTER_TYPE_UD);
} else {
/* The vertex index is non-constant. We need to use indirect
} else {
assert(gs_prog_data->invocations > 1);
- if (vertex_const) {
- assert(devinfo->gen >= 9 || vertex_const->i32[0] <= 5);
+ if (nir_src_is_const(vertex_src)) {
+ unsigned vertex = nir_src_as_uint(vertex_src);
+ assert(devinfo->gen >= 9 || vertex <= 5);
bld.MOV(icp_handle,
- retype(brw_vec1_grf(first_icp_handle +
- vertex_const->i32[0] / 8,
- vertex_const->i32[0] % 8),
+ retype(brw_vec1_grf(first_icp_handle + vertex / 8, vertex % 8),
BRW_REGISTER_TYPE_UD));
} else {
/* The vertex index is non-constant. We need to use indirect
}
fs_inst *inst;
-
- fs_reg tmp_dst = dst;
fs_reg indirect_offset = get_nir_src(offset_src);
- unsigned num_iterations = 1;
- unsigned orig_num_components = num_components;
-
- if (type_sz(dst.type) == 8) {
- if (num_components > 2) {
- num_iterations = 2;
- num_components = 2;
- }
- fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dst.type);
- tmp_dst = tmp;
- first_component = first_component / 2;
- }
- for (unsigned iter = 0; iter < num_iterations; iter++) {
- if (offset_const) {
- /* Constant indexing - use global offset. */
- if (first_component != 0) {
- unsigned read_components = num_components + first_component;
- fs_reg tmp = bld.vgrf(dst.type, read_components);
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
- inst->size_written = read_components *
- tmp.component_size(inst->exec_size);
- for (unsigned i = 0; i < num_components; i++) {
- bld.MOV(offset(tmp_dst, bld, i),
- offset(tmp, bld, i + first_component));
- }
- } else {
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp_dst,
- icp_handle);
- inst->size_written = num_components *
- tmp_dst.component_size(inst->exec_size);
- }
- inst->offset = base_offset + offset_const->u32[0];
- inst->mlen = 1;
- } else {
- /* Indirect indexing - use per-slot offsets as well. */
- const fs_reg srcs[] = { icp_handle, indirect_offset };
+ if (nir_src_is_const(offset_src)) {
+ /* Constant indexing - use global offset. */
+ if (first_component != 0) {
unsigned read_components = num_components + first_component;
fs_reg tmp = bld.vgrf(dst.type, read_components);
- fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
- bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
- if (first_component != 0) {
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
- payload);
- inst->size_written = read_components *
- tmp.component_size(inst->exec_size);
- for (unsigned i = 0; i < num_components; i++) {
- bld.MOV(offset(tmp_dst, bld, i),
- offset(tmp, bld, i + first_component));
- }
- } else {
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp_dst,
- payload);
- inst->size_written = num_components *
- tmp_dst.component_size(inst->exec_size);
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
+ inst->size_written = read_components *
+ tmp.component_size(inst->exec_size);
+ for (unsigned i = 0; i < num_components; i++) {
+ bld.MOV(offset(dst, bld, i),
+ offset(tmp, bld, i + first_component));
}
- inst->offset = base_offset;
- inst->mlen = 2;
- }
-
- if (type_sz(dst.type) == 8) {
- shuffle_32bit_load_result_to_64bit_data(
- bld, tmp_dst, retype(tmp_dst, BRW_REGISTER_TYPE_F), num_components);
-
- for (unsigned c = 0; c < num_components; c++)
- bld.MOV(offset(dst, bld, iter * 2 + c), offset(tmp_dst, bld, c));
+ } else {
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
+ inst->size_written = num_components *
+ dst.component_size(inst->exec_size);
}
-
- if (num_iterations > 1) {
- num_components = orig_num_components - 2;
- if(offset_const) {
- base_offset++;
- } else {
- fs_reg new_indirect = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
- bld.ADD(new_indirect, indirect_offset, brw_imm_ud(1u));
- indirect_offset = new_indirect;
+ inst->offset = base_offset + nir_src_as_uint(offset_src);
+ inst->mlen = 1;
+ } else {
+ /* Indirect indexing - use per-slot offsets as well. */
+ const fs_reg srcs[] = { icp_handle, indirect_offset };
+ unsigned read_components = num_components + first_component;
+ fs_reg tmp = bld.vgrf(dst.type, read_components);
+ fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
+ bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
+ if (first_component != 0) {
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
+ payload);
+ inst->size_written = read_components *
+ tmp.component_size(inst->exec_size);
+ for (unsigned i = 0; i < num_components; i++) {
+ bld.MOV(offset(dst, bld, i),
+ offset(tmp, bld, i + first_component));
}
+ } else {
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload);
+ inst->size_written = num_components *
+ dst.component_size(inst->exec_size);
}
+ inst->offset = base_offset;
+ inst->mlen = 2;
}
}
fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr)
{
nir_src *offset_src = nir_get_io_offset_src(instr);
- nir_const_value *const_value = nir_src_as_const_value(*offset_src);
- if (const_value) {
+ if (nir_src_is_const(*offset_src)) {
/* The only constant offset we should find is 0. brw_nir.c's
* add_const_offset_to_base() will fold other constant offsets
* into instr->const_index[0].
*/
- assert(const_value->u32[0] == 0);
+ assert(nir_src_as_uint(*offset_src) == 0);
return fs_reg();
}
return get_nir_src(*offset_src);
}
-static void
-do_untyped_vector_read(const fs_builder &bld,
- const fs_reg dest,
- const fs_reg surf_index,
- const fs_reg offset_reg,
- unsigned num_components)
-{
- if (type_sz(dest.type) <= 2) {
- assert(dest.stride == 1);
- boolean is_const_offset = offset_reg.file == BRW_IMMEDIATE_VALUE;
-
- if (is_const_offset) {
- uint32_t start = offset_reg.ud & ~3;
- uint32_t end = offset_reg.ud + num_components * type_sz(dest.type);
- end = ALIGN(end, 4);
- assert (end - start <= 16);
-
- /* At this point we have 16-bit component/s that have constant
- * offset aligned to 4-bytes that can be read with untyped_reads.
- * untyped_read message requires 32-bit aligned offsets.
- */
- unsigned first_component = (offset_reg.ud & 3) / type_sz(dest.type);
- unsigned num_components_32bit = (end - start) / 4;
-
- fs_reg read_result =
- emit_untyped_read(bld, surf_index, brw_imm_ud(start),
- 1 /* dims */,
- num_components_32bit,
- BRW_PREDICATE_NONE);
- shuffle_32bit_load_result_to_16bit_data(bld,
- retype(dest, BRW_REGISTER_TYPE_W),
- retype(read_result, BRW_REGISTER_TYPE_D),
- first_component, num_components);
- } else {
- fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD);
- for (unsigned i = 0; i < num_components; i++) {
- if (i == 0) {
- bld.MOV(read_offset, offset_reg);
- } else {
- bld.ADD(read_offset, offset_reg,
- brw_imm_ud(i * type_sz(dest.type)));
- }
- /* Non constant offsets are not guaranteed to be aligned 32-bits
- * so they are read using one byte_scattered_read message
- * for each component.
- */
- fs_reg read_result =
- emit_byte_scattered_read(bld, surf_index, read_offset,
- 1 /* dims */, 1,
- type_sz(dest.type) * 8 /* bit_size */,
- BRW_PREDICATE_NONE);
- bld.MOV(offset(dest, bld, i),
- subscript (read_result, dest.type, 0));
- }
- }
- } else if (type_sz(dest.type) == 4) {
- fs_reg read_result = emit_untyped_read(bld, surf_index, offset_reg,
- 1 /* dims */,
- num_components,
- BRW_PREDICATE_NONE);
- read_result.type = dest.type;
- for (unsigned i = 0; i < num_components; i++)
- bld.MOV(offset(dest, bld, i), offset(read_result, bld, i));
- } else if (type_sz(dest.type) == 8) {
- /* Reading a dvec, so we need to:
- *
- * 1. Multiply num_components by 2, to account for the fact that we
- * need to read 64-bit components.
- * 2. Shuffle the result of the load to form valid 64-bit elements
- * 3. Emit a second load (for components z/w) if needed.
- */
- fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD);
- bld.MOV(read_offset, offset_reg);
-
- int iters = num_components <= 2 ? 1 : 2;
-
- /* Load the dvec, the first iteration loads components x/y, the second
- * iteration, if needed, loads components z/w
- */
- for (int it = 0; it < iters; it++) {
- /* Compute number of components to read in this iteration */
- int iter_components = MIN2(2, num_components);
- num_components -= iter_components;
-
- /* Read. Since this message reads 32-bit components, we need to
- * read twice as many components.
- */
- fs_reg read_result = emit_untyped_read(bld, surf_index, read_offset,
- 1 /* dims */,
- iter_components * 2,
- BRW_PREDICATE_NONE);
-
- /* Shuffle the 32-bit load result into valid 64-bit data */
- const fs_reg packed_result = bld.vgrf(dest.type, iter_components);
- shuffle_32bit_load_result_to_64bit_data(
- bld, packed_result, read_result, iter_components);
-
- /* Move each component to its destination */
- read_result = retype(read_result, BRW_REGISTER_TYPE_DF);
- for (int c = 0; c < iter_components; c++) {
- bld.MOV(offset(dest, bld, it * 2 + c),
- offset(packed_result, bld, c));
- }
-
- bld.ADD(read_offset, read_offset, brw_imm_ud(16));
- }
- } else {
- unreachable("Unsupported type");
- }
-}
-
void
fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld,
nir_intrinsic_instr *instr)
switch (instr->intrinsic) {
case nir_intrinsic_load_vertex_id:
- unreachable("should be lowered by lower_vertex_id()");
-
- case nir_intrinsic_load_vertex_id_zero_base:
case nir_intrinsic_load_base_vertex:
- case nir_intrinsic_load_instance_id:
- case nir_intrinsic_load_base_instance:
- case nir_intrinsic_load_draw_id: {
- gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
- fs_reg val = nir_system_values[sv];
- assert(val.file != BAD_FILE);
- dest.type = val.type;
- bld.MOV(dest, val);
- break;
- }
+ unreachable("should be lowered by nir_lower_system_values()");
case nir_intrinsic_load_input: {
+ assert(nir_dest_bit_size(instr->dest) == 32);
fs_reg src = fs_reg(ATTR, nir_intrinsic_base(instr) * 4, dest.type);
- unsigned first_component = nir_intrinsic_component(instr);
- unsigned num_components = instr->num_components;
-
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
- assert(const_offset && "Indirect input loads not allowed");
- src = offset(src, bld, const_offset->u32[0]);
-
- if (type_sz(dest.type) == 8)
- first_component /= 2;
-
- for (unsigned j = 0; j < num_components; j++) {
- bld.MOV(offset(dest, bld, j), offset(src, bld, j + first_component));
- }
+ src = offset(src, bld, nir_intrinsic_component(instr));
+ src = offset(src, bld, nir_src_as_uint(instr->src[0]));
- if (type_sz(dest.type) == 8) {
- shuffle_32bit_load_result_to_64bit_data(bld,
- dest,
- retype(dest, BRW_REGISTER_TYPE_F),
- instr->num_components);
- }
+ for (unsigned i = 0; i < instr->num_components; i++)
+ bld.MOV(offset(dest, bld, i), offset(src, bld, i));
break;
}
+ case nir_intrinsic_load_vertex_id_zero_base:
+ case nir_intrinsic_load_instance_id:
+ case nir_intrinsic_load_base_instance:
+ case nir_intrinsic_load_draw_id:
+ case nir_intrinsic_load_first_vertex:
+ case nir_intrinsic_load_is_indexed_draw:
+ unreachable("lowered by brw_nir_lower_vs_inputs");
+
default:
nir_emit_intrinsic(bld, instr);
break;
}
}
-void
-fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld,
- nir_intrinsic_instr *instr)
+fs_reg
+fs_visitor::get_tcs_single_patch_icp_handle(const fs_builder &bld,
+ nir_intrinsic_instr *instr)
{
- assert(stage == MESA_SHADER_TESS_CTRL);
- struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
+ const nir_src &vertex_src = instr->src[0];
+ nir_intrinsic_instr *vertex_intrin = nir_src_as_intrinsic(vertex_src);
+ fs_reg icp_handle;
+
+ if (nir_src_is_const(vertex_src)) {
+ /* Emit a MOV to resolve <0,1,0> regioning. */
+ icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+ unsigned vertex = nir_src_as_uint(vertex_src);
+ bld.MOV(icp_handle,
+ retype(brw_vec1_grf(1 + (vertex >> 3), vertex & 7),
+ BRW_REGISTER_TYPE_UD));
+ } else if (tcs_prog_data->instances == 1 && vertex_intrin &&
+ vertex_intrin->intrinsic == nir_intrinsic_load_invocation_id) {
+ /* For the common case of only 1 instance, an array index of
+ * gl_InvocationID means reading g1. Skip all the indirect work.
+ */
+ icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
+ } else {
+ /* The vertex index is non-constant. We need to use indirect
+ * addressing to fetch the proper URB handle.
+ */
+ icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+
+ /* Each ICP handle is a single DWord (4 bytes) */
+ fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+ bld.SHL(vertex_offset_bytes,
+ retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
+ brw_imm_ud(2u));
+
+ /* Start at g1. We might read up to 4 registers. */
+ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
+ retype(brw_vec8_grf(1, 0), icp_handle.type), vertex_offset_bytes,
+ brw_imm_ud(4 * REG_SIZE));
+ }
+
+ return icp_handle;
+}
+
+fs_reg
+fs_visitor::get_tcs_eight_patch_icp_handle(const fs_builder &bld,
+ nir_intrinsic_instr *instr)
+{
+ struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
+ struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
+ const nir_src &vertex_src = instr->src[0];
+
+ unsigned first_icp_handle = tcs_prog_data->include_primitive_id ? 3 : 2;
+
+ if (nir_src_is_const(vertex_src)) {
+ return fs_reg(retype(brw_vec8_grf(first_icp_handle +
+ nir_src_as_uint(vertex_src), 0),
+ BRW_REGISTER_TYPE_UD));
+ }
+
+ /* The vertex index is non-constant. We need to use indirect
+ * addressing to fetch the proper URB handle.
+ *
+ * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0>
+ * indicating that channel <n> should read the handle from
+ * DWord <n>. We convert that to bytes by multiplying by 4.
+ *
+ * Next, we convert the vertex index to bytes by multiplying
+ * by 32 (shifting by 5), and add the two together. This is
+ * the final indirect byte offset.
+ */
+ fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+ fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
+ fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+ fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+ fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
+
+ /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */
+ bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
+ /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */
+ bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
+ /* Convert vertex_index to bytes (multiply by 32) */
+ bld.SHL(vertex_offset_bytes,
+ retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
+ brw_imm_ud(5u));
+ bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
+
+ /* Use first_icp_handle as the base offset. There is one register
+ * of URB handles per vertex, so inform the register allocator that
+ * we might read up to nir->info.gs.vertices_in registers.
+ */
+ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
+ retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
+ icp_offset_bytes, brw_imm_ud(tcs_key->input_vertices * REG_SIZE));
+
+ return icp_handle;
+}
+
+struct brw_reg
+fs_visitor::get_tcs_output_urb_handle()
+{
+ struct brw_vue_prog_data *vue_prog_data = brw_vue_prog_data(prog_data);
+
+ if (vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_SINGLE_PATCH) {
+ return retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD);
+ } else {
+ assert(vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH);
+ return retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
+ }
+}
+
+void
+fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld,
+ nir_intrinsic_instr *instr)
+{
+ assert(stage == MESA_SHADER_TESS_CTRL);
+ struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
+ struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
+ struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base;
+
+ bool eight_patch =
+ vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH;
fs_reg dst;
if (nir_intrinsic_infos[instr->intrinsic].has_dest)
switch (instr->intrinsic) {
case nir_intrinsic_load_primitive_id:
- bld.MOV(dst, fs_reg(brw_vec1_grf(0, 1)));
+ bld.MOV(dst, fs_reg(eight_patch ? brw_vec8_grf(2, 0)
+ : brw_vec1_grf(0, 1)));
break;
case nir_intrinsic_load_invocation_id:
bld.MOV(retype(dst, invocation_id.type), invocation_id);
/* Zero the message header */
bld.exec_all().MOV(m0, brw_imm_ud(0u));
- /* Copy "Barrier ID" from r0.2, bits 16:13 */
- chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
- brw_imm_ud(INTEL_MASK(16, 13)));
+ if (devinfo->gen < 11) {
+ /* Copy "Barrier ID" from r0.2, bits 16:13 */
+ chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
+ brw_imm_ud(INTEL_MASK(16, 13)));
- /* Shift it up to bits 27:24. */
- chanbld.SHL(m0_2, m0_2, brw_imm_ud(11));
+ /* Shift it up to bits 27:24. */
+ chanbld.SHL(m0_2, m0_2, brw_imm_ud(11));
+ } else {
+ chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
+ brw_imm_ud(INTEL_MASK(30, 24)));
+ }
/* Set the Barrier Count and the enable bit */
- chanbld.OR(m0_2, m0_2,
- brw_imm_ud(tcs_prog_data->instances << 9 | (1 << 15)));
+ if (devinfo->gen < 11) {
+ chanbld.OR(m0_2, m0_2,
+ brw_imm_ud(tcs_prog_data->instances << 9 | (1 << 15)));
+ } else {
+ chanbld.OR(m0_2, m0_2,
+ brw_imm_ud(tcs_prog_data->instances << 8 | (1 << 15)));
+ }
bld.emit(SHADER_OPCODE_BARRIER, bld.null_reg_ud(), m0);
break;
break;
case nir_intrinsic_load_per_vertex_input: {
+ assert(nir_dest_bit_size(instr->dest) == 32);
fs_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
-
- const nir_src &vertex_src = instr->src[0];
- nir_const_value *vertex_const = nir_src_as_const_value(vertex_src);
-
fs_inst *inst;
- fs_reg icp_handle;
-
- if (vertex_const) {
- /* Emit a MOV to resolve <0,1,0> regioning. */
- icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
- bld.MOV(icp_handle,
- retype(brw_vec1_grf(1 + (vertex_const->i32[0] >> 3),
- vertex_const->i32[0] & 7),
- BRW_REGISTER_TYPE_UD));
- } else if (tcs_prog_data->instances == 1 &&
- vertex_src.is_ssa &&
- vertex_src.ssa->parent_instr->type == nir_instr_type_intrinsic &&
- nir_instr_as_intrinsic(vertex_src.ssa->parent_instr)->intrinsic == nir_intrinsic_load_invocation_id) {
- /* For the common case of only 1 instance, an array index of
- * gl_InvocationID means reading g1. Skip all the indirect work.
- */
- icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
- } else {
- /* The vertex index is non-constant. We need to use indirect
- * addressing to fetch the proper URB handle.
- */
- icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
-
- /* Each ICP handle is a single DWord (4 bytes) */
- fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
- bld.SHL(vertex_offset_bytes,
- retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
- brw_imm_ud(2u));
-
- /* Start at g1. We might read up to 4 registers. */
- bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
- retype(brw_vec8_grf(1, 0), icp_handle.type), vertex_offset_bytes,
- brw_imm_ud(4 * REG_SIZE));
- }
+ fs_reg icp_handle =
+ eight_patch ? get_tcs_eight_patch_icp_handle(bld, instr)
+ : get_tcs_single_patch_icp_handle(bld, instr);
/* We can only read two double components with each URB read, so
* we send two read messages in that case, each one loading up to
* two double components.
*/
- unsigned num_iterations = 1;
unsigned num_components = instr->num_components;
unsigned first_component = nir_intrinsic_component(instr);
- fs_reg orig_dst = dst;
- if (type_sz(dst.type) == 8) {
- first_component = first_component / 2;
- if (instr->num_components > 2) {
- num_iterations = 2;
- num_components = 2;
- }
- fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dst.type);
- dst = tmp;
- }
-
- for (unsigned iter = 0; iter < num_iterations; iter++) {
- if (indirect_offset.file == BAD_FILE) {
- /* Constant indexing - use global offset. */
- if (first_component != 0) {
- unsigned read_components = num_components + first_component;
- fs_reg tmp = bld.vgrf(dst.type, read_components);
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
- for (unsigned i = 0; i < num_components; i++) {
- bld.MOV(offset(dst, bld, i),
- offset(tmp, bld, i + first_component));
- }
- } else {
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
+ if (indirect_offset.file == BAD_FILE) {
+ /* Constant indexing - use global offset. */
+ if (first_component != 0) {
+ unsigned read_components = num_components + first_component;
+ fs_reg tmp = bld.vgrf(dst.type, read_components);
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
+ for (unsigned i = 0; i < num_components; i++) {
+ bld.MOV(offset(dst, bld, i),
+ offset(tmp, bld, i + first_component));
}
- inst->offset = imm_offset;
- inst->mlen = 1;
} else {
- /* Indirect indexing - use per-slot offsets as well. */
- const fs_reg srcs[] = { icp_handle, indirect_offset };
- fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
- bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
- if (first_component != 0) {
- unsigned read_components = num_components + first_component;
- fs_reg tmp = bld.vgrf(dst.type, read_components);
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
- payload);
- for (unsigned i = 0; i < num_components; i++) {
- bld.MOV(offset(dst, bld, i),
- offset(tmp, bld, i + first_component));
- }
- } else {
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
- payload);
- }
- inst->offset = imm_offset;
- inst->mlen = 2;
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
}
- inst->size_written = (num_components + first_component) *
- inst->dst.component_size(inst->exec_size);
-
- /* If we are reading 64-bit data using 32-bit read messages we need
- * build proper 64-bit data elements by shuffling the low and high
- * 32-bit components around like we do for other things like UBOs
- * or SSBOs.
- */
- if (type_sz(dst.type) == 8) {
- shuffle_32bit_load_result_to_64bit_data(
- bld, dst, retype(dst, BRW_REGISTER_TYPE_F), num_components);
-
- for (unsigned c = 0; c < num_components; c++) {
- bld.MOV(offset(orig_dst, bld, iter * 2 + c),
- offset(dst, bld, c));
+ inst->offset = imm_offset;
+ inst->mlen = 1;
+ } else {
+ /* Indirect indexing - use per-slot offsets as well. */
+ const fs_reg srcs[] = { icp_handle, indirect_offset };
+ fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
+ bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
+ if (first_component != 0) {
+ unsigned read_components = num_components + first_component;
+ fs_reg tmp = bld.vgrf(dst.type, read_components);
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
+ payload);
+ for (unsigned i = 0; i < num_components; i++) {
+ bld.MOV(offset(dst, bld, i),
+ offset(tmp, bld, i + first_component));
}
+ } else {
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
+ payload);
}
+ inst->offset = imm_offset;
+ inst->mlen = 2;
+ }
+ inst->size_written = (num_components + first_component) *
+ inst->dst.component_size(inst->exec_size);
- /* Copy the temporary to the destination to deal with writemasking.
- *
- * Also attempt to deal with gl_PointSize being in the .w component.
- */
- if (inst->offset == 0 && indirect_offset.file == BAD_FILE) {
- assert(type_sz(dst.type) < 8);
- inst->dst = bld.vgrf(dst.type, 4);
- inst->size_written = 4 * REG_SIZE;
- bld.MOV(dst, offset(inst->dst, bld, 3));
- }
-
- /* If we are loading double data and we need a second read message
- * adjust the write offset
- */
- if (num_iterations > 1) {
- num_components = instr->num_components - 2;
- imm_offset++;
- }
+ /* Copy the temporary to the destination to deal with writemasking.
+ *
+ * Also attempt to deal with gl_PointSize being in the .w component.
+ */
+ if (inst->offset == 0 && indirect_offset.file == BAD_FILE) {
+ assert(type_sz(dst.type) == 4);
+ inst->dst = bld.vgrf(dst.type, 4);
+ inst->size_written = 4 * REG_SIZE;
+ bld.MOV(dst, offset(inst->dst, bld, 3));
}
break;
}
case nir_intrinsic_load_output:
case nir_intrinsic_load_per_vertex_output: {
+ assert(nir_dest_bit_size(instr->dest) == 32);
fs_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
unsigned first_component = nir_intrinsic_component(instr);
+ struct brw_reg output_handles = get_tcs_output_urb_handle();
+
fs_inst *inst;
if (indirect_offset.file == BAD_FILE) {
- /* Replicate the patch handle to all enabled channels */
+ /* This MOV replicates the output handle to all enabled channels
+ * is SINGLE_PATCH mode.
+ */
fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
- bld.MOV(patch_handle,
- retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD));
+ bld.MOV(patch_handle, output_handles);
{
if (first_component != 0) {
}
} else {
/* Indirect indexing - use per-slot offsets as well. */
- const fs_reg srcs[] = {
- retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD),
- indirect_offset
- };
+ const fs_reg srcs[] = { output_handles, indirect_offset };
fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
if (first_component != 0) {
case nir_intrinsic_store_output:
case nir_intrinsic_store_per_vertex_output: {
+ assert(nir_src_bit_size(instr->src[0]) == 32);
fs_reg value = get_nir_src(instr->src[0]);
- bool is_64bit = (instr->src[0].is_ssa ?
- instr->src[0].ssa->bit_size : instr->src[0].reg.reg->bit_size) == 64;
fs_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
unsigned mask = instr->const_index[1];
unsigned header_regs = 0;
+ struct brw_reg output_handles = get_tcs_output_urb_handle();
+
fs_reg srcs[7];
- srcs[header_regs++] = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD);
+ srcs[header_regs++] = output_handles;
if (indirect_offset.file != BAD_FILE) {
srcs[header_regs++] = indirect_offset;
/* We can only pack two 64-bit components in a single message, so send
* 2 messages if we have more components
*/
- unsigned num_iterations = 1;
- unsigned iter_components = num_components;
unsigned first_component = nir_intrinsic_component(instr);
- if (is_64bit) {
- first_component = first_component / 2;
- if (instr->num_components > 2) {
- num_iterations = 2;
- iter_components = 2;
- }
- }
-
mask = mask << first_component;
- for (unsigned iter = 0; iter < num_iterations; iter++) {
- if (!is_64bit && mask != WRITEMASK_XYZW) {
- srcs[header_regs++] = brw_imm_ud(mask << 16);
- opcode = indirect_offset.file != BAD_FILE ?
- SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
- SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
- } else if (is_64bit && ((mask & WRITEMASK_XY) != WRITEMASK_XY)) {
- /* Expand the 64-bit mask to 32-bit channels. We only handle
- * two channels in each iteration, so we only care about X/Y.
- */
- unsigned mask32 = 0;
- if (mask & WRITEMASK_X)
- mask32 |= WRITEMASK_XY;
- if (mask & WRITEMASK_Y)
- mask32 |= WRITEMASK_ZW;
-
- /* If the mask does not include any of the channels X or Y there
- * is nothing to do in this iteration. Move on to the next couple
- * of 64-bit channels.
- */
- if (!mask32) {
- mask >>= 2;
- imm_offset++;
- continue;
- }
-
- srcs[header_regs++] = brw_imm_ud(mask32 << 16);
- opcode = indirect_offset.file != BAD_FILE ?
- SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
- SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
- } else {
- opcode = indirect_offset.file != BAD_FILE ?
- SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT :
- SHADER_OPCODE_URB_WRITE_SIMD8;
- }
-
- for (unsigned i = 0; i < iter_components; i++) {
- if (!(mask & (1 << (i + first_component))))
- continue;
-
- if (!is_64bit) {
- srcs[header_regs + i + first_component] = offset(value, bld, i);
- } else {
- /* We need to shuffle the 64-bit data to match the layout
- * expected by our 32-bit URB write messages. We use a temporary
- * for that.
- */
- unsigned channel = iter * 2 + i;
- fs_reg dest = shuffle_64bit_data_for_32bit_write(bld,
- offset(value, bld, channel), 1);
+ if (mask != WRITEMASK_XYZW) {
+ srcs[header_regs++] = brw_imm_ud(mask << 16);
+ opcode = indirect_offset.file != BAD_FILE ?
+ SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
+ SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
+ } else {
+ opcode = indirect_offset.file != BAD_FILE ?
+ SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT :
+ SHADER_OPCODE_URB_WRITE_SIMD8;
+ }
- srcs[header_regs + (i + first_component) * 2] = dest;
- srcs[header_regs + (i + first_component) * 2 + 1] =
- offset(dest, bld, 1);
- }
- }
+ for (unsigned i = 0; i < num_components; i++) {
+ if (!(mask & (1 << (i + first_component))))
+ continue;
- unsigned mlen =
- header_regs + (is_64bit ? 2 * iter_components : iter_components) +
- (is_64bit ? 2 * first_component : first_component);
- fs_reg payload =
- bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
- bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs);
+ srcs[header_regs + i + first_component] = offset(value, bld, i);
+ }
- fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload);
- inst->offset = imm_offset;
- inst->mlen = mlen;
+ unsigned mlen = header_regs + num_components + first_component;
+ fs_reg payload =
+ bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
+ bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs);
- /* If this is a 64-bit attribute, select the next two 64-bit channels
- * to be handled in the next iteration.
- */
- if (is_64bit) {
- mask >>= 2;
- imm_offset++;
- }
- }
+ fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload);
+ inst->offset = imm_offset;
+ inst->mlen = mlen;
break;
}
case nir_intrinsic_load_input:
case nir_intrinsic_load_per_vertex_input: {
+ assert(nir_dest_bit_size(instr->dest) == 32);
fs_reg indirect_offset = get_indirect_offset(instr);
unsigned imm_offset = instr->const_index[0];
unsigned first_component = nir_intrinsic_component(instr);
- if (type_sz(dest.type) == 8) {
- first_component = first_component / 2;
- }
-
fs_inst *inst;
if (indirect_offset.file == BAD_FILE) {
/* Arbitrarily only push up to 32 vec4 slots worth of data,
* which is 16 registers (since each holds 2 vec4 slots).
*/
- unsigned slot_count = 1;
- if (type_sz(dest.type) == 8 && instr->num_components > 2)
- slot_count++;
-
const unsigned max_push_slots = 32;
- if (imm_offset + slot_count <= max_push_slots) {
+ if (imm_offset < max_push_slots) {
fs_reg src = fs_reg(ATTR, imm_offset / 2, dest.type);
for (int i = 0; i < instr->num_components; i++) {
- unsigned comp = 16 / type_sz(dest.type) * (imm_offset % 2) +
- i + first_component;
+ unsigned comp = 4 * (imm_offset % 2) + i + first_component;
bld.MOV(offset(dest, bld, i), component(src, comp));
}
tes_prog_data->base.urb_read_length =
MAX2(tes_prog_data->base.urb_read_length,
- DIV_ROUND_UP(imm_offset + slot_count, 2));
+ (imm_offset / 2) + 1);
} else {
/* Replicate the patch handle to all enabled channels */
const fs_reg srcs[] = {
* we send two read messages in that case, each one loading up to
* two double components.
*/
- unsigned num_iterations = 1;
unsigned num_components = instr->num_components;
- fs_reg orig_dest = dest;
- if (type_sz(dest.type) == 8) {
- if (instr->num_components > 2) {
- num_iterations = 2;
- num_components = 2;
- }
- fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dest.type);
- dest = tmp;
- }
-
- for (unsigned iter = 0; iter < num_iterations; iter++) {
- const fs_reg srcs[] = {
- retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD),
- indirect_offset
- };
- fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
- bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
-
- if (first_component != 0) {
- unsigned read_components =
- num_components + first_component;
- fs_reg tmp = bld.vgrf(dest.type, read_components);
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
- payload);
- for (unsigned i = 0; i < num_components; i++) {
- bld.MOV(offset(dest, bld, i),
- offset(tmp, bld, i + first_component));
- }
- } else {
- inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dest,
- payload);
- }
- inst->mlen = 2;
- inst->offset = imm_offset;
- inst->size_written = (num_components + first_component) *
- inst->dst.component_size(inst->exec_size);
-
- /* If we are reading 64-bit data using 32-bit read messages we need
- * build proper 64-bit data elements by shuffling the low and high
- * 32-bit components around like we do for other things like UBOs
- * or SSBOs.
- */
- if (type_sz(dest.type) == 8) {
- shuffle_32bit_load_result_to_64bit_data(
- bld, dest, retype(dest, BRW_REGISTER_TYPE_F), num_components);
-
- for (unsigned c = 0; c < num_components; c++) {
- bld.MOV(offset(orig_dest, bld, iter * 2 + c),
- offset(dest, bld, c));
- }
- }
+ const fs_reg srcs[] = {
+ retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD),
+ indirect_offset
+ };
+ fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
+ bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
- /* If we are loading double data and we need a second read message
- * adjust the offset
- */
- if (num_iterations > 1) {
- num_components = instr->num_components - 2;
- imm_offset++;
+ if (first_component != 0) {
+ unsigned read_components =
+ num_components + first_component;
+ fs_reg tmp = bld.vgrf(dest.type, read_components);
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
+ payload);
+ for (unsigned i = 0; i < num_components; i++) {
+ bld.MOV(offset(dest, bld, i),
+ offset(tmp, bld, i + first_component));
}
+ } else {
+ inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dest,
+ payload);
}
+ inst->mlen = 2;
+ inst->offset = imm_offset;
+ inst->size_written = (num_components + first_component) *
+ inst->dst.component_size(inst->exec_size);
}
break;
}
wm_prog_data->binding_table.render_target_read_start -
wm_prog_data->base.binding_table.texture_start;
- brw_mark_surface_used(
- bld.shader->stage_prog_data,
- wm_prog_data->binding_table.render_target_read_start + target);
-
/* Calculate the fragment coordinates. */
const fs_reg coords = bld.vgrf(BRW_REGISTER_TYPE_UD, 3);
bld.MOV(offset(coords, bld, 0), pixel_x);
const fs_reg sample = nir_system_values[SYSTEM_VALUE_SAMPLE_ID];
const fs_reg mcs = wm_key->multisample_fbo ?
- emit_mcs_fetch(coords, 3, brw_imm_ud(surface)) : fs_reg();
+ emit_mcs_fetch(coords, 3, brw_imm_ud(surface), fs_reg()) : fs_reg();
/* Use either a normal or a CMS texel fetch message depending on whether
* the framebuffer is single or multisample. On SKL+ use the wide CMS
SHADER_OPCODE_TXF_CMS_LOGICAL;
/* Emit the instruction. */
- const fs_reg srcs[] = { coords, fs_reg(), brw_imm_ud(0), fs_reg(),
- sample, mcs,
- brw_imm_ud(surface), brw_imm_ud(0),
- fs_reg(), brw_imm_ud(3), brw_imm_ud(0) };
- STATIC_ASSERT(ARRAY_SIZE(srcs) == TEX_LOGICAL_NUM_SRCS);
+ fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
+ srcs[TEX_LOGICAL_SRC_COORDINATE] = coords;
+ srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_ud(0);
+ srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = sample;
+ srcs[TEX_LOGICAL_SRC_MCS] = mcs;
+ srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(surface);
+ srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(0);
+ srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_ud(3);
+ srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_ud(0);
fs_inst *inst = bld.emit(op, dst, srcs, ARRAY_SIZE(srcs));
inst->size_written = 4 * inst->dst.component_size(inst->exec_size);
unreachable("Invalid location");
}
+/* Annoyingly, we get the barycentrics into the shader in a layout that's
+ * optimized for PLN but it doesn't work nearly as well as one would like for
+ * manual interpolation.
+ */
+static void
+shuffle_from_pln_layout(const fs_builder &bld, fs_reg dest, fs_reg pln_data)
+{
+ dest.type = BRW_REGISTER_TYPE_F;
+ pln_data.type = BRW_REGISTER_TYPE_F;
+ const fs_reg dest_u = offset(dest, bld, 0);
+ const fs_reg dest_v = offset(dest, bld, 1);
+
+ for (unsigned g = 0; g < bld.dispatch_width() / 8; g++) {
+ const fs_builder gbld = bld.group(8, g);
+ gbld.MOV(horiz_offset(dest_u, g * 8),
+ byte_offset(pln_data, (g * 2 + 0) * REG_SIZE));
+ gbld.MOV(horiz_offset(dest_v, g * 8),
+ byte_offset(pln_data, (g * 2 + 1) * REG_SIZE));
+ }
+}
+
+static void
+shuffle_to_pln_layout(const fs_builder &bld, fs_reg pln_data, fs_reg src)
+{
+ pln_data.type = BRW_REGISTER_TYPE_F;
+ src.type = BRW_REGISTER_TYPE_F;
+ const fs_reg src_u = offset(src, bld, 0);
+ const fs_reg src_v = offset(src, bld, 1);
+
+ for (unsigned g = 0; g < bld.dispatch_width() / 8; g++) {
+ const fs_builder gbld = bld.group(8, g);
+ gbld.MOV(byte_offset(pln_data, (g * 2 + 0) * REG_SIZE),
+ horiz_offset(src_u, g * 8));
+ gbld.MOV(byte_offset(pln_data, (g * 2 + 1) * REG_SIZE),
+ horiz_offset(src_v, g * 8));
+ }
+}
+
void
fs_visitor::nir_emit_fs_intrinsic(const fs_builder &bld,
nir_intrinsic_instr *instr)
bld.MOV(dest, fetch_render_target_array_index(bld));
break;
+ case nir_intrinsic_is_helper_invocation: {
+ /* Unlike the regular gl_HelperInvocation, that is defined at dispatch,
+ * the helperInvocationEXT() (aka SpvOpIsHelperInvocationEXT) takes into
+ * consideration demoted invocations. That information is stored in
+ * f0.1.
+ */
+ dest.type = BRW_REGISTER_TYPE_UD;
+
+ bld.MOV(dest, brw_imm_ud(0));
+
+ fs_inst *mov = bld.MOV(dest, brw_imm_ud(~0));
+ mov->predicate = BRW_PREDICATE_NORMAL;
+ mov->predicate_inverse = true;
+ mov->flag_subreg = 1;
+ break;
+ }
+
case nir_intrinsic_load_helper_invocation:
case nir_intrinsic_load_sample_mask_in:
case nir_intrinsic_load_sample_id: {
case nir_intrinsic_store_output: {
const fs_reg src = get_nir_src(instr->src[0]);
- const nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
- assert(const_offset && "Indirect output stores not allowed");
+ const unsigned store_offset = nir_src_as_uint(instr->src[1]);
const unsigned location = nir_intrinsic_base(instr) +
- SET_FIELD(const_offset->u32[0], BRW_NIR_FRAG_OUTPUT_LOCATION);
+ SET_FIELD(store_offset, BRW_NIR_FRAG_OUTPUT_LOCATION);
const fs_reg new_dest = retype(alloc_frag_output(this, location),
src.type);
const unsigned l = GET_FIELD(nir_intrinsic_base(instr),
BRW_NIR_FRAG_OUTPUT_LOCATION);
assert(l >= FRAG_RESULT_DATA0);
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
- assert(const_offset && "Indirect output loads not allowed");
- const unsigned target = l - FRAG_RESULT_DATA0 + const_offset->u32[0];
+ const unsigned load_offset = nir_src_as_uint(instr->src[0]);
+ const unsigned target = l - FRAG_RESULT_DATA0 + load_offset;
const fs_reg tmp = bld.vgrf(dest.type, 4);
if (reinterpret_cast<const brw_wm_prog_key *>(key)->coherent_fb_fetch)
break;
}
+ case nir_intrinsic_demote:
case nir_intrinsic_discard:
+ case nir_intrinsic_demote_if:
case nir_intrinsic_discard_if: {
/* We track our discarded pixels in f0.1. By predicating on it, we can
* update just the flag bits that aren't yet discarded. If there's no
* condition, we emit a CMP of g0 != g0, so all currently executing
* channels will get turned off.
*/
- fs_inst *cmp;
- if (instr->intrinsic == nir_intrinsic_discard_if) {
- cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
- brw_imm_d(0), BRW_CONDITIONAL_Z);
+ fs_inst *cmp = NULL;
+ if (instr->intrinsic == nir_intrinsic_demote_if ||
+ instr->intrinsic == nir_intrinsic_discard_if) {
+ nir_alu_instr *alu = nir_src_as_alu_instr(instr->src[0]);
+
+ if (alu != NULL &&
+ alu->op != nir_op_bcsel &&
+ alu->op != nir_op_inot) {
+ /* Re-emit the instruction that generated the Boolean value, but
+ * do not store it. Since this instruction will be conditional,
+ * other instructions that want to use the real Boolean value may
+ * get garbage. This was a problem for piglit's fs-discard-exit-2
+ * test.
+ *
+ * Ideally we'd detect that the instruction cannot have a
+ * conditional modifier before emitting the instructions. Alas,
+ * that is nigh impossible. Instead, we're going to assume the
+ * instruction (or last instruction) generated can have a
+ * conditional modifier. If it cannot, fallback to the old-style
+ * compare, and hope dead code elimination will clean up the
+ * extra instructions generated.
+ */
+ nir_emit_alu(bld, alu, false);
+
+ cmp = (fs_inst *) instructions.get_tail();
+ if (cmp->conditional_mod == BRW_CONDITIONAL_NONE) {
+ if (cmp->can_do_cmod())
+ cmp->conditional_mod = BRW_CONDITIONAL_Z;
+ else
+ cmp = NULL;
+ } else {
+ /* The old sequence that would have been generated is,
+ * basically, bool_result == false. This is equivalent to
+ * !bool_result, so negate the old modifier.
+ */
+ cmp->conditional_mod = brw_negate_cmod(cmp->conditional_mod);
+ }
+ }
+
+ if (cmp == NULL) {
+ cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
+ brw_imm_d(0), BRW_CONDITIONAL_Z);
+ }
} else {
fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
BRW_REGISTER_TYPE_UW));
cmp = bld.CMP(bld.null_reg_f(), some_reg, some_reg, BRW_CONDITIONAL_NZ);
}
+
cmp->predicate = BRW_PREDICATE_NORMAL;
cmp->flag_subreg = 1;
if (devinfo->gen >= 6) {
+ /* Due to the way we implement discard, the jump will only happen
+ * when the whole quad is discarded. So we can do this even for
+ * demote as it won't break its uniformity promises.
+ */
emit_discard_jump();
}
+
+ limit_dispatch_width(16, "Fragment discard/demote not implemented in SIMD32 mode.");
break;
}
case nir_intrinsic_load_input: {
/* load_input is only used for flat inputs */
+ assert(nir_dest_bit_size(instr->dest) == 32);
unsigned base = nir_intrinsic_base(instr);
- unsigned component = nir_intrinsic_component(instr);
+ unsigned comp = nir_intrinsic_component(instr);
unsigned num_components = instr->num_components;
- enum brw_reg_type type = dest.type;
/* Special case fields in the VUE header */
if (base == VARYING_SLOT_LAYER)
- component = 1;
+ comp = 1;
else if (base == VARYING_SLOT_VIEWPORT)
- component = 2;
-
- if (nir_dest_bit_size(instr->dest) == 64) {
- /* const_index is in 32-bit type size units that could not be aligned
- * with DF. We need to read the double vector as if it was a float
- * vector of twice the number of components to fetch the right data.
- */
- type = BRW_REGISTER_TYPE_F;
- num_components *= 2;
- }
+ comp = 2;
for (unsigned int i = 0; i < num_components; i++) {
- struct brw_reg interp = interp_reg(base, component + i);
- interp = suboffset(interp, 3);
- bld.emit(FS_OPCODE_CINTERP, offset(retype(dest, type), bld, i),
- retype(fs_reg(interp), type));
+ bld.MOV(offset(dest, bld, i),
+ retype(component(interp_reg(base, comp + i), 3), dest.type));
}
+ break;
+ }
- if (nir_dest_bit_size(instr->dest) == 64) {
- shuffle_32bit_load_result_to_64bit_data(bld,
- dest,
- retype(dest, type),
- instr->num_components);
- }
+ case nir_intrinsic_load_fs_input_interp_deltas: {
+ assert(stage == MESA_SHADER_FRAGMENT);
+ assert(nir_src_as_uint(instr->src[0]) == 0);
+ fs_reg interp = interp_reg(nir_intrinsic_base(instr),
+ nir_intrinsic_component(instr));
+ dest.type = BRW_REGISTER_TYPE_F;
+ bld.MOV(offset(dest, bld, 0), component(interp, 3));
+ bld.MOV(offset(dest, bld, 1), component(interp, 1));
+ bld.MOV(offset(dest, bld, 2), component(interp, 0));
break;
}
case nir_intrinsic_load_barycentric_pixel:
case nir_intrinsic_load_barycentric_centroid:
- case nir_intrinsic_load_barycentric_sample:
- /* Do nothing - load_interpolated_input handling will handle it later. */
+ case nir_intrinsic_load_barycentric_sample: {
+ /* Use the delta_xy values computed from the payload */
+ const glsl_interp_mode interp_mode =
+ (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
+ enum brw_barycentric_mode bary =
+ brw_barycentric_mode(interp_mode, instr->intrinsic);
+
+ shuffle_from_pln_layout(bld, dest, this->delta_xy[bary]);
break;
+ }
case nir_intrinsic_load_barycentric_at_sample: {
const glsl_interp_mode interpolation =
(enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
- nir_const_value *const_sample = nir_src_as_const_value(instr->src[0]);
-
- if (const_sample) {
- unsigned msg_data = const_sample->i32[0] << 4;
+ fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, 2);
+ if (nir_src_is_const(instr->src[0])) {
+ unsigned msg_data = nir_src_as_uint(instr->src[0]) << 4;
emit_pixel_interpolater_send(bld,
FS_OPCODE_INTERPOLATE_AT_SAMPLE,
- dest,
+ tmp,
fs_reg(), /* src */
brw_imm_ud(msg_data),
interpolation);
.SHL(msg_data, sample_id, brw_imm_ud(4u));
emit_pixel_interpolater_send(bld,
FS_OPCODE_INTERPOLATE_AT_SAMPLE,
- dest,
+ tmp,
fs_reg(), /* src */
msg_data,
interpolation);
fs_inst *inst =
emit_pixel_interpolater_send(bld,
FS_OPCODE_INTERPOLATE_AT_SAMPLE,
- dest,
+ tmp,
fs_reg(), /* src */
- msg_data,
+ component(msg_data, 0),
interpolation);
set_predicate(BRW_PREDICATE_NORMAL, inst);
bld.emit(BRW_OPCODE_WHILE));
}
}
+ shuffle_from_pln_layout(bld, dest, tmp);
break;
}
nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
+ fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, 2);
if (const_offset) {
- unsigned off_x = MIN2((int)(const_offset->f32[0] * 16), 7) & 0xf;
- unsigned off_y = MIN2((int)(const_offset->f32[1] * 16), 7) & 0xf;
+ assert(nir_src_bit_size(instr->src[0]) == 32);
+ unsigned off_x = MIN2((int)(const_offset[0].f32 * 16), 7) & 0xf;
+ unsigned off_y = MIN2((int)(const_offset[1].f32 * 16), 7) & 0xf;
emit_pixel_interpolater_send(bld,
FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET,
- dest,
+ tmp,
fs_reg(), /* src */
brw_imm_ud(off_x | (off_y << 4)),
interpolation);
const enum opcode opcode = FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET;
emit_pixel_interpolater_send(bld,
opcode,
- dest,
+ tmp,
src,
brw_imm_ud(0u),
interpolation);
}
+ shuffle_from_pln_layout(bld, dest, tmp);
break;
}
- case nir_intrinsic_load_interpolated_input: {
- if (nir_intrinsic_base(instr) == VARYING_SLOT_POS) {
- emit_fragcoord_interpolation(dest);
- break;
- }
+ case nir_intrinsic_load_frag_coord:
+ emit_fragcoord_interpolation(dest);
+ break;
+ case nir_intrinsic_load_interpolated_input: {
assert(instr->src[0].ssa &&
instr->src[0].ssa->parent_instr->type == nir_instr_type_intrinsic);
nir_intrinsic_instr *bary_intrinsic =
if (bary_intrin == nir_intrinsic_load_barycentric_at_offset ||
bary_intrin == nir_intrinsic_load_barycentric_at_sample) {
- /* Use the result of the PI message */
- dst_xy = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F);
+ /* Use the result of the PI message. Because the load_barycentric
+ * intrinsics return a regular vec2 and we need it in PLN layout, we
+ * have to do a translation. Fortunately, copy-prop cleans this up
+ * reliably.
+ */
+ dst_xy = bld.vgrf(BRW_REGISTER_TYPE_F, 2);
+ shuffle_to_pln_layout(bld, dst_xy, get_nir_src(instr->src[0]));
} else {
/* Use the delta_xy values computed from the payload */
enum brw_barycentric_mode bary =
for (unsigned int i = 0; i < instr->num_components; i++) {
fs_reg interp =
- fs_reg(interp_reg(nir_intrinsic_base(instr),
- nir_intrinsic_component(instr) + i));
+ component(interp_reg(nir_intrinsic_base(instr),
+ nir_intrinsic_component(instr) + i), 0);
interp.type = BRW_REGISTER_TYPE_F;
dest.type = BRW_REGISTER_TYPE_F;
cs_prog_data->uses_num_work_groups = true;
- fs_reg surf_index = brw_imm_ud(surface);
- brw_mark_surface_used(prog_data, surface);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(surface);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); /* num components */
/* Read the 3 GLuint components of gl_NumWorkGroups */
for (unsigned i = 0; i < 3; i++) {
- fs_reg read_result =
- emit_untyped_read(bld, surf_index,
- brw_imm_ud(i << 2),
- 1 /* dims */, 1 /* size */,
- BRW_PREDICATE_NONE);
- read_result.type = dest.type;
- bld.MOV(dest, read_result);
- dest = offset(dest, bld, 1);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = brw_imm_ud(i << 2);
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
+ offset(dest, bld, i), srcs, SURFACE_LOGICAL_NUM_SRCS);
}
break;
}
case nir_intrinsic_shared_atomic_add:
- nir_emit_shared_atomic(bld, BRW_AOP_ADD, instr);
- break;
case nir_intrinsic_shared_atomic_imin:
- nir_emit_shared_atomic(bld, BRW_AOP_IMIN, instr);
- break;
case nir_intrinsic_shared_atomic_umin:
- nir_emit_shared_atomic(bld, BRW_AOP_UMIN, instr);
- break;
case nir_intrinsic_shared_atomic_imax:
- nir_emit_shared_atomic(bld, BRW_AOP_IMAX, instr);
- break;
case nir_intrinsic_shared_atomic_umax:
- nir_emit_shared_atomic(bld, BRW_AOP_UMAX, instr);
- break;
case nir_intrinsic_shared_atomic_and:
- nir_emit_shared_atomic(bld, BRW_AOP_AND, instr);
- break;
case nir_intrinsic_shared_atomic_or:
- nir_emit_shared_atomic(bld, BRW_AOP_OR, instr);
- break;
case nir_intrinsic_shared_atomic_xor:
- nir_emit_shared_atomic(bld, BRW_AOP_XOR, instr);
- break;
case nir_intrinsic_shared_atomic_exchange:
- nir_emit_shared_atomic(bld, BRW_AOP_MOV, instr);
- break;
case nir_intrinsic_shared_atomic_comp_swap:
- nir_emit_shared_atomic(bld, BRW_AOP_CMPWR, instr);
+ nir_emit_shared_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
+ break;
+ case nir_intrinsic_shared_atomic_fmin:
+ case nir_intrinsic_shared_atomic_fmax:
+ case nir_intrinsic_shared_atomic_fcomp_swap:
+ nir_emit_shared_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
break;
case nir_intrinsic_load_shared: {
assert(devinfo->gen >= 7);
+ assert(stage == MESA_SHADER_COMPUTE);
- fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM);
+ const unsigned bit_size = nir_dest_bit_size(instr->dest);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[0]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
- /* Get the offset to read from */
- fs_reg offset_reg;
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
- if (const_offset) {
- offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]);
- } else {
- offset_reg = vgrf(glsl_type::uint_type);
- bld.ADD(offset_reg,
- retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
- brw_imm_ud(instr->const_index[0]));
- }
+ /* Make dest unsigned because that's what the temporary will be */
+ dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
/* Read the vector */
- do_untyped_vector_read(bld, dest, surf_index, offset_reg,
- instr->num_components);
- break;
+ if (nir_intrinsic_align(instr) >= 4) {
+ assert(nir_dest_bit_size(instr->dest) == 32);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+ fs_inst *inst =
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ inst->size_written = instr->num_components * dispatch_width * 4;
+ } else {
+ assert(nir_dest_bit_size(instr->dest) <= 32);
+ assert(nir_dest_num_components(instr->dest) == 1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
+
+ fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
+ bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
+ read_result, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ bld.MOV(dest, subscript(read_result, dest.type, 0));
+ }
+ break;
}
case nir_intrinsic_store_shared: {
assert(devinfo->gen >= 7);
+ assert(stage == MESA_SHADER_COMPUTE);
+
+ const unsigned bit_size = nir_src_bit_size(instr->src[0]);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+
+ fs_reg data = get_nir_src(instr->src[0]);
+ data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
+
+ assert(nir_intrinsic_write_mask(instr) ==
+ (1u << instr->num_components) - 1);
+ if (nir_intrinsic_align(instr) >= 4) {
+ assert(nir_src_bit_size(instr->src[0]) == 32);
+ assert(nir_src_num_components(instr->src[0]) <= 4);
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
+ fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
+ } else {
+ assert(nir_src_bit_size(instr->src[0]) <= 32);
+ assert(nir_src_num_components(instr->src[0]) == 1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
- /* Block index */
- fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM);
+ srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
+ bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
- /* Value */
- fs_reg val_reg = get_nir_src(instr->src[0]);
+ bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
+ fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
+ }
+ break;
+ }
- /* Writemask */
- unsigned writemask = instr->const_index[1];
+ default:
+ nir_emit_intrinsic(bld, instr);
+ break;
+ }
+}
- /* get_nir_src() retypes to integer. Be wary of 64-bit types though
- * since the untyped writes below operate in units of 32-bits, which
- * means that we need to write twice as many components each time.
- * Also, we have to suffle 64-bit data to be in the appropriate layout
- * expected by our 32-bit write messages.
- */
- unsigned type_size = 4;
- if (nir_src_bit_size(instr->src[0]) == 64) {
- type_size = 8;
- val_reg = shuffle_64bit_data_for_32bit_write(bld,
- val_reg, instr->num_components);
+static fs_reg
+brw_nir_reduction_op_identity(const fs_builder &bld,
+ nir_op op, brw_reg_type type)
+{
+ nir_const_value value = nir_alu_binop_identity(op, type_sz(type) * 8);
+ switch (type_sz(type)) {
+ case 1:
+ if (type == BRW_REGISTER_TYPE_UB) {
+ return brw_imm_uw(value.u8);
+ } else {
+ assert(type == BRW_REGISTER_TYPE_B);
+ return brw_imm_w(value.i8);
}
+ case 2:
+ return retype(brw_imm_uw(value.u16), type);
+ case 4:
+ return retype(brw_imm_ud(value.u32), type);
+ case 8:
+ if (type == BRW_REGISTER_TYPE_DF)
+ return setup_imm_df(bld, value.f64);
+ else
+ return retype(brw_imm_u64(value.u64), type);
+ default:
+ unreachable("Invalid type size");
+ }
+}
- unsigned type_slots = type_size / 4;
-
- /* Combine groups of consecutive enabled channels in one write
- * message. We use ffs to find the first enabled channel and then ffs on
- * the bit-inverse, down-shifted writemask to determine the length of
- * the block of enabled bits.
- */
- while (writemask) {
- unsigned first_component = ffs(writemask) - 1;
- unsigned length = ffs(~(writemask >> first_component)) - 1;
+static opcode
+brw_op_for_nir_reduction_op(nir_op op)
+{
+ switch (op) {
+ case nir_op_iadd: return BRW_OPCODE_ADD;
+ case nir_op_fadd: return BRW_OPCODE_ADD;
+ case nir_op_imul: return BRW_OPCODE_MUL;
+ case nir_op_fmul: return BRW_OPCODE_MUL;
+ case nir_op_imin: return BRW_OPCODE_SEL;
+ case nir_op_umin: return BRW_OPCODE_SEL;
+ case nir_op_fmin: return BRW_OPCODE_SEL;
+ case nir_op_imax: return BRW_OPCODE_SEL;
+ case nir_op_umax: return BRW_OPCODE_SEL;
+ case nir_op_fmax: return BRW_OPCODE_SEL;
+ case nir_op_iand: return BRW_OPCODE_AND;
+ case nir_op_ior: return BRW_OPCODE_OR;
+ case nir_op_ixor: return BRW_OPCODE_XOR;
+ default:
+ unreachable("Invalid reduction operation");
+ }
+}
- /* We can't write more than 2 64-bit components at once. Limit the
- * length of the write to what we can do and let the next iteration
- * handle the rest
- */
- if (type_size > 4)
- length = MIN2(2, length);
-
- fs_reg offset_reg;
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
- if (const_offset) {
- offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0] +
- type_size * first_component);
- } else {
- offset_reg = vgrf(glsl_type::uint_type);
- bld.ADD(offset_reg,
- retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD),
- brw_imm_ud(instr->const_index[0] + type_size * first_component));
- }
+static brw_conditional_mod
+brw_cond_mod_for_nir_reduction_op(nir_op op)
+{
+ switch (op) {
+ case nir_op_iadd: return BRW_CONDITIONAL_NONE;
+ case nir_op_fadd: return BRW_CONDITIONAL_NONE;
+ case nir_op_imul: return BRW_CONDITIONAL_NONE;
+ case nir_op_fmul: return BRW_CONDITIONAL_NONE;
+ case nir_op_imin: return BRW_CONDITIONAL_L;
+ case nir_op_umin: return BRW_CONDITIONAL_L;
+ case nir_op_fmin: return BRW_CONDITIONAL_L;
+ case nir_op_imax: return BRW_CONDITIONAL_GE;
+ case nir_op_umax: return BRW_CONDITIONAL_GE;
+ case nir_op_fmax: return BRW_CONDITIONAL_GE;
+ case nir_op_iand: return BRW_CONDITIONAL_NONE;
+ case nir_op_ior: return BRW_CONDITIONAL_NONE;
+ case nir_op_ixor: return BRW_CONDITIONAL_NONE;
+ default:
+ unreachable("Invalid reduction operation");
+ }
+}
- emit_untyped_write(bld, surf_index, offset_reg,
- offset(val_reg, bld, first_component * type_slots),
- 1 /* dims */, length * type_slots,
- BRW_PREDICATE_NONE);
+fs_reg
+fs_visitor::get_nir_image_intrinsic_image(const brw::fs_builder &bld,
+ nir_intrinsic_instr *instr)
+{
+ fs_reg image = retype(get_nir_src_imm(instr->src[0]), BRW_REGISTER_TYPE_UD);
- /* Clear the bits in the writemask that we just wrote, then try
- * again to see if more channels are left.
- */
- writemask &= (15 << (first_component + length));
+ if (stage_prog_data->binding_table.image_start > 0) {
+ if (image.file == BRW_IMMEDIATE_VALUE) {
+ image.d += stage_prog_data->binding_table.image_start;
+ } else {
+ bld.ADD(image, image,
+ brw_imm_d(stage_prog_data->binding_table.image_start));
}
+ }
- break;
+ return bld.emit_uniformize(image);
+}
+
+fs_reg
+fs_visitor::get_nir_ssbo_intrinsic_index(const brw::fs_builder &bld,
+ nir_intrinsic_instr *instr)
+{
+ /* SSBO stores are weird in that their index is in src[1] */
+ const unsigned src = instr->intrinsic == nir_intrinsic_store_ssbo ? 1 : 0;
+
+ fs_reg surf_index;
+ if (nir_src_is_const(instr->src[src])) {
+ unsigned index = stage_prog_data->binding_table.ssbo_start +
+ nir_src_as_uint(instr->src[src]);
+ surf_index = brw_imm_ud(index);
+ } else {
+ surf_index = vgrf(glsl_type::uint_type);
+ bld.ADD(surf_index, get_nir_src(instr->src[src]),
+ brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
}
+ return bld.emit_uniformize(surf_index);
+}
+
+static unsigned
+image_intrinsic_coord_components(nir_intrinsic_instr *instr)
+{
+ switch (nir_intrinsic_image_dim(instr)) {
+ case GLSL_SAMPLER_DIM_1D:
+ return 1 + nir_intrinsic_image_array(instr);
+ case GLSL_SAMPLER_DIM_2D:
+ case GLSL_SAMPLER_DIM_RECT:
+ return 2 + nir_intrinsic_image_array(instr);
+ case GLSL_SAMPLER_DIM_3D:
+ case GLSL_SAMPLER_DIM_CUBE:
+ return 3;
+ case GLSL_SAMPLER_DIM_BUF:
+ return 1;
+ case GLSL_SAMPLER_DIM_MS:
+ return 2 + nir_intrinsic_image_array(instr);
default:
- nir_emit_intrinsic(bld, instr);
- break;
+ unreachable("Invalid image dimension");
}
}
case nir_intrinsic_image_load:
case nir_intrinsic_image_store:
case nir_intrinsic_image_atomic_add:
- case nir_intrinsic_image_atomic_min:
- case nir_intrinsic_image_atomic_max:
+ case nir_intrinsic_image_atomic_imin:
+ case nir_intrinsic_image_atomic_umin:
+ case nir_intrinsic_image_atomic_imax:
+ case nir_intrinsic_image_atomic_umax:
case nir_intrinsic_image_atomic_and:
case nir_intrinsic_image_atomic_or:
case nir_intrinsic_image_atomic_xor:
case nir_intrinsic_image_atomic_exchange:
- case nir_intrinsic_image_atomic_comp_swap: {
- using namespace image_access;
-
+ case nir_intrinsic_image_atomic_comp_swap:
+ case nir_intrinsic_bindless_image_load:
+ case nir_intrinsic_bindless_image_store:
+ case nir_intrinsic_bindless_image_atomic_add:
+ case nir_intrinsic_bindless_image_atomic_imin:
+ case nir_intrinsic_bindless_image_atomic_umin:
+ case nir_intrinsic_bindless_image_atomic_imax:
+ case nir_intrinsic_bindless_image_atomic_umax:
+ case nir_intrinsic_bindless_image_atomic_and:
+ case nir_intrinsic_bindless_image_atomic_or:
+ case nir_intrinsic_bindless_image_atomic_xor:
+ case nir_intrinsic_bindless_image_atomic_exchange:
+ case nir_intrinsic_bindless_image_atomic_comp_swap: {
if (stage == MESA_SHADER_FRAGMENT &&
instr->intrinsic != nir_intrinsic_image_load)
brw_wm_prog_data(prog_data)->has_side_effects = true;
- /* Get the referenced image variable and type. */
- const nir_variable *var = instr->variables[0]->var;
- const glsl_type *type = var->type->without_array();
- const brw_reg_type base_type = get_image_base_type(type);
-
/* Get some metadata from the image intrinsic. */
const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
- const unsigned arr_dims = type->sampler_array ? 1 : 0;
- const unsigned surf_dims = type->coordinate_components() - arr_dims;
- const unsigned format = var->data.image.format;
-
- /* Get the arguments of the image intrinsic. */
- const fs_reg image = get_nir_image_deref(instr->variables[0]);
- const fs_reg addr = retype(get_nir_src(instr->src[0]),
- BRW_REGISTER_TYPE_UD);
- const fs_reg src0 = (info->num_srcs >= 3 ?
- retype(get_nir_src(instr->src[2]), base_type) :
- fs_reg());
- const fs_reg src1 = (info->num_srcs >= 4 ?
- retype(get_nir_src(instr->src[3]), base_type) :
- fs_reg());
- fs_reg tmp;
+
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+
+ switch (instr->intrinsic) {
+ case nir_intrinsic_image_load:
+ case nir_intrinsic_image_store:
+ case nir_intrinsic_image_atomic_add:
+ case nir_intrinsic_image_atomic_imin:
+ case nir_intrinsic_image_atomic_umin:
+ case nir_intrinsic_image_atomic_imax:
+ case nir_intrinsic_image_atomic_umax:
+ case nir_intrinsic_image_atomic_and:
+ case nir_intrinsic_image_atomic_or:
+ case nir_intrinsic_image_atomic_xor:
+ case nir_intrinsic_image_atomic_exchange:
+ case nir_intrinsic_image_atomic_comp_swap:
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] =
+ get_nir_image_intrinsic_image(bld, instr);
+ break;
+
+ default:
+ /* Bindless */
+ srcs[SURFACE_LOGICAL_SRC_SURFACE_HANDLE] =
+ bld.emit_uniformize(get_nir_src(instr->src[0]));
+ break;
+ }
+
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] =
+ brw_imm_ud(image_intrinsic_coord_components(instr));
/* Emit an image load, store or atomic op. */
- if (instr->intrinsic == nir_intrinsic_image_load)
- tmp = emit_image_load(bld, image, addr, surf_dims, arr_dims, format);
+ if (instr->intrinsic == nir_intrinsic_image_load ||
+ instr->intrinsic == nir_intrinsic_bindless_image_load) {
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+ fs_inst *inst =
+ bld.emit(SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ inst->size_written = instr->num_components * dispatch_width * 4;
+ } else if (instr->intrinsic == nir_intrinsic_image_store ||
+ instr->intrinsic == nir_intrinsic_bindless_image_store) {
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+ srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[3]);
+ bld.emit(SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL,
+ fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
+ } else {
+ unsigned num_srcs = info->num_srcs;
+ int op = brw_aop_for_nir_intrinsic(instr);
+ if (op == BRW_AOP_INC || op == BRW_AOP_DEC) {
+ assert(num_srcs == 4);
+ num_srcs = 3;
+ }
+
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
- else if (instr->intrinsic == nir_intrinsic_image_store)
- emit_image_store(bld, image, addr, src0, surf_dims, arr_dims,
- var->data.image.write_only ? GL_NONE : format);
+ fs_reg data;
+ if (num_srcs >= 4)
+ data = get_nir_src(instr->src[3]);
+ if (num_srcs >= 5) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[4]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
+ }
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+ bld.emit(SHADER_OPCODE_TYPED_ATOMIC_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ }
+ break;
+ }
+
+ case nir_intrinsic_image_size:
+ case nir_intrinsic_bindless_image_size: {
+ /* Unlike the [un]typed load and store opcodes, the TXS that this turns
+ * into will handle the binding table index for us in the geneerator.
+ * Incidentally, this means that we can handle bindless with exactly the
+ * same code.
+ */
+ fs_reg image = retype(get_nir_src_imm(instr->src[0]),
+ BRW_REGISTER_TYPE_UD);
+ image = bld.emit_uniformize(image);
+
+ fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
+ if (instr->intrinsic == nir_intrinsic_image_size)
+ srcs[TEX_LOGICAL_SRC_SURFACE] = image;
else
- tmp = emit_image_atomic(bld, image, addr, src0, src1,
- surf_dims, arr_dims, info->dest_components,
- get_image_atomic_op(instr->intrinsic, type));
+ srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = image;
+ srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_d(0);
+ srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(0);
+ srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(0);
- /* Assign the result. */
- for (unsigned c = 0; c < info->dest_components; ++c)
- bld.MOV(offset(retype(dest, base_type), bld, c),
- offset(tmp, bld, c));
+ /* Since the image size is always uniform, we can just emit a SIMD8
+ * query instruction and splat the result out.
+ */
+ const fs_builder ubld = bld.exec_all().group(8, 0);
+
+ fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 4);
+ fs_inst *inst = ubld.emit(SHADER_OPCODE_IMAGE_SIZE_LOGICAL,
+ tmp, srcs, ARRAY_SIZE(srcs));
+ inst->size_written = 4 * REG_SIZE;
+
+ for (unsigned c = 0; c < instr->dest.ssa.num_components; ++c) {
+ if (c == 2 && nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_CUBE) {
+ bld.emit(SHADER_OPCODE_INT_QUOTIENT,
+ offset(retype(dest, tmp.type), bld, c),
+ component(offset(tmp, ubld, c), 0), brw_imm_ud(6));
+ } else {
+ bld.MOV(offset(retype(dest, tmp.type), bld, c),
+ component(offset(tmp, ubld, c), 0));
+ }
+ }
break;
}
- case nir_intrinsic_memory_barrier_atomic_counter:
- case nir_intrinsic_memory_barrier_buffer:
- case nir_intrinsic_memory_barrier_image:
- case nir_intrinsic_memory_barrier: {
- const fs_builder ubld = bld.group(8, 0);
- const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
- ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp)
- ->size_written = 2 * REG_SIZE;
+ case nir_intrinsic_image_load_raw_intel: {
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] =
+ get_nir_image_intrinsic_image(bld, instr);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+
+ fs_inst *inst =
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ inst->size_written = instr->num_components * dispatch_width * 4;
+ break;
+ }
+
+ case nir_intrinsic_image_store_raw_intel: {
+ if (stage == MESA_SHADER_FRAGMENT)
+ brw_wm_prog_data(prog_data)->has_side_effects = true;
+
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] =
+ get_nir_image_intrinsic_image(bld, instr);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[2]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
+ fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
break;
}
case nir_intrinsic_group_memory_barrier:
case nir_intrinsic_memory_barrier_shared:
- /* We treat these workgroup-level barriers as no-ops. This should be
- * safe at present and as long as:
- *
- * - Memory access instructions are not subsequently reordered by the
- * compiler back-end.
+ case nir_intrinsic_memory_barrier_atomic_counter:
+ case nir_intrinsic_memory_barrier_buffer:
+ case nir_intrinsic_memory_barrier_image:
+ case nir_intrinsic_memory_barrier: {
+ bool l3_fence, slm_fence;
+ if (devinfo->gen >= 11) {
+ l3_fence = instr->intrinsic != nir_intrinsic_memory_barrier_shared;
+ slm_fence = instr->intrinsic == nir_intrinsic_group_memory_barrier ||
+ instr->intrinsic == nir_intrinsic_memory_barrier ||
+ instr->intrinsic == nir_intrinsic_memory_barrier_shared;
+ } else {
+ /* Prior to gen11, we only have one kind of fence. */
+ l3_fence = true;
+ slm_fence = false;
+ }
+
+ /* Be conservative in Gen11+ and always stall in a fence. Since there
+ * are two different fences, and shader might want to synchronize
+ * between them.
*
- * - All threads from a given compute shader workgroup fit within a
- * single subslice and therefore talk to the same HDC shared unit
- * what supposedly guarantees ordering and coherency between threads
- * from the same workgroup. This may change in the future when we
- * start splitting workgroups across multiple subslices.
+ * TODO: Improve NIR so that scope and visibility information for the
+ * barriers is available here to make a better decision.
*
- * - The context is not in fault-and-stream mode, which could cause
- * memory transactions (including to SLM) prior to the barrier to be
- * replayed after the barrier if a pagefault occurs. This shouldn't
- * be a problem up to and including SKL because fault-and-stream is
- * not usable due to hardware issues, but that's likely to change in
- * the future.
+ * TODO: When emitting more than one fence, it might help emit all
+ * the fences first and then generate the stall moves.
*/
+ const bool stall = devinfo->gen >= 11;
+
+ const fs_builder ubld = bld.group(8, 0);
+ const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
+
+ if (l3_fence) {
+ ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp,
+ brw_vec8_grf(0, 0), brw_imm_ud(stall),
+ /* bti */ brw_imm_ud(0))
+ ->size_written = 2 * REG_SIZE;
+ }
+
+ if (slm_fence) {
+ ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp,
+ brw_vec8_grf(0, 0), brw_imm_ud(stall),
+ brw_imm_ud(GEN7_BTI_SLM))
+ ->size_written = 2 * REG_SIZE;
+ }
+
break;
+ }
case nir_intrinsic_shader_clock: {
/* We cannot do anything if there is an event, so ignore it for now */
break;
}
- case nir_intrinsic_image_size: {
- /* Get the referenced image variable and type. */
- const nir_variable *var = instr->variables[0]->var;
- const glsl_type *type = var->type->without_array();
-
- /* Get the size of the image. */
- const fs_reg image = get_nir_image_deref(instr->variables[0]);
- const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);
-
- /* For 1DArray image types, the array index is stored in the Z component.
- * Fix this by swizzling the Z component to the Y component.
- */
- const bool is_1d_array_image =
- type->sampler_dimensionality == GLSL_SAMPLER_DIM_1D &&
- type->sampler_array;
-
- /* For CubeArray images, we should count the number of cubes instead
- * of the number of faces. Fix it by dividing the (Z component) by 6.
- */
- const bool is_cube_array_image =
- type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE &&
- type->sampler_array;
-
- /* Copy all the components. */
- for (unsigned c = 0; c < instr->dest.ssa.num_components; ++c) {
- if ((int)c >= type->coordinate_components()) {
- bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
- brw_imm_d(1));
- } else if (c == 1 && is_1d_array_image) {
- bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
- offset(size, bld, 2));
- } else if (c == 2 && is_cube_array_image) {
- bld.emit(SHADER_OPCODE_INT_QUOTIENT,
- offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
- offset(size, bld, c), brw_imm_d(6));
- } else {
- bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
- offset(size, bld, c));
- }
- }
-
- break;
- }
-
case nir_intrinsic_image_samples:
/* The driver does not support multi-sampled images. */
bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(1));
fs_reg src(UNIFORM, instr->const_index[0] / 4, dest.type);
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
- if (const_offset) {
- assert(const_offset->u32[0] % type_sz(dest.type) == 0);
+ if (nir_src_is_const(instr->src[0])) {
+ unsigned load_offset = nir_src_as_uint(instr->src[0]);
+ assert(load_offset % type_sz(dest.type) == 0);
/* For 16-bit types we add the module of the const_index[0]
* offset to access to not 32-bit aligned element
*/
- src.offset = const_offset->u32[0] + instr->const_index[0] % 4;
+ src.offset = load_offset + instr->const_index[0] % 4;
for (unsigned j = 0; j < instr->num_components; j++) {
bld.MOV(offset(dest, bld, j), offset(src, bld, j));
}
case nir_intrinsic_load_ubo: {
- nir_const_value *const_index = nir_src_as_const_value(instr->src[0]);
fs_reg surf_index;
-
- if (const_index) {
+ if (nir_src_is_const(instr->src[0])) {
const unsigned index = stage_prog_data->binding_table.ubo_start +
- const_index->u32[0];
+ nir_src_as_uint(instr->src[0]);
surf_index = brw_imm_ud(index);
- brw_mark_surface_used(prog_data, index);
} else {
/* The block index is not a constant. Evaluate the index expression
* per-channel and add the base UBO index; we have to select a value
bld.ADD(surf_index, get_nir_src(instr->src[0]),
brw_imm_ud(stage_prog_data->binding_table.ubo_start));
surf_index = bld.emit_uniformize(surf_index);
-
- /* Assume this may touch any UBO. It would be nice to provide
- * a tighter bound, but the array information is already lowered away.
- */
- brw_mark_surface_used(prog_data,
- stage_prog_data->binding_table.ubo_start +
- nir->info.num_ubos - 1);
}
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
- if (const_offset == NULL) {
+ if (!nir_src_is_const(instr->src[1])) {
fs_reg base_offset = retype(get_nir_src(instr->src[1]),
BRW_REGISTER_TYPE_UD);
for (int i = 0; i < instr->num_components; i++)
VARYING_PULL_CONSTANT_LOAD(bld, offset(dest, bld, i), surf_index,
base_offset, i * type_sz(dest.type));
+
+ prog_data->has_ubo_pull = true;
} else {
/* Even if we are loading doubles, a pull constant load will load
* a 32-bit vec4, so should only reserve vgrf space for that. If we
* and we have to split it if necessary.
*/
const unsigned type_size = type_sz(dest.type);
+ const unsigned load_offset = nir_src_as_uint(instr->src[1]);
/* See if we've selected this as a push constant candidate */
- if (const_index) {
- const unsigned ubo_block = const_index->u32[0];
- const unsigned offset_256b = const_offset->u32[0] / 32;
+ if (nir_src_is_const(instr->src[0])) {
+ const unsigned ubo_block = nir_src_as_uint(instr->src[0]);
+ const unsigned offset_256b = load_offset / 32;
fs_reg push_reg;
for (int i = 0; i < 4; i++) {
offset_256b < range->start + range->length) {
push_reg = fs_reg(UNIFORM, UBO_START + i, dest.type);
- push_reg.offset = const_offset->u32[0] - 32 * range->start;
+ push_reg.offset = load_offset - 32 * range->start;
break;
}
}
}
}
+ prog_data->has_ubo_pull = true;
+
const unsigned block_sz = 64; /* Fetch one cacheline at a time. */
const fs_builder ubld = bld.exec_all().group(block_sz / 4, 0);
const fs_reg packed_consts = ubld.vgrf(BRW_REGISTER_TYPE_UD);
for (unsigned c = 0; c < instr->num_components;) {
- const unsigned base = const_offset->u32[0] + c * type_size;
+ const unsigned base = load_offset + c * type_size;
/* Number of usable components in the next block-aligned load. */
const unsigned count = MIN2(instr->num_components - c,
(block_sz - base % block_sz) / type_size);
break;
}
- case nir_intrinsic_load_ssbo: {
- assert(devinfo->gen >= 7);
-
- nir_const_value *const_uniform_block =
- nir_src_as_const_value(instr->src[0]);
-
- fs_reg surf_index;
- if (const_uniform_block) {
- unsigned index = stage_prog_data->binding_table.ssbo_start +
- const_uniform_block->u32[0];
- surf_index = brw_imm_ud(index);
- brw_mark_surface_used(prog_data, index);
- } else {
- surf_index = vgrf(glsl_type::uint_type);
- bld.ADD(surf_index, get_nir_src(instr->src[0]),
- brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
-
- /* Assume this may touch any UBO. It would be nice to provide
- * a tighter bound, but the array information is already lowered away.
- */
- brw_mark_surface_used(prog_data,
- stage_prog_data->binding_table.ssbo_start +
- nir->info.num_ssbos - 1);
- }
-
- fs_reg offset_reg;
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
- if (const_offset) {
- offset_reg = brw_imm_ud(const_offset->u32[0]);
+ case nir_intrinsic_load_global: {
+ assert(devinfo->gen >= 8);
+
+ if (nir_intrinsic_align(instr) >= 4) {
+ assert(nir_dest_bit_size(instr->dest) == 32);
+ fs_inst *inst = bld.emit(SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL,
+ dest,
+ get_nir_src(instr->src[0]), /* Address */
+ fs_reg(), /* No source data */
+ brw_imm_ud(instr->num_components));
+ inst->size_written = instr->num_components *
+ inst->dst.component_size(inst->exec_size);
} else {
- offset_reg = get_nir_src(instr->src[1]);
+ const unsigned bit_size = nir_dest_bit_size(instr->dest);
+ assert(bit_size <= 32);
+ assert(nir_dest_num_components(instr->dest) == 1);
+ fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
+ bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL,
+ tmp,
+ get_nir_src(instr->src[0]), /* Address */
+ fs_reg(), /* No source data */
+ brw_imm_ud(bit_size));
+ bld.MOV(dest, subscript(tmp, dest.type, 0));
}
-
- /* Read the vector */
- do_untyped_vector_read(bld, dest, surf_index, offset_reg,
- instr->num_components);
-
break;
}
- case nir_intrinsic_store_ssbo: {
- assert(devinfo->gen >= 7);
+ case nir_intrinsic_store_global:
+ assert(devinfo->gen >= 8);
if (stage == MESA_SHADER_FRAGMENT)
brw_wm_prog_data(prog_data)->has_side_effects = true;
- /* Block index */
- fs_reg surf_index;
- nir_const_value *const_uniform_block =
- nir_src_as_const_value(instr->src[1]);
- if (const_uniform_block) {
- unsigned index = stage_prog_data->binding_table.ssbo_start +
- const_uniform_block->u32[0];
- surf_index = brw_imm_ud(index);
- brw_mark_surface_used(prog_data, index);
+ if (nir_intrinsic_align(instr) >= 4) {
+ assert(nir_src_bit_size(instr->src[0]) == 32);
+ bld.emit(SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL,
+ fs_reg(),
+ get_nir_src(instr->src[1]), /* Address */
+ get_nir_src(instr->src[0]), /* Data */
+ brw_imm_ud(instr->num_components));
} else {
- surf_index = vgrf(glsl_type::uint_type);
- bld.ADD(surf_index, get_nir_src(instr->src[1]),
- brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
-
- brw_mark_surface_used(prog_data,
- stage_prog_data->binding_table.ssbo_start +
- nir->info.num_ssbos - 1);
+ const unsigned bit_size = nir_src_bit_size(instr->src[0]);
+ assert(bit_size <= 32);
+ assert(nir_src_num_components(instr->src[0]) == 1);
+ brw_reg_type data_type =
+ brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
+ fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
+ bld.MOV(tmp, retype(get_nir_src(instr->src[0]), data_type));
+ bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL,
+ fs_reg(),
+ get_nir_src(instr->src[1]), /* Address */
+ tmp, /* Data */
+ brw_imm_ud(nir_src_bit_size(instr->src[0])));
}
+ break;
- /* Value */
- fs_reg val_reg = get_nir_src(instr->src[0]);
-
- /* Writemask */
- unsigned writemask = instr->const_index[0];
+ case nir_intrinsic_global_atomic_add:
+ case nir_intrinsic_global_atomic_imin:
+ case nir_intrinsic_global_atomic_umin:
+ case nir_intrinsic_global_atomic_imax:
+ case nir_intrinsic_global_atomic_umax:
+ case nir_intrinsic_global_atomic_and:
+ case nir_intrinsic_global_atomic_or:
+ case nir_intrinsic_global_atomic_xor:
+ case nir_intrinsic_global_atomic_exchange:
+ case nir_intrinsic_global_atomic_comp_swap:
+ nir_emit_global_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
+ break;
+ case nir_intrinsic_global_atomic_fmin:
+ case nir_intrinsic_global_atomic_fmax:
+ case nir_intrinsic_global_atomic_fcomp_swap:
+ nir_emit_global_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
+ break;
- /* get_nir_src() retypes to integer. Be wary of 64-bit types though
- * since the untyped writes below operate in units of 32-bits, which
- * means that we need to write twice as many components each time.
- * Also, we have to suffle 64-bit data to be in the appropriate layout
- * expected by our 32-bit write messages.
- */
- unsigned bit_size = nir_src_bit_size(instr->src[0]);
- unsigned type_size = bit_size / 8;
+ case nir_intrinsic_load_ssbo: {
+ assert(devinfo->gen >= 7);
- /* Combine groups of consecutive enabled channels in one write
- * message. We use ffs to find the first enabled channel and then ffs on
- * the bit-inverse, down-shifted writemask to determine the num_components
- * of the block of enabled bits.
- */
- while (writemask) {
- unsigned first_component = ffs(writemask) - 1;
- unsigned num_components = ffs(~(writemask >> first_component)) - 1;
- fs_reg write_src = offset(val_reg, bld, first_component);
+ const unsigned bit_size = nir_dest_bit_size(instr->dest);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] =
+ get_nir_ssbo_intrinsic_index(bld, instr);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[2]);
+ /* Make dest unsigned because that's what the temporary will be */
+ dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
- if (type_size > 4) {
- /* We can't write more than 2 64-bit components at once. Limit
- * the num_components of the write to what we can do and let the next
- * iteration handle the rest.
- */
- num_components = MIN2(2, num_components);
- write_src = shuffle_64bit_data_for_32bit_write(bld, write_src,
- num_components);
- } else if (type_size < 4) {
- assert(type_size == 2);
- /* For 16-bit types we pack two consecutive values into a 32-bit
- * word and use an untyped write message. For single values or not
- * 32-bit-aligned we need to use byte-scattered writes because
- * untyped writes works with 32-bit components with 32-bit
- * alignment. byte_scattered_write messages only support one
- * 16-bit component at a time. As VK_KHR_relaxed_block_layout
- * could be enabled we can not guarantee that not constant offsets
- * to be 32-bit aligned for 16-bit types. For example an array, of
- * 16-bit vec3 with array element stride of 6.
- *
- * In the case of 32-bit aligned constant offsets if there is
- * a 3-components vector we submit one untyped-write message
- * of 32-bit (first two components), and one byte-scattered
- * write message (the last component).
- */
+ /* Read the vector */
+ if (nir_intrinsic_align(instr) >= 4) {
+ assert(nir_dest_bit_size(instr->dest) == 32);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+ fs_inst *inst =
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ inst->size_written = instr->num_components * dispatch_width * 4;
+ } else {
+ assert(nir_dest_bit_size(instr->dest) <= 32);
+ assert(nir_dest_num_components(instr->dest) == 1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
+
+ fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
+ bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
+ read_result, srcs, SURFACE_LOGICAL_NUM_SRCS);
+ bld.MOV(dest, subscript(read_result, dest.type, 0));
+ }
+ break;
+ }
- if ( !const_offset || ((const_offset->u32[0] +
- type_size * first_component) % 4)) {
- /* If we use a .yz writemask we also need to emit 2
- * byte-scattered write messages because of y-component not
- * being aligned to 32-bit.
- */
- num_components = 1;
- } else if (num_components > 2 && (num_components % 2)) {
- /* If there is an odd number of consecutive components we left
- * the not paired component for a following emit of length == 1
- * with byte_scattered_write.
- */
- num_components --;
- }
- /* For num_components == 1 we are also shuffling the component
- * because byte scattered writes of 16-bit need values to be dword
- * aligned. Shuffling only one component would be the same as
- * striding it.
- */
- fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_D,
- DIV_ROUND_UP(num_components, 2));
- shuffle_16bit_data_for_32bit_write(bld, tmp, write_src,
- num_components);
- write_src = tmp;
- }
+ case nir_intrinsic_store_ssbo: {
+ assert(devinfo->gen >= 7);
- fs_reg offset_reg;
+ if (stage == MESA_SHADER_FRAGMENT)
+ brw_wm_prog_data(prog_data)->has_side_effects = true;
- if (const_offset) {
- offset_reg = brw_imm_ud(const_offset->u32[0] +
- type_size * first_component);
- } else {
- offset_reg = vgrf(glsl_type::uint_type);
- bld.ADD(offset_reg,
- retype(get_nir_src(instr->src[2]), BRW_REGISTER_TYPE_UD),
- brw_imm_ud(type_size * first_component));
- }
+ const unsigned bit_size = nir_src_bit_size(instr->src[0]);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] =
+ get_nir_ssbo_intrinsic_index(bld, instr);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[2]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+
+ fs_reg data = get_nir_src(instr->src[0]);
+ data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
+
+ assert(nir_intrinsic_write_mask(instr) ==
+ (1u << instr->num_components) - 1);
+ if (nir_intrinsic_align(instr) >= 4) {
+ assert(nir_src_bit_size(instr->src[0]) == 32);
+ assert(nir_src_num_components(instr->src[0]) <= 4);
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
+ bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
+ fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
+ } else {
+ assert(nir_src_bit_size(instr->src[0]) <= 32);
+ assert(nir_src_num_components(instr->src[0]) == 1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
- if (type_size < 4 && num_components == 1) {
- assert(type_size == 2);
- /* Untyped Surface messages have a fixed 32-bit size, so we need
- * to rely on byte scattered in order to write 16-bit elements.
- * The byte_scattered_write message needs that every written 16-bit
- * type to be aligned 32-bits (stride=2).
- */
- emit_byte_scattered_write(bld, surf_index, offset_reg,
- write_src,
- 1 /* dims */, 1,
- bit_size,
- BRW_PREDICATE_NONE);
- } else {
- assert(num_components * type_size <= 16);
- assert((num_components * type_size) % 4 == 0);
- assert(offset_reg.file != BRW_IMMEDIATE_VALUE ||
- offset_reg.ud % 4 == 0);
- unsigned num_slots = (num_components * type_size) / 4;
-
- emit_untyped_write(bld, surf_index, offset_reg,
- write_src,
- 1 /* dims */, num_slots,
- BRW_PREDICATE_NONE);
- }
+ srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
+ bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
- /* Clear the bits in the writemask that we just wrote, then try
- * again to see if more channels are left.
- */
- writemask &= (15 << (first_component + num_components));
+ bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
+ fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
}
break;
}
case nir_intrinsic_store_output: {
+ assert(nir_src_bit_size(instr->src[0]) == 32);
fs_reg src = get_nir_src(instr->src[0]);
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
- assert(const_offset && "Indirect output stores not allowed");
-
+ unsigned store_offset = nir_src_as_uint(instr->src[1]);
unsigned num_components = instr->num_components;
unsigned first_component = nir_intrinsic_component(instr);
- if (nir_src_bit_size(instr->src[0]) == 64) {
- src = shuffle_64bit_data_for_32bit_write(bld, src, num_components);
- num_components *= 2;
- }
fs_reg new_dest = retype(offset(outputs[instr->const_index[0]], bld,
- 4 * const_offset->u32[0]), src.type);
+ 4 * store_offset), src.type);
for (unsigned j = 0; j < num_components; j++) {
bld.MOV(offset(new_dest, bld, j + first_component),
offset(src, bld, j));
}
case nir_intrinsic_ssbo_atomic_add:
- nir_emit_ssbo_atomic(bld, BRW_AOP_ADD, instr);
- break;
case nir_intrinsic_ssbo_atomic_imin:
- nir_emit_ssbo_atomic(bld, BRW_AOP_IMIN, instr);
- break;
case nir_intrinsic_ssbo_atomic_umin:
- nir_emit_ssbo_atomic(bld, BRW_AOP_UMIN, instr);
- break;
case nir_intrinsic_ssbo_atomic_imax:
- nir_emit_ssbo_atomic(bld, BRW_AOP_IMAX, instr);
- break;
case nir_intrinsic_ssbo_atomic_umax:
- nir_emit_ssbo_atomic(bld, BRW_AOP_UMAX, instr);
- break;
case nir_intrinsic_ssbo_atomic_and:
- nir_emit_ssbo_atomic(bld, BRW_AOP_AND, instr);
- break;
case nir_intrinsic_ssbo_atomic_or:
- nir_emit_ssbo_atomic(bld, BRW_AOP_OR, instr);
- break;
case nir_intrinsic_ssbo_atomic_xor:
- nir_emit_ssbo_atomic(bld, BRW_AOP_XOR, instr);
- break;
case nir_intrinsic_ssbo_atomic_exchange:
- nir_emit_ssbo_atomic(bld, BRW_AOP_MOV, instr);
- break;
case nir_intrinsic_ssbo_atomic_comp_swap:
- nir_emit_ssbo_atomic(bld, BRW_AOP_CMPWR, instr);
+ nir_emit_ssbo_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
+ break;
+ case nir_intrinsic_ssbo_atomic_fmin:
+ case nir_intrinsic_ssbo_atomic_fmax:
+ case nir_intrinsic_ssbo_atomic_fcomp_swap:
+ nir_emit_ssbo_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
break;
case nir_intrinsic_get_buffer_size: {
- nir_const_value *const_uniform_block = nir_src_as_const_value(instr->src[0]);
- unsigned ssbo_index = const_uniform_block ? const_uniform_block->u32[0] : 0;
+ assert(nir_src_num_components(instr->src[0]) == 1);
+ unsigned ssbo_index = nir_src_is_const(instr->src[0]) ?
+ nir_src_as_uint(instr->src[0]) : 0;
/* A resinfo's sampler message is used to get the buffer size. The
* SIMD8's writeback message consists of four registers and SIMD16's
ubld.ADD(buffer_size, size_aligned4, negate(size_padding));
bld.MOV(retype(dest, ret_payload.type), component(buffer_size, 0));
-
- brw_mark_surface_used(prog_data, index);
break;
}
+ case nir_intrinsic_load_subgroup_size:
+ /* This should only happen for fragment shaders because every other case
+ * is lowered in NIR so we can optimize on it.
+ */
+ assert(stage == MESA_SHADER_FRAGMENT);
+ bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(dispatch_width));
+ break;
+
case nir_intrinsic_load_subgroup_invocation:
bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION]);
bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
break;
}
+ case nir_intrinsic_vote_feq:
case nir_intrinsic_vote_ieq: {
fs_reg value = get_nir_src(instr->src[0]);
+ if (instr->intrinsic == nir_intrinsic_vote_feq) {
+ const unsigned bit_size = nir_src_bit_size(instr->src[0]);
+ value.type = bit_size == 8 ? BRW_REGISTER_TYPE_B :
+ brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_F);
+ }
+
fs_reg uniformized = bld.emit_uniformize(value);
const fs_builder ubld = bld.exec_all().group(1, 0);
break;
}
+ case nir_intrinsic_shuffle: {
+ const fs_reg value = get_nir_src(instr->src[0]);
+ const fs_reg index = get_nir_src(instr->src[1]);
+
+ bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, index);
+ break;
+ }
+
case nir_intrinsic_first_invocation: {
fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
bld.exec_all().emit(SHADER_OPCODE_FIND_LIVE_CHANNEL, tmp);
break;
}
+ case nir_intrinsic_quad_broadcast: {
+ const fs_reg value = get_nir_src(instr->src[0]);
+ const unsigned index = nir_src_as_uint(instr->src[1]);
+
+ bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, retype(dest, value.type),
+ value, brw_imm_ud(index), brw_imm_ud(4));
+ break;
+ }
+
+ case nir_intrinsic_quad_swap_horizontal: {
+ const fs_reg value = get_nir_src(instr->src[0]);
+ const fs_reg tmp = bld.vgrf(value.type);
+ if (devinfo->gen <= 7) {
+ /* The hardware doesn't seem to support these crazy regions with
+ * compressed instructions on gen7 and earlier so we fall back to
+ * using quad swizzles. Fortunately, we don't support 64-bit
+ * anything in Vulkan on gen7.
+ */
+ assert(nir_src_bit_size(instr->src[0]) == 32);
+ const fs_builder ubld = bld.exec_all();
+ ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value,
+ brw_imm_ud(BRW_SWIZZLE4(1,0,3,2)));
+ bld.MOV(retype(dest, value.type), tmp);
+ } else {
+ const fs_builder ubld = bld.exec_all().group(dispatch_width / 2, 0);
+
+ const fs_reg src_left = horiz_stride(value, 2);
+ const fs_reg src_right = horiz_stride(horiz_offset(value, 1), 2);
+ const fs_reg tmp_left = horiz_stride(tmp, 2);
+ const fs_reg tmp_right = horiz_stride(horiz_offset(tmp, 1), 2);
+
+ ubld.MOV(tmp_left, src_right);
+ ubld.MOV(tmp_right, src_left);
+
+ }
+ bld.MOV(retype(dest, value.type), tmp);
+ break;
+ }
+
+ case nir_intrinsic_quad_swap_vertical: {
+ const fs_reg value = get_nir_src(instr->src[0]);
+ if (nir_src_bit_size(instr->src[0]) == 32) {
+ /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */
+ const fs_reg tmp = bld.vgrf(value.type);
+ const fs_builder ubld = bld.exec_all();
+ ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value,
+ brw_imm_ud(BRW_SWIZZLE4(2,3,0,1)));
+ bld.MOV(retype(dest, value.type), tmp);
+ } else {
+ /* For larger data types, we have to either emit dispatch_width many
+ * MOVs or else fall back to doing indirects.
+ */
+ fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
+ bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
+ brw_imm_w(0x2));
+ bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx);
+ }
+ break;
+ }
+
+ case nir_intrinsic_quad_swap_diagonal: {
+ const fs_reg value = get_nir_src(instr->src[0]);
+ if (nir_src_bit_size(instr->src[0]) == 32) {
+ /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */
+ const fs_reg tmp = bld.vgrf(value.type);
+ const fs_builder ubld = bld.exec_all();
+ ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value,
+ brw_imm_ud(BRW_SWIZZLE4(3,2,1,0)));
+ bld.MOV(retype(dest, value.type), tmp);
+ } else {
+ /* For larger data types, we have to either emit dispatch_width many
+ * MOVs or else fall back to doing indirects.
+ */
+ fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
+ bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
+ brw_imm_w(0x3));
+ bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx);
+ }
+ break;
+ }
+
+ case nir_intrinsic_reduce: {
+ fs_reg src = get_nir_src(instr->src[0]);
+ nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr);
+ unsigned cluster_size = nir_intrinsic_cluster_size(instr);
+ if (cluster_size == 0 || cluster_size > dispatch_width)
+ cluster_size = dispatch_width;
+
+ /* Figure out the source type */
+ src.type = brw_type_for_nir_type(devinfo,
+ (nir_alu_type)(nir_op_infos[redop].input_types[0] |
+ nir_src_bit_size(instr->src[0])));
+
+ fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type);
+ opcode brw_op = brw_op_for_nir_reduction_op(redop);
+ brw_conditional_mod cond_mod = brw_cond_mod_for_nir_reduction_op(redop);
+
+ /* There are a couple of register region issues that make things
+ * complicated for 8-bit types:
+ *
+ * 1. Only raw moves are allowed to write to a packed 8-bit
+ * destination.
+ * 2. If we use a strided destination, the efficient way to do scan
+ * operations ends up using strides that are too big to encode in
+ * an instruction.
+ *
+ * To get around these issues, we just do all 8-bit scan operations in
+ * 16 bits. It's actually fewer instructions than what we'd have to do
+ * if we were trying to do it in native 8-bit types and the results are
+ * the same once we truncate to 8 bits at the end.
+ */
+ brw_reg_type scan_type = src.type;
+ if (type_sz(scan_type) == 1)
+ scan_type = brw_reg_type_from_bit_size(16, src.type);
+
+ /* Set up a register for all of our scratching around and initialize it
+ * to reduction operation's identity value.
+ */
+ fs_reg scan = bld.vgrf(scan_type);
+ bld.exec_all().emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity);
+
+ bld.emit_scan(brw_op, scan, cluster_size, cond_mod);
+
+ dest.type = src.type;
+ if (cluster_size * type_sz(src.type) >= REG_SIZE * 2) {
+ /* In this case, CLUSTER_BROADCAST instruction isn't needed because
+ * the distance between clusters is at least 2 GRFs. In this case,
+ * we don't need the weird striding of the CLUSTER_BROADCAST
+ * instruction and can just do regular MOVs.
+ */
+ assert((cluster_size * type_sz(src.type)) % (REG_SIZE * 2) == 0);
+ const unsigned groups =
+ (dispatch_width * type_sz(src.type)) / (REG_SIZE * 2);
+ const unsigned group_size = dispatch_width / groups;
+ for (unsigned i = 0; i < groups; i++) {
+ const unsigned cluster = (i * group_size) / cluster_size;
+ const unsigned comp = cluster * cluster_size + (cluster_size - 1);
+ bld.group(group_size, i).MOV(horiz_offset(dest, i * group_size),
+ component(scan, comp));
+ }
+ } else {
+ bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, dest, scan,
+ brw_imm_ud(cluster_size - 1), brw_imm_ud(cluster_size));
+ }
+ break;
+ }
+
+ case nir_intrinsic_inclusive_scan:
+ case nir_intrinsic_exclusive_scan: {
+ fs_reg src = get_nir_src(instr->src[0]);
+ nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr);
+
+ /* Figure out the source type */
+ src.type = brw_type_for_nir_type(devinfo,
+ (nir_alu_type)(nir_op_infos[redop].input_types[0] |
+ nir_src_bit_size(instr->src[0])));
+
+ fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type);
+ opcode brw_op = brw_op_for_nir_reduction_op(redop);
+ brw_conditional_mod cond_mod = brw_cond_mod_for_nir_reduction_op(redop);
+
+ /* There are a couple of register region issues that make things
+ * complicated for 8-bit types:
+ *
+ * 1. Only raw moves are allowed to write to a packed 8-bit
+ * destination.
+ * 2. If we use a strided destination, the efficient way to do scan
+ * operations ends up using strides that are too big to encode in
+ * an instruction.
+ *
+ * To get around these issues, we just do all 8-bit scan operations in
+ * 16 bits. It's actually fewer instructions than what we'd have to do
+ * if we were trying to do it in native 8-bit types and the results are
+ * the same once we truncate to 8 bits at the end.
+ */
+ brw_reg_type scan_type = src.type;
+ if (type_sz(scan_type) == 1)
+ scan_type = brw_reg_type_from_bit_size(16, src.type);
+
+ /* Set up a register for all of our scratching around and initialize it
+ * to reduction operation's identity value.
+ */
+ fs_reg scan = bld.vgrf(scan_type);
+ const fs_builder allbld = bld.exec_all();
+ allbld.emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity);
+
+ if (instr->intrinsic == nir_intrinsic_exclusive_scan) {
+ /* Exclusive scan is a bit harder because we have to do an annoying
+ * shift of the contents before we can begin. To make things worse,
+ * we can't do this with a normal stride; we have to use indirects.
+ */
+ fs_reg shifted = bld.vgrf(scan_type);
+ fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
+ allbld.ADD(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
+ brw_imm_w(-1));
+ allbld.emit(SHADER_OPCODE_SHUFFLE, shifted, scan, idx);
+ allbld.group(1, 0).MOV(component(shifted, 0), identity);
+ scan = shifted;
+ }
+
+ bld.emit_scan(brw_op, scan, dispatch_width, cond_mod);
+
+ bld.MOV(retype(dest, src.type), scan);
+ break;
+ }
+
+ case nir_intrinsic_begin_invocation_interlock: {
+ const fs_builder ubld = bld.group(8, 0);
+ const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
+
+ ubld.emit(SHADER_OPCODE_INTERLOCK, tmp, brw_vec8_grf(0, 0))
+ ->size_written = 2 * REG_SIZE;
+ break;
+ }
+
+ case nir_intrinsic_end_invocation_interlock: {
+ /* For endInvocationInterlock(), we need to insert a memory fence which
+ * stalls in the shader until the memory transactions prior to that
+ * fence are complete. This ensures that the shader does not end before
+ * any writes from its critical section have landed. Otherwise, you can
+ * end up with a case where the next invocation on that pixel properly
+ * stalls for previous FS invocation on its pixel to complete but
+ * doesn't actually wait for the dataport memory transactions from that
+ * thread to land before submitting its own.
+ */
+ const fs_builder ubld = bld.group(8, 0);
+ const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
+ ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp,
+ brw_vec8_grf(0, 0), brw_imm_ud(1), brw_imm_ud(0))
+ ->size_written = 2 * REG_SIZE;
+ break;
+ }
+
default:
unreachable("unknown intrinsic");
}
if (stage == MESA_SHADER_FRAGMENT)
brw_wm_prog_data(prog_data)->has_side_effects = true;
+ /* The BTI untyped atomic messages only support 32-bit atomics. If you
+ * just look at the big table of messages in the Vol 7 of the SKL PRM, they
+ * appear to exist. However, if you look at Vol 2a, there are no message
+ * descriptors provided for Qword atomic ops except for A64 messages.
+ */
+ assert(nir_dest_bit_size(instr->dest) == 32);
+
fs_reg dest;
if (nir_intrinsic_infos[instr->intrinsic].has_dest)
dest = get_nir_dest(instr->dest);
- fs_reg surface;
- nir_const_value *const_surface = nir_src_as_const_value(instr->src[0]);
- if (const_surface) {
- unsigned surf_index = stage_prog_data->binding_table.ssbo_start +
- const_surface->u32[0];
- surface = brw_imm_ud(surf_index);
- brw_mark_surface_used(prog_data, surf_index);
- } else {
- surface = vgrf(glsl_type::uint_type);
- bld.ADD(surface, get_nir_src(instr->src[0]),
- brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
-
- /* Assume this may touch any SSBO. This is the same we do for other
- * UBO/SSBO accesses with non-constant surface.
- */
- brw_mark_surface_used(prog_data,
- stage_prog_data->binding_table.ssbo_start +
- nir->info.num_ssbos - 1);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
+
+ fs_reg data;
+ if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
+ data = get_nir_src(instr->src[2]);
+
+ if (op == BRW_AOP_CMPWR) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[3]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
}
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+
+ /* Emit the actual atomic operation */
+
+ bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+}
+
+void
+fs_visitor::nir_emit_ssbo_atomic_float(const fs_builder &bld,
+ int op, nir_intrinsic_instr *instr)
+{
+ if (stage == MESA_SHADER_FRAGMENT)
+ brw_wm_prog_data(prog_data)->has_side_effects = true;
- fs_reg offset = get_nir_src(instr->src[1]);
- fs_reg data1 = get_nir_src(instr->src[2]);
- fs_reg data2;
- if (op == BRW_AOP_CMPWR)
- data2 = get_nir_src(instr->src[3]);
+ fs_reg dest;
+ if (nir_intrinsic_infos[instr->intrinsic].has_dest)
+ dest = get_nir_dest(instr->dest);
+
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
+
+ fs_reg data = get_nir_src(instr->src[2]);
+ if (op == BRW_AOP_FCMPWR) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[3]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
+ }
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
/* Emit the actual atomic operation */
- fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
- data1, data2,
- 1 /* dims */, 1 /* rsize */,
- op,
- BRW_PREDICATE_NONE);
- dest.type = atomic_result.type;
- bld.MOV(dest, atomic_result);
+ bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
}
void
if (nir_intrinsic_infos[instr->intrinsic].has_dest)
dest = get_nir_dest(instr->dest);
- fs_reg surface = brw_imm_ud(GEN7_BTI_SLM);
- fs_reg offset;
- fs_reg data1 = get_nir_src(instr->src[1]);
- fs_reg data2;
- if (op == BRW_AOP_CMPWR)
- data2 = get_nir_src(instr->src[2]);
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
+
+ fs_reg data;
+ if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
+ data = get_nir_src(instr->src[1]);
+ if (op == BRW_AOP_CMPWR) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
+ }
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+
+ /* Get the offset */
+ if (nir_src_is_const(instr->src[0])) {
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
+ brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0]));
+ } else {
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = vgrf(glsl_type::uint_type);
+ bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS],
+ retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
+ brw_imm_ud(instr->const_index[0]));
+ }
+
+ /* Emit the actual atomic operation operation */
+
+ bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+}
+
+void
+fs_visitor::nir_emit_shared_atomic_float(const fs_builder &bld,
+ int op, nir_intrinsic_instr *instr)
+{
+ fs_reg dest;
+ if (nir_intrinsic_infos[instr->intrinsic].has_dest)
+ dest = get_nir_dest(instr->dest);
+
+ fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+ srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
+ srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
+ srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
+
+ fs_reg data = get_nir_src(instr->src[1]);
+ if (op == BRW_AOP_FCMPWR) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
+ }
+ srcs[SURFACE_LOGICAL_SRC_DATA] = data;
/* Get the offset */
- nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
- if (const_offset) {
- offset = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]);
+ if (nir_src_is_const(instr->src[0])) {
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
+ brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0]));
} else {
- offset = vgrf(glsl_type::uint_type);
- bld.ADD(offset,
+ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = vgrf(glsl_type::uint_type);
+ bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS],
retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
brw_imm_ud(instr->const_index[0]));
}
/* Emit the actual atomic operation operation */
- fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
- data1, data2,
- 1 /* dims */, 1 /* rsize */,
- op,
- BRW_PREDICATE_NONE);
- dest.type = atomic_result.type;
- bld.MOV(dest, atomic_result);
+ bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL,
+ dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+}
+
+void
+fs_visitor::nir_emit_global_atomic(const fs_builder &bld,
+ int op, nir_intrinsic_instr *instr)
+{
+ if (stage == MESA_SHADER_FRAGMENT)
+ brw_wm_prog_data(prog_data)->has_side_effects = true;
+
+ fs_reg dest;
+ if (nir_intrinsic_infos[instr->intrinsic].has_dest)
+ dest = get_nir_dest(instr->dest);
+
+ fs_reg addr = get_nir_src(instr->src[0]);
+
+ fs_reg data;
+ if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
+ data = get_nir_src(instr->src[1]);
+
+ if (op == BRW_AOP_CMPWR) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
+ }
+
+ if (nir_dest_bit_size(instr->dest) == 64) {
+ bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT64_LOGICAL,
+ dest, addr, data, brw_imm_ud(op));
+ } else {
+ assert(nir_dest_bit_size(instr->dest) == 32);
+ bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL,
+ dest, addr, data, brw_imm_ud(op));
+ }
+}
+
+void
+fs_visitor::nir_emit_global_atomic_float(const fs_builder &bld,
+ int op, nir_intrinsic_instr *instr)
+{
+ if (stage == MESA_SHADER_FRAGMENT)
+ brw_wm_prog_data(prog_data)->has_side_effects = true;
+
+ assert(nir_intrinsic_infos[instr->intrinsic].has_dest);
+ fs_reg dest = get_nir_dest(instr->dest);
+
+ fs_reg addr = get_nir_src(instr->src[0]);
+
+ assert(op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC);
+ fs_reg data = get_nir_src(instr->src[1]);
+
+ if (op == BRW_AOP_FCMPWR) {
+ fs_reg tmp = bld.vgrf(data.type, 2);
+ fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
+ bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
+ data = tmp;
+ }
+
+ bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL,
+ dest, addr, data, brw_imm_ud(op));
}
void
break;
}
break;
+ case nir_tex_src_min_lod:
+ srcs[TEX_LOGICAL_SRC_MIN_LOD] =
+ retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
+ break;
case nir_tex_src_ms_index:
srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = retype(src, BRW_REGISTER_TYPE_UD);
break;
case nir_tex_src_offset: {
- nir_const_value *const_offset =
- nir_src_as_const_value(instr->src[i].src);
- unsigned offset_bits = 0;
- if (const_offset &&
- brw_texture_offset(const_offset->i32,
- nir_tex_instr_src_size(instr, i),
- &offset_bits)) {
+ uint32_t offset_bits = 0;
+ if (brw_texture_offset(instr, i, &offset_bits)) {
header_bits |= offset_bits;
} else {
srcs[TEX_LOGICAL_SRC_TG4_OFFSET] =
unreachable("should be lowered");
case nir_tex_src_texture_offset: {
- /* Figure out the highest possible texture index and mark it as used */
- uint32_t max_used = texture + instr->texture_array_size - 1;
- if (instr->op == nir_texop_tg4 && devinfo->gen < 8) {
- max_used += stage_prog_data->binding_table.gather_texture_start;
- } else {
- max_used += stage_prog_data->binding_table.texture_start;
- }
- brw_mark_surface_used(prog_data, max_used);
-
/* Emit code to evaluate the actual indexing expression */
fs_reg tmp = vgrf(glsl_type::uint_type);
bld.ADD(tmp, src, brw_imm_ud(texture));
break;
}
+ case nir_tex_src_texture_handle:
+ assert(nir_tex_instr_src_index(instr, nir_tex_src_texture_offset) == -1);
+ srcs[TEX_LOGICAL_SRC_SURFACE] = fs_reg();
+ srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = bld.emit_uniformize(src);
+ break;
+
+ case nir_tex_src_sampler_handle:
+ assert(nir_tex_instr_src_index(instr, nir_tex_src_sampler_offset) == -1);
+ srcs[TEX_LOGICAL_SRC_SAMPLER] = fs_reg();
+ srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = bld.emit_uniformize(src);
+ break;
+
case nir_tex_src_ms_mcs:
assert(instr->op == nir_texop_txf_ms);
srcs[TEX_LOGICAL_SRC_MCS] = retype(src, BRW_REGISTER_TYPE_D);
break;
case nir_tex_src_plane: {
- nir_const_value *const_plane =
- nir_src_as_const_value(instr->src[i].src);
- const uint32_t plane = const_plane->u32[0];
+ const uint32_t plane = nir_src_as_uint(instr->src[i].src);
const uint32_t texture_index =
instr->texture_index +
stage_prog_data->binding_table.plane_start[plane] -
srcs[TEX_LOGICAL_SRC_MCS] =
emit_mcs_fetch(srcs[TEX_LOGICAL_SRC_COORDINATE],
instr->coord_components,
- srcs[TEX_LOGICAL_SRC_SURFACE]);
+ srcs[TEX_LOGICAL_SRC_SURFACE],
+ srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE]);
} else {
srcs[TEX_LOGICAL_SRC_MCS] = brw_imm_ud(0u);
}
enum opcode opcode;
switch (instr->op) {
case nir_texop_tex:
- opcode = (stage == MESA_SHADER_FRAGMENT ? SHADER_OPCODE_TEX_LOGICAL :
- SHADER_OPCODE_TXL_LOGICAL);
+ opcode = SHADER_OPCODE_TEX_LOGICAL;
break;
case nir_texop_txb:
opcode = FS_OPCODE_TXB_LOGICAL;
}
}
-/**
- * This helper takes the result of a load operation that reads 32-bit elements
- * in this format:
+/*
+ * This helper takes a source register and un/shuffles it into the destination
+ * register.
+ *
+ * If source type size is smaller than destination type size the operation
+ * needed is a component shuffle. The opposite case would be an unshuffle. If
+ * source/destination type size is equal a shuffle is done that would be
+ * equivalent to a simple MOV.
+ *
+ * For example, if source is a 16-bit type and destination is 32-bit. A 3
+ * components .xyz 16-bit vector on SIMD8 would be.
+ *
+ * |x1|x2|x3|x4|x5|x6|x7|x8|y1|y2|y3|y4|y5|y6|y7|y8|
+ * |z1|z2|z3|z4|z5|z6|z7|z8| | | | | | | | |
*
- * x x x x x x x x
- * y y y y y y y y
- * z z z z z z z z
- * w w w w w w w w
+ * This helper will return the following 2 32-bit components with the 16-bit
+ * values shuffled:
*
- * and shuffles the data to get this:
+ * |x1 y1|x2 y2|x3 y3|x4 y4|x5 y5|x6 y6|x7 y7|x8 y8|
+ * |z1 |z2 |z3 |z4 |z5 |z6 |z7 |z8 |
*
- * x y x y x y x y
- * x y x y x y x y
- * z w z w z w z w
- * z w z w z w z w
+ * For unshuffle, the example would be the opposite, a 64-bit type source
+ * and a 32-bit destination. A 2 component .xy 64-bit vector on SIMD8
+ * would be:
*
- * Which is exactly what we want if the load is reading 64-bit components
- * like doubles, where x represents the low 32-bit of the x double component
- * and y represents the high 32-bit of the x double component (likewise with
- * z and w for double component y). The parameter @components represents
- * the number of 64-bit components present in @src. This would typically be
- * 2 at most, since we can only fit 2 double elements in the result of a
- * vec4 load.
+ * | x1l x1h | x2l x2h | x3l x3h | x4l x4h |
+ * | x5l x5h | x6l x6h | x7l x7h | x8l x8h |
+ * | y1l y1h | y2l y2h | y3l y3h | y4l y4h |
+ * | y5l y5h | y6l y6h | y7l y7h | y8l y8h |
*
- * Notice that @dst and @src can be the same register.
+ * The returned result would be the following 4 32-bit components unshuffled:
+ *
+ * | x1l | x2l | x3l | x4l | x5l | x6l | x7l | x8l |
+ * | x1h | x2h | x3h | x4h | x5h | x6h | x7h | x8h |
+ * | y1l | y2l | y3l | y4l | y5l | y6l | y7l | y8l |
+ * | y1h | y2h | y3h | y4h | y5h | y6h | y7h | y8h |
+ *
+ * - Source and destination register must not be overlapped.
+ * - components units are measured in terms of the smaller type between
+ * source and destination because we are un/shuffling the smaller
+ * components from/into the bigger ones.
+ * - first_component parameter allows skipping source components.
*/
void
-shuffle_32bit_load_result_to_64bit_data(const fs_builder &bld,
- const fs_reg &dst,
- const fs_reg &src,
- uint32_t components)
+shuffle_src_to_dst(const fs_builder &bld,
+ const fs_reg &dst,
+ const fs_reg &src,
+ uint32_t first_component,
+ uint32_t components)
{
- assert(type_sz(src.type) == 4);
- assert(type_sz(dst.type) == 8);
-
- /* A temporary that we will use to shuffle the 32-bit data of each
- * component in the vector into valid 64-bit data. We can't write directly
- * to dst because dst can be (and would usually be) the same as src
- * and in that case the first MOV in the loop below would overwrite the
- * data read in the second MOV.
- */
- fs_reg tmp = bld.vgrf(dst.type);
-
- for (unsigned i = 0; i < components; i++) {
- const fs_reg component_i = offset(src, bld, 2 * i);
-
- bld.MOV(subscript(tmp, src.type, 0), component_i);
- bld.MOV(subscript(tmp, src.type, 1), offset(component_i, bld, 1));
-
- bld.MOV(offset(dst, bld, i), tmp);
+ if (type_sz(src.type) == type_sz(dst.type)) {
+ assert(!regions_overlap(dst,
+ type_sz(dst.type) * bld.dispatch_width() * components,
+ offset(src, bld, first_component),
+ type_sz(src.type) * bld.dispatch_width() * components));
+ for (unsigned i = 0; i < components; i++) {
+ bld.MOV(retype(offset(dst, bld, i), src.type),
+ offset(src, bld, i + first_component));
+ }
+ } else if (type_sz(src.type) < type_sz(dst.type)) {
+ /* Source is shuffled into destination */
+ unsigned size_ratio = type_sz(dst.type) / type_sz(src.type);
+ assert(!regions_overlap(dst,
+ type_sz(dst.type) * bld.dispatch_width() *
+ DIV_ROUND_UP(components, size_ratio),
+ offset(src, bld, first_component),
+ type_sz(src.type) * bld.dispatch_width() * components));
+
+ brw_reg_type shuffle_type =
+ brw_reg_type_from_bit_size(8 * type_sz(src.type),
+ BRW_REGISTER_TYPE_D);
+ for (unsigned i = 0; i < components; i++) {
+ fs_reg shuffle_component_i =
+ subscript(offset(dst, bld, i / size_ratio),
+ shuffle_type, i % size_ratio);
+ bld.MOV(shuffle_component_i,
+ retype(offset(src, bld, i + first_component), shuffle_type));
+ }
+ } else {
+ /* Source is unshuffled into destination */
+ unsigned size_ratio = type_sz(src.type) / type_sz(dst.type);
+ assert(!regions_overlap(dst,
+ type_sz(dst.type) * bld.dispatch_width() * components,
+ offset(src, bld, first_component / size_ratio),
+ type_sz(src.type) * bld.dispatch_width() *
+ DIV_ROUND_UP(components + (first_component % size_ratio),
+ size_ratio)));
+
+ brw_reg_type shuffle_type =
+ brw_reg_type_from_bit_size(8 * type_sz(dst.type),
+ BRW_REGISTER_TYPE_D);
+ for (unsigned i = 0; i < components; i++) {
+ fs_reg shuffle_component_i =
+ subscript(offset(src, bld, (first_component + i) / size_ratio),
+ shuffle_type, (first_component + i) % size_ratio);
+ bld.MOV(retype(offset(dst, bld, i), shuffle_type),
+ shuffle_component_i);
+ }
}
}
void
-shuffle_32bit_load_result_to_16bit_data(const fs_builder &bld,
- const fs_reg &dst,
- const fs_reg &src,
- uint32_t first_component,
- uint32_t components)
+shuffle_from_32bit_read(const fs_builder &bld,
+ const fs_reg &dst,
+ const fs_reg &src,
+ uint32_t first_component,
+ uint32_t components)
{
assert(type_sz(src.type) == 4);
- assert(type_sz(dst.type) == 2);
- /* A temporary is used to un-shuffle the 32-bit data of each component in
- * into a valid 16-bit vector. We can't write directly to dst because it
- * can be the same register as src and in that case the first MOV in the
- * loop below would overwrite the data read in the second MOV.
+ /* This function takes components in units of the destination type while
+ * shuffle_src_to_dst takes components in units of the smallest type
*/
- fs_reg tmp = retype(bld.vgrf(src.type), dst.type);
-
- for (unsigned i = 0; i < components; i++) {
- const fs_reg component_i =
- subscript(offset(src, bld, (first_component + i) / 2), dst.type,
- (first_component + i) % 2);
-
- bld.MOV(offset(tmp, bld, i % 2), component_i);
-
- if (i % 2) {
- bld.MOV(offset(dst, bld, i -1), offset(tmp, bld, 0));
- bld.MOV(offset(dst, bld, i), offset(tmp, bld, 1));
- }
- }
- if (components % 2) {
- bld.MOV(offset(dst, bld, components - 1), tmp);
- }
-}
-
-/**
- * This helper does the inverse operation of
- * SHUFFLE_32BIT_LOAD_RESULT_TO_64BIT_DATA.
- *
- * We need to do this when we are going to use untyped write messsages that
- * operate with 32-bit components in order to arrange our 64-bit data to be
- * in the expected layout.
- *
- * Notice that callers of this function, unlike in the case of the inverse
- * operation, would typically need to call this with dst and src being
- * different registers, since they would otherwise corrupt the original
- * 64-bit data they are about to write. Because of this the function checks
- * that the src and dst regions involved in the operation do not overlap.
- */
-fs_reg
-shuffle_64bit_data_for_32bit_write(const fs_builder &bld,
- const fs_reg &src,
- uint32_t components)
-{
- assert(type_sz(src.type) == 8);
-
- fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_D, 2 * components);
-
- for (unsigned i = 0; i < components; i++) {
- const fs_reg component_i = offset(src, bld, i);
- bld.MOV(offset(dst, bld, 2 * i), subscript(component_i, dst.type, 0));
- bld.MOV(offset(dst, bld, 2 * i + 1), subscript(component_i, dst.type, 1));
+ if (type_sz(dst.type) > 4) {
+ assert(type_sz(dst.type) == 8);
+ first_component *= 2;
+ components *= 2;
}
- return dst;
-}
-
-void
-shuffle_16bit_data_for_32bit_write(const fs_builder &bld,
- const fs_reg &dst,
- const fs_reg &src,
- uint32_t components)
-{
- assert(type_sz(src.type) == 2);
- assert(type_sz(dst.type) == 4);
-
- /* A temporary is used to shuffle the 16-bit data of each component in the
- * 32-bit data vector. We can't write directly to dst because it can be the
- * same register as src and in that case the first MOV in the loop below
- * would overwrite the data read in the second MOV.
- */
- fs_reg tmp = bld.vgrf(dst.type);
-
- for (unsigned i = 0; i < components; i++) {
- const fs_reg component_i = offset(src, bld, i);
- bld.MOV(subscript(tmp, src.type, i % 2), component_i);
- if (i % 2) {
- bld.MOV(offset(dst, bld, i / 2), tmp);
- }
- }
- if (components % 2) {
- bld.MOV(offset(dst, bld, components / 2), tmp);
- }
+ shuffle_src_to_dst(bld, dst, src, first_component, components);
}
fs_reg
return component(retype(tmp, BRW_REGISTER_TYPE_DF), 0);
}
+
+fs_reg
+setup_imm_b(const fs_builder &bld, int8_t v)
+{
+ const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_B);
+ bld.MOV(tmp, brw_imm_w(v));
+ return tmp;
+}
+
+fs_reg
+setup_imm_ub(const fs_builder &bld, uint8_t v)
+{
+ const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UB);
+ bld.MOV(tmp, brw_imm_uw(v));
+ return tmp;
+}
projects / mesa.git / blobdiff