*/
unsigned
brw_reg_type_to_hw_type(const struct brw_device_info *devinfo,
- enum brw_reg_type type, unsigned file)
+ enum brw_reg_type type, enum brw_reg_file file)
{
if (file == BRW_IMMEDIATE_VALUE) {
- const static int imm_hw_types[] = {
+ static const int imm_hw_types[] = {
[BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
[BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D,
[BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
return imm_hw_types[type];
} else {
/* Non-immediate registers */
- const static int hw_types[] = {
+ static const int hw_types[] = {
[BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
[BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D,
[BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
{
const struct brw_device_info *devinfo = p->devinfo;
- if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE &&
- dest.file != BRW_MESSAGE_REGISTER_FILE)
+ if (dest.file == BRW_MESSAGE_REGISTER_FILE)
+ assert((dest.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
+ else if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE)
assert(dest.nr < 128);
gen7_convert_mrf_to_grf(p, &dest);
brw_inst_set_dst_hstride(devinfo, inst, dest.hstride);
} else {
brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16);
- brw_inst_set_da16_writemask(devinfo, inst, dest.dw1.bits.writemask);
+ brw_inst_set_da16_writemask(devinfo, inst, dest.writemask);
if (dest.file == BRW_GENERAL_REGISTER_FILE ||
dest.file == BRW_MESSAGE_REGISTER_FILE) {
- assert(dest.dw1.bits.writemask != 0);
+ assert(dest.writemask != 0);
}
/* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
* Although Dst.HorzStride is a don't care for Align16, HW needs
*/
if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
brw_inst_set_dst_ia1_addr_imm(devinfo, inst,
- dest.dw1.bits.indirect_offset);
+ dest.indirect_offset);
if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
dest.hstride = BRW_HORIZONTAL_STRIDE_1;
brw_inst_set_dst_hstride(devinfo, inst, dest.hstride);
} else {
brw_inst_set_dst_ia16_addr_imm(devinfo, inst,
- dest.dw1.bits.indirect_offset);
+ dest.indirect_offset);
/* even ignored in da16, still need to set as '01' */
brw_inst_set_dst_hstride(devinfo, inst, 1);
}
/* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
* or 16 (SIMD16), as that's normally correct. However, when dealing with
* small registers, we automatically reduce it to match the register size.
+ *
+ * In platforms that support fp64 we can emit instructions with a width of
+ * 4 that need two SIMD8 registers and an exec_size of 8 or 16. In these
+ * cases we need to make sure that these instructions have their exec sizes
+ * set properly when they are emitted and we can't rely on this code to fix
+ * it.
*/
- if (dest.width < BRW_EXECUTE_8)
+ bool fix_exec_size;
+ if (devinfo->gen >= 6)
+ fix_exec_size = dest.width < BRW_EXECUTE_4;
+ else
+ fix_exec_size = dest.width < BRW_EXECUTE_8;
+
+ if (fix_exec_size)
brw_inst_set_exec_size(devinfo, inst, dest.width);
}
reg.file == BRW_ARF_NULL)
return;
+ /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
+ *
+ * "Swizzling is not allowed when an accumulator is used as an implicit
+ * source or an explicit source in an instruction."
+ */
+ if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE &&
+ reg.nr == BRW_ARF_ACCUMULATOR)
+ assert(reg.swizzle == BRW_SWIZZLE_XYZW);
+
assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg));
hstride = hstride_for_reg[reg.hstride];
{
const struct brw_device_info *devinfo = p->devinfo;
- if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
+ if (reg.file == BRW_MESSAGE_REGISTER_FILE)
+ assert((reg.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
+ else if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
assert(reg.nr < 128);
gen7_convert_mrf_to_grf(p, ®);
brw_inst_set_src0_address_mode(devinfo, inst, reg.address_mode);
if (reg.file == BRW_IMMEDIATE_VALUE) {
- brw_inst_set_imm_ud(devinfo, inst, reg.dw1.ud);
+ if (reg.type == BRW_REGISTER_TYPE_DF ||
+ brw_inst_opcode(devinfo, inst) == BRW_OPCODE_DIM)
+ brw_inst_set_imm_df(devinfo, inst, reg.df);
+ else
+ brw_inst_set_imm_ud(devinfo, inst, reg.ud);
/* The Bspec's section titled "Non-present Operands" claims that if src0
* is an immediate that src1's type must be the same as that of src0.
* The GM45 instruction compaction tables do not contain mapped meanings
* so it's not clear whether it has the restriction. We'll assume it was
* lifted on SNB. (FINISHME: decode the GM45 tables and check.)
+ *
+ * Don't do any of this for 64-bit immediates, since the src1 fields
+ * overlap with the immediate and setting them would overwrite the
+ * immediate we set.
*/
- brw_inst_set_src1_reg_file(devinfo, inst, BRW_ARCHITECTURE_REGISTER_FILE);
- if (devinfo->gen < 6) {
- brw_inst_set_src1_reg_type(devinfo, inst,
- brw_inst_src0_reg_type(devinfo, inst));
- } else {
- brw_inst_set_src1_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD);
+ if (type_sz(reg.type) < 8) {
+ brw_inst_set_src1_reg_file(devinfo, inst,
+ BRW_ARCHITECTURE_REGISTER_FILE);
+ if (devinfo->gen < 6) {
+ brw_inst_set_src1_reg_type(devinfo, inst,
+ brw_inst_src0_reg_type(devinfo, inst));
+ } else {
+ brw_inst_set_src1_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD);
+ }
}
/* Compacted instructions only have 12-bits (plus 1 for the other 20)
brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.subnr);
if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
- brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.dw1.bits.indirect_offset);
+ brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.indirect_offset);
} else {
- brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.dw1.bits.indirect_offset);
+ brw_inst_set_src0_ia16_addr_imm(devinfo, inst, reg.indirect_offset);
}
}
}
} else {
brw_inst_set_src0_da16_swiz_x(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X));
brw_inst_set_src0_da16_swiz_y(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y));
brw_inst_set_src0_da16_swiz_z(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z));
brw_inst_set_src0_da16_swiz_w(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W));
/* This is an oddity of the fact we're using the same
* descriptions for registers in align_16 as align_1:
if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
assert(reg.nr < 128);
+ /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
+ *
+ * "Accumulator registers may be accessed explicitly as src0
+ * operands only."
+ */
+ assert(reg.file != BRW_ARCHITECTURE_REGISTER_FILE ||
+ reg.nr != BRW_ARF_ACCUMULATOR);
+
gen7_convert_mrf_to_grf(p, ®);
assert(reg.file != BRW_MESSAGE_REGISTER_FILE);
assert(brw_inst_src0_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE);
if (reg.file == BRW_IMMEDIATE_VALUE) {
- brw_inst_set_imm_ud(devinfo, inst, reg.dw1.ud);
+ /* two-argument instructions can only use 32-bit immediates */
+ assert(type_sz(reg.type) < 8);
+ brw_inst_set_imm_ud(devinfo, inst, reg.ud);
} else {
/* This is a hardware restriction, which may or may not be lifted
* in the future:
}
} else {
brw_inst_set_src1_da16_swiz_x(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X));
brw_inst_set_src1_da16_swiz_y(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y));
brw_inst_set_src1_da16_swiz_z(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z));
brw_inst_set_src1_da16_swiz_w(devinfo, inst,
- BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W));
+ BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W));
/* This is an oddity of the fact we're using the same
* descriptions for registers in align_16 as align_1:
* \b before filling out any message-specific data. Callers can
* choose not to fill in irrelevant bits; they will be zero.
*/
-static void
+void
brw_set_message_descriptor(struct brw_codegen *p,
brw_inst *inst,
enum brw_message_target sfid,
const struct brw_device_info *devinfo = p->devinfo;
assert(num_regs == 1 || num_regs == 2 || num_regs == 4 ||
(devinfo->gen >= 8 && num_regs == 8));
+ const unsigned block_size = (devinfo->gen >= 8 ? _mesa_logbase2(num_regs) :
+ num_regs - 1);
+
brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE,
mlen, rlen, header_present, false);
brw_inst_set_dp_category(devinfo, inst, 1); /* Scratch Block Read/Write msgs */
brw_inst_set_scratch_read_write(devinfo, inst, write);
brw_inst_set_scratch_type(devinfo, inst, dword);
brw_inst_set_scratch_invalidate_after_read(devinfo, inst, invalidate_after_read);
- brw_inst_set_scratch_block_size(devinfo, inst, ffs(num_regs) - 1);
+ brw_inst_set_scratch_block_size(devinfo, inst, block_size);
brw_inst_set_scratch_addr_offset(devinfo, inst, addr_offset);
}
brw_alu2(struct brw_codegen *p, unsigned opcode,
struct brw_reg dest, struct brw_reg src0, struct brw_reg src1)
{
+ /* 64-bit immediates are only supported on 1-src instructions */
+ assert(src0.file != BRW_IMMEDIATE_VALUE || type_sz(src0.type) <= 4);
+ assert(src1.file != BRW_IMMEDIATE_VALUE || type_sz(src1.type) <= 4);
+
brw_inst *insn = next_insn(p, opcode);
brw_set_dest(p, insn, dest);
brw_set_src0(p, insn, src0);
static int
get_3src_subreg_nr(struct brw_reg reg)
{
- if (reg.vstride == BRW_VERTICAL_STRIDE_0) {
- assert(brw_is_single_value_swizzle(reg.dw1.bits.swizzle));
- return reg.subnr / 4 + BRW_GET_SWZ(reg.dw1.bits.swizzle, 0);
- } else {
- return reg.subnr / 4;
- }
+ /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
+ * use 32-bit units (components 0..7). Since they only support F/D/UD
+ * types, this doesn't lose any flexibility, but uses fewer bits.
+ */
+ return reg.subnr / 4;
}
static brw_inst *
dest.file == BRW_MESSAGE_REGISTER_FILE);
assert(dest.nr < 128);
assert(dest.address_mode == BRW_ADDRESS_DIRECT);
- assert(dest.type == BRW_REGISTER_TYPE_F ||
- dest.type == BRW_REGISTER_TYPE_D ||
+ assert(dest.type == BRW_REGISTER_TYPE_F ||
+ dest.type == BRW_REGISTER_TYPE_DF ||
+ dest.type == BRW_REGISTER_TYPE_D ||
dest.type == BRW_REGISTER_TYPE_UD);
if (devinfo->gen == 6) {
brw_inst_set_3src_dst_reg_file(devinfo, inst,
}
brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr);
brw_inst_set_3src_dst_subreg_nr(devinfo, inst, dest.subnr / 16);
- brw_inst_set_3src_dst_writemask(devinfo, inst, dest.dw1.bits.writemask);
+ brw_inst_set_3src_dst_writemask(devinfo, inst, dest.writemask);
assert(src0.file == BRW_GENERAL_REGISTER_FILE);
assert(src0.address_mode == BRW_ADDRESS_DIRECT);
assert(src0.nr < 128);
- brw_inst_set_3src_src0_swizzle(devinfo, inst, src0.dw1.bits.swizzle);
+ brw_inst_set_3src_src0_swizzle(devinfo, inst, src0.swizzle);
brw_inst_set_3src_src0_subreg_nr(devinfo, inst, get_3src_subreg_nr(src0));
brw_inst_set_3src_src0_reg_nr(devinfo, inst, src0.nr);
brw_inst_set_3src_src0_abs(devinfo, inst, src0.abs);
assert(src1.file == BRW_GENERAL_REGISTER_FILE);
assert(src1.address_mode == BRW_ADDRESS_DIRECT);
assert(src1.nr < 128);
- brw_inst_set_3src_src1_swizzle(devinfo, inst, src1.dw1.bits.swizzle);
+ brw_inst_set_3src_src1_swizzle(devinfo, inst, src1.swizzle);
brw_inst_set_3src_src1_subreg_nr(devinfo, inst, get_3src_subreg_nr(src1));
brw_inst_set_3src_src1_reg_nr(devinfo, inst, src1.nr);
brw_inst_set_3src_src1_abs(devinfo, inst, src1.abs);
assert(src2.file == BRW_GENERAL_REGISTER_FILE);
assert(src2.address_mode == BRW_ADDRESS_DIRECT);
assert(src2.nr < 128);
- brw_inst_set_3src_src2_swizzle(devinfo, inst, src2.dw1.bits.swizzle);
+ brw_inst_set_3src_src2_swizzle(devinfo, inst, src2.swizzle);
brw_inst_set_3src_src2_subreg_nr(devinfo, inst, get_3src_subreg_nr(src2));
brw_inst_set_3src_src2_reg_nr(devinfo, inst, src2.nr);
brw_inst_set_3src_src2_abs(devinfo, inst, src2.abs);
brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_F);
brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_F);
break;
+ case BRW_REGISTER_TYPE_DF:
+ brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_DF);
+ brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_DF);
+ break;
case BRW_REGISTER_TYPE_D:
brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_D);
brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_D);
struct brw_reg src1, \
struct brw_reg src2) \
{ \
- assert(dest.type == BRW_REGISTER_TYPE_F); \
- assert(src0.type == BRW_REGISTER_TYPE_F); \
- assert(src1.type == BRW_REGISTER_TYPE_F); \
- assert(src2.type == BRW_REGISTER_TYPE_F); \
+ assert(dest.type == BRW_REGISTER_TYPE_F || \
+ dest.type == BRW_REGISTER_TYPE_DF); \
+ if (dest.type == BRW_REGISTER_TYPE_F) { \
+ assert(src0.type == BRW_REGISTER_TYPE_F); \
+ assert(src1.type == BRW_REGISTER_TYPE_F); \
+ assert(src2.type == BRW_REGISTER_TYPE_F); \
+ } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
+ assert(src0.type == BRW_REGISTER_TYPE_DF); \
+ assert(src1.type == BRW_REGISTER_TYPE_DF); \
+ assert(src2.type == BRW_REGISTER_TYPE_DF); \
+ } \
return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
}
ALU2(XOR)
ALU2(SHR)
ALU2(SHL)
+ALU1(DIM)
ALU2(ASR)
ALU1(FRC)
ALU1(RNDD)
void brw_NOP(struct brw_codegen *p)
{
brw_inst *insn = next_insn(p, BRW_OPCODE_NOP);
- brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
- brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
+ brw_inst_set_exec_size(p->devinfo, insn, BRW_EXECUTE_1);
+ brw_set_dest(p, insn, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD));
+ brw_set_src0(p, insn, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD));
brw_set_src1(p, insn, brw_imm_ud(0x0));
}
static void
push_loop_stack(struct brw_codegen *p, brw_inst *inst)
{
- if (p->loop_stack_array_size < p->loop_stack_depth) {
+ if (p->loop_stack_array_size <= (p->loop_stack_depth + 1)) {
p->loop_stack_array_size *= 2;
p->loop_stack = reralloc(p->mem_ctx, p->loop_stack, int,
p->loop_stack_array_size);
insn = next_insn(p, BRW_OPCODE_IF);
brw_set_dest(p, insn, brw_imm_w(0));
- brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16
- : BRW_EXECUTE_8);
+ brw_inst_set_exec_size(devinfo, insn,
+ brw_inst_exec_size(devinfo, p->current));
brw_inst_set_gen6_jump_count(devinfo, insn, 0);
brw_set_src0(p, insn, src0);
brw_set_src1(p, insn, src1);
}
if (devinfo->gen < 6) {
- brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
- brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
+ brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
+ brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
brw_set_src1(p, insn, brw_imm_d(0x0));
} else if (devinfo->gen == 6) {
brw_set_dest(p, insn, brw_imm_w(0));
p->if_depth_in_loop[p->loop_stack_depth]);
}
brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
- brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16
- : BRW_EXECUTE_8);
+ brw_inst_set_exec_size(devinfo, insn,
+ brw_inst_exec_size(devinfo, p->current));
return insn;
}
p->if_depth_in_loop[p->loop_stack_depth]);
}
brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
- brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16
- : BRW_EXECUTE_8);
+ brw_inst_set_exec_size(devinfo, insn,
+ brw_inst_exec_size(devinfo, p->current));
return insn;
}
brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
}
- if (p->compressed) {
- brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_16);
- } else {
- brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
- brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_8);
- }
+ brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
+ brw_inst_set_exec_size(devinfo, insn,
+ brw_inst_exec_size(devinfo, p->current));
return insn;
}
brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
}
- brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16
- : BRW_EXECUTE_8);
+ brw_inst_set_exec_size(devinfo, insn,
+ brw_inst_exec_size(devinfo, p->current));
+
} else {
if (p->single_program_flow) {
insn = next_insn(p, BRW_OPCODE_ADD);
brw_set_src1(p, insn, src1);
}
+/**
+ * Return the right surface index to access the thread scratch space using
+ * stateless dataport messages.
+ */
+unsigned
+brw_scratch_surface_idx(const struct brw_codegen *p)
+{
+ /* The scratch space is thread-local so IA coherency is unnecessary. */
+ if (p->devinfo->gen >= 8)
+ return GEN8_BTI_STATELESS_NON_COHERENT;
+ else
+ return BRW_BTI_STATELESS;
+}
/**
* Write a block of OWORDs (half a GRF each) from the scratch buffer,
unsigned offset)
{
const struct brw_device_info *devinfo = p->devinfo;
- uint32_t msg_control, msg_type;
- int mlen;
+ uint32_t msg_type;
if (devinfo->gen >= 6)
offset /= 16;
mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
- if (num_regs == 1) {
- msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS;
- mlen = 2;
- } else {
- msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS;
- mlen = 3;
- }
+ const unsigned mlen = 1 + num_regs;
+ const unsigned msg_control =
+ (num_regs == 1 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS :
+ num_regs == 2 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS :
+ num_regs == 4 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : 0);
+ assert(msg_control);
/* Set up the message header. This is g0, with g0.2 filled with
* the offset. We don't want to leave our offset around in g0 or
struct brw_reg src_header = retype(brw_vec8_grf(0, 0),
BRW_REGISTER_TYPE_UW);
- if (brw_inst_qtr_control(devinfo, insn) != BRW_COMPRESSION_NONE) {
- brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
+ brw_inst_set_compression(devinfo, insn, false);
+
+ if (brw_inst_exec_size(devinfo, insn) >= 16)
src_header = vec16(src_header);
- }
+
assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE);
if (devinfo->gen < 6)
brw_inst_set_base_mrf(devinfo, insn, mrf.nr);
brw_set_dp_write_message(p,
insn,
- 255, /* binding table index (255=stateless) */
+ brw_scratch_surface_idx(p),
msg_control,
msg_type,
mlen,
unsigned offset)
{
const struct brw_device_info *devinfo = p->devinfo;
- uint32_t msg_control;
- int rlen;
if (devinfo->gen >= 6)
offset /= 16;
}
dest = retype(dest, BRW_REGISTER_TYPE_UW);
- if (num_regs == 1) {
- msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS;
- rlen = 1;
- } else {
- msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS;
- rlen = 2;
- }
+ const unsigned rlen = num_regs;
+ const unsigned msg_control =
+ (num_regs == 1 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS :
+ num_regs == 2 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS :
+ num_regs == 4 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : 0);
+ assert(msg_control);
{
brw_push_insn_state(p);
brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
assert(brw_inst_pred_control(devinfo, insn) == 0);
- brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
+ brw_inst_set_compression(devinfo, insn, false);
brw_set_dest(p, insn, dest); /* UW? */
if (devinfo->gen >= 6) {
brw_set_dp_read_message(p,
insn,
- 255, /* binding table index (255=stateless) */
+ brw_scratch_surface_idx(p),
msg_control,
BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ, /* msg_type */
BRW_DATAPORT_READ_TARGET_RENDER_CACHE,
brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE);
- brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW));
/* The HW requires that the header is present; this is to get the g0.5
void brw_fb_WRITE(struct brw_codegen *p,
- int dispatch_width,
struct brw_reg payload,
struct brw_reg implied_header,
unsigned msg_control,
unsigned msg_type;
struct brw_reg dest, src0;
- if (dispatch_width == 16)
+ if (brw_inst_exec_size(devinfo, p->current) >= BRW_EXECUTE_16)
dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
else
dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW);
} else {
insn = next_insn(p, BRW_OPCODE_SEND);
}
- brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
+ brw_inst_set_compression(devinfo, insn, false);
if (devinfo->gen >= 6) {
/* headerless version, just submit color payload */
* are allowed in SIMD16 mode and they could not work without SecHalf. For
* these reasons, we allow BRW_COMPRESSION_2NDHALF here.
*/
- if (brw_inst_qtr_control(devinfo, insn) != BRW_COMPRESSION_2NDHALF)
- brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
+ brw_inst_set_compression(devinfo, insn, false);
if (devinfo->gen < 6)
brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);
if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
const int sampler_state_size = 16; /* 16 bytes */
- uint32_t sampler = sampler_index.dw1.ud;
+ uint32_t sampler = sampler_index.ud;
if (sampler >= 16) {
assert(devinfo->is_haswell || devinfo->gen >= 8);
insn = next_insn(p, BRW_OPCODE_SEND);
- assert(msg_length < BRW_MAX_MRF);
+ assert(msg_length < BRW_MAX_MRF(devinfo->gen));
brw_set_dest(p, insn, dest);
brw_set_src0(p, insn, src0);
struct brw_reg desc)
{
const struct brw_device_info *devinfo = p->devinfo;
- struct brw_inst *send, *setup;
+ struct brw_inst *send;
+ int setup;
+
+ dst = retype(dst, BRW_REGISTER_TYPE_UW);
assert(desc.type == BRW_REGISTER_TYPE_UD);
+ /* We hold on to the setup instruction (the SEND in the direct case, the OR
+ * in the indirect case) by its index in the instruction store. The
+ * pointer returned by next_insn() may become invalid if emitting the SEND
+ * in the indirect case reallocs the store.
+ */
+
if (desc.file == BRW_IMMEDIATE_VALUE) {
- setup = send = next_insn(p, BRW_OPCODE_SEND);
+ setup = p->nr_insn;
+ send = next_insn(p, BRW_OPCODE_SEND);
brw_set_src1(p, send, desc);
} else {
* caller can specify additional descriptor bits with the usual
* brw_set_*_message() helper functions.
*/
- setup = brw_OR(p, addr, desc, brw_imm_ud(0));
+ setup = p->nr_insn;
+ brw_OR(p, addr, desc, brw_imm_ud(0));
brw_pop_insn_state(p);
brw_set_src1(p, send, addr);
}
+ if (dst.width < BRW_EXECUTE_8)
+ brw_inst_set_exec_size(devinfo, send, dst.width);
+
brw_set_dest(p, send, dst);
brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
brw_inst_set_sfid(devinfo, send, sfid);
- return setup;
+ return &p->store[setup];
}
static struct brw_inst *
*/
insn = brw_AND(p, addr,
suboffset(vec1(retype(surface, BRW_REGISTER_TYPE_UD)),
- BRW_GET_SWZ(surface.dw1.bits.swizzle, 0)),
+ BRW_GET_SWZ(surface.swizzle, 0)),
brw_imm_ud(0xff));
brw_pop_insn_state(p);
return insn;
}
+static bool
+while_jumps_before_offset(const struct brw_device_info *devinfo,
+ brw_inst *insn, int while_offset, int start_offset)
+{
+ int scale = 16 / brw_jump_scale(devinfo);
+ int jip = devinfo->gen == 6 ? brw_inst_gen6_jump_count(devinfo, insn)
+ : brw_inst_jip(devinfo, insn);
+ return while_offset + jip * scale <= start_offset;
+}
+
+
static int
brw_find_next_block_end(struct brw_codegen *p, int start_offset)
{
void *store = p->store;
const struct brw_device_info *devinfo = p->devinfo;
+ int depth = 0;
+
for (offset = next_offset(devinfo, store, start_offset);
offset < p->next_insn_offset;
offset = next_offset(devinfo, store, offset)) {
brw_inst *insn = store + offset;
switch (brw_inst_opcode(devinfo, insn)) {
+ case BRW_OPCODE_IF:
+ depth++;
+ break;
case BRW_OPCODE_ENDIF:
- case BRW_OPCODE_ELSE:
+ if (depth == 0)
+ return offset;
+ depth--;
+ break;
case BRW_OPCODE_WHILE:
+ /* If the while doesn't jump before our instruction, it's the end
+ * of a sibling do...while loop. Ignore it.
+ */
+ if (!while_jumps_before_offset(devinfo, insn, offset, start_offset))
+ continue;
+ /* fallthrough */
+ case BRW_OPCODE_ELSE:
case BRW_OPCODE_HALT:
- return offset;
+ if (depth == 0)
+ return offset;
}
}
{
const struct brw_device_info *devinfo = p->devinfo;
int offset;
- int scale = 16 / brw_jump_scale(devinfo);
void *store = p->store;
assert(devinfo->gen >= 6);
brw_inst *insn = store + offset;
if (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_WHILE) {
- int jip = devinfo->gen == 6 ? brw_inst_gen6_jump_count(devinfo, insn)
- : brw_inst_jip(devinfo, insn);
- if (offset + jip * scale <= start_offset)
+ if (while_jumps_before_offset(devinfo, insn, offset, start_offset))
return offset;
}
}
bool has_simd4x2,
bool has_simd16)
{
- if (has_simd4x2 && brw_inst_access_mode(p->devinfo, p->current) == BRW_ALIGN_16)
+ if (has_simd4x2 &&
+ brw_inst_access_mode(p->devinfo, p->current) == BRW_ALIGN_16)
return 1;
- else if (has_simd16 && p->compressed)
+ else if (has_simd16 &&
+ brw_inst_exec_size(p->devinfo, p->current) == BRW_EXECUTE_16)
return 2 * num_channels;
else
return num_channels;
if (devinfo->gen >= 8 || devinfo->is_haswell) {
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (!p->compressed)
+ if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16)
msg_control |= 1 << 4; /* SIMD8 mode */
brw_inst_set_dp_msg_type(devinfo, insn,
brw_inst_set_dp_msg_type(devinfo, insn,
GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP);
- if (!p->compressed)
+ if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16)
msg_control |= 1 << 4; /* SIMD8 mode */
}
unsigned msg_control = 0xf & (0xf << num_channels);
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (p->compressed)
+ if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16)
msg_control |= 1 << 4; /* SIMD16 mode */
else
msg_control |= 2 << 4; /* SIMD8 mode */
const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
HSW_SFID_DATAPORT_DATA_CACHE_1 :
GEN7_SFID_DATAPORT_DATA_CACHE);
- const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1);
struct brw_inst *insn = brw_send_indirect_surface_message(
p, sfid, dst, payload, surface, msg_length,
brw_surface_payload_size(p, num_channels, true, true),
- align1);
+ false);
brw_set_dp_untyped_surface_read_message(
p, insn, num_channels);
unsigned msg_control = 0xf & (0xf << num_channels);
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (p->compressed)
+ if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16)
msg_control |= 1 << 4; /* SIMD16 mode */
else
msg_control |= 2 << 4; /* SIMD8 mode */
if (devinfo->gen >= 8 || devinfo->is_haswell) {
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q)
+ if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */
brw_inst_set_dp_msg_type(devinfo, insn,
brw_inst_set_dp_msg_type(devinfo, insn,
GEN7_DATAPORT_RC_TYPED_ATOMIC_OP);
- if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q)
+ if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */
}
if (devinfo->gen >= 8 || devinfo->is_haswell) {
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q)
+ if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */
else
msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */
HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ);
} else {
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q)
+ if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */
}
if (devinfo->gen >= 8 || devinfo->is_haswell) {
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q)
+ if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */
else
msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */
} else {
if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
- if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q)
+ if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */
}
const bool commit_enable = devinfo->gen == 7 && !devinfo->is_haswell;
struct brw_inst *insn;
+ brw_push_insn_state(p);
+ brw_set_default_mask_control(p, BRW_MASK_DISABLE);
+ brw_set_default_exec_size(p, BRW_EXECUTE_1);
+ dst = vec1(dst);
+
/* Set dst as destination for dependency tracking, the MEMORY_FENCE
* message doesn't write anything back.
*/
insn = next_insn(p, BRW_OPCODE_SEND);
+ dst = retype(dst, BRW_REGISTER_TYPE_UW);
brw_set_dest(p, insn, dst);
brw_set_src0(p, insn, dst);
brw_set_memory_fence_message(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
* cache messages will be properly ordered with respect to past data and
* render cache messages.
*/
- brw_push_insn_state(p);
- brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
- brw_set_default_mask_control(p, BRW_MASK_DISABLE);
brw_MOV(p, dst, offset(dst, 1));
- brw_pop_insn_state(p);
}
+
+ brw_pop_insn_state(p);
}
void
struct brw_reg mrf,
bool noperspective,
unsigned mode,
- unsigned data,
+ struct brw_reg data,
unsigned msg_length,
unsigned response_length)
{
const struct brw_device_info *devinfo = p->devinfo;
- struct brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
-
- brw_set_dest(p, insn, dest);
- brw_set_src0(p, insn, mrf);
- brw_set_message_descriptor(p, insn, GEN7_SFID_PIXEL_INTERPOLATOR,
- msg_length, response_length,
- false /* header is never present for PI */,
- false);
+ struct brw_inst *insn;
+ const uint16_t exec_size = brw_inst_exec_size(devinfo, p->current);
- brw_inst_set_pi_simd_mode(
- devinfo, insn, brw_inst_exec_size(devinfo, insn) == BRW_EXECUTE_16);
+ /* brw_send_indirect_message will automatically use a direct send message
+ * if data is actually immediate.
+ */
+ insn = brw_send_indirect_message(p,
+ GEN7_SFID_PIXEL_INTERPOLATOR,
+ dest,
+ mrf,
+ vec1(data));
+ brw_inst_set_mlen(devinfo, insn, msg_length);
+ brw_inst_set_rlen(devinfo, insn, response_length);
+
+ brw_inst_set_pi_simd_mode(devinfo, insn, exec_size == BRW_EXECUTE_16);
brw_inst_set_pi_slot_group(devinfo, insn, 0); /* zero unless 32/64px dispatch */
brw_inst_set_pi_nopersp(devinfo, insn, noperspective);
brw_inst_set_pi_message_type(devinfo, insn, mode);
- brw_inst_set_pi_message_data(devinfo, insn, data);
}
void
brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst)
{
const struct brw_device_info *devinfo = p->devinfo;
+ const unsigned exec_size = 1 << brw_inst_exec_size(devinfo, p->current);
+ const unsigned qtr_control = brw_inst_qtr_control(devinfo, p->current);
brw_inst *inst;
assert(devinfo->gen >= 7);
retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD));
/* Quarter control has the effect of magically shifting the value of
- * this register. Make sure it's set to zero.
+ * this register so you'll get the first active channel relative to
+ * the specified quarter control as result.
*/
- brw_inst_set_qtr_control(devinfo, inst, GEN6_COMPRESSION_1Q);
} else {
- const struct brw_reg flag = retype(brw_flag_reg(1, 0),
- BRW_REGISTER_TYPE_UD);
+ const struct brw_reg flag = brw_flag_reg(1, 0);
- brw_MOV(p, flag, brw_imm_ud(0));
+ brw_MOV(p, retype(flag, BRW_REGISTER_TYPE_UD), brw_imm_ud(0));
- /* Run a 16-wide instruction returning zero with execution masking
- * and a conditional modifier enabled in order to get the current
- * execution mask in f1.0.
+ /* Run enough instructions returning zero with execution masking and
+ * a conditional modifier enabled in order to get the full execution
+ * mask in f1.0. We could use a single 32-wide move here if it
+ * weren't because of the hardware bug that causes channel enables to
+ * be applied incorrectly to the second half of 32-wide instructions
+ * on Gen7.
*/
- inst = brw_MOV(p, brw_null_reg(), brw_imm_ud(0));
- brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_16);
- brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
- brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z);
- brw_inst_set_flag_reg_nr(devinfo, inst, 1);
+ const unsigned lower_size = MIN2(16, exec_size);
+ for (unsigned i = 0; i < exec_size / lower_size; i++) {
+ inst = brw_MOV(p, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW),
+ brw_imm_uw(0));
+ brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
+ brw_inst_set_group(devinfo, inst, lower_size * i + 8 * qtr_control);
+ brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z);
+ brw_inst_set_flag_reg_nr(devinfo, inst, 1);
+ brw_inst_set_exec_size(devinfo, inst, cvt(lower_size) - 1);
+ }
- brw_FBL(p, vec1(dst), flag);
+ /* Find the first bit set in the exec_size-wide portion of the flag
+ * register that was updated by the last sequence of MOV
+ * instructions.
+ */
+ const enum brw_reg_type type = brw_int_type(exec_size / 8, false);
+ brw_FBL(p, vec1(dst), byte_offset(retype(flag, type), qtr_control));
}
} else {
brw_set_default_mask_control(p, BRW_MASK_DISABLE);
/* Overwrite the destination without and with execution masking to
* find out which of the channels is active.
*/
+ brw_push_insn_state(p);
+ brw_set_default_exec_size(p, BRW_EXECUTE_4);
brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X),
brw_imm_ud(1));
inst = brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X),
brw_imm_ud(0));
+ brw_pop_insn_state(p);
brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
}
}
const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1;
brw_inst *inst;
+ brw_push_insn_state(p);
+ brw_set_default_mask_control(p, BRW_MASK_DISABLE);
+ brw_set_default_exec_size(p, align1 ? BRW_EXECUTE_1 : BRW_EXECUTE_4);
+
assert(src.file == BRW_GENERAL_REGISTER_FILE &&
src.address_mode == BRW_ADDRESS_DIRECT);
* We will typically not get here if the optimizer is doing its job, but
* asserting would be mean.
*/
- const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.dw1.ud : 0;
+ const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.ud : 0;
brw_MOV(p, dst,
(align1 ? stride(suboffset(src, i), 0, 1, 0) :
stride(suboffset(src, 4 * i), 0, 4, 1)));
*/
inst = brw_MOV(p,
brw_null_reg(),
- stride(brw_swizzle1(idx, 0), 0, 4, 1));
+ stride(brw_swizzle(idx, BRW_SWIZZLE_XXXX), 4, 4, 1));
brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NONE);
brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_NZ);
brw_inst_set_flag_reg_nr(devinfo, inst, 1);
/* and use predicated SEL to pick the right channel. */
inst = brw_SEL(p, dst,
- stride(suboffset(src, 4), 0, 4, 1),
- stride(src, 0, 4, 1));
+ stride(suboffset(src, 4), 4, 4, 1),
+ stride(src, 4, 4, 1));
brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NORMAL);
brw_inst_set_flag_reg_nr(devinfo, inst, 1);
}
}
+
+ brw_pop_insn_state(p);
}
/**
}
+/**
+ * Emit the SEND message for a barrier
+ */
+void
+brw_barrier(struct brw_codegen *p, struct brw_reg src)
+{
+ const struct brw_device_info *devinfo = p->devinfo;
+ struct brw_inst *inst;
+
+ assert(devinfo->gen >= 7);
+
+ inst = next_insn(p, BRW_OPCODE_SEND);
+ brw_set_dest(p, inst, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
+ brw_set_src0(p, inst, src);
+ brw_set_src1(p, inst, brw_null_reg());
+
+ brw_set_message_descriptor(p, inst, BRW_SFID_MESSAGE_GATEWAY,
+ 1 /* msg_length */,
+ 0 /* response_length */,
+ false /* header_present */,
+ false /* end_of_thread */);
+
+ brw_inst_set_gateway_notify(devinfo, inst, 1);
+ brw_inst_set_gateway_subfuncid(devinfo, inst,
+ BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG);
+
+ brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE);
+}
+
+
/**
* Emit the wait instruction for a barrier
*/