X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fbrw_eu_emit.c;h=3b12030ec0c1f66562b3c8e4b4ae73b293e2f4d5;hb=31a40202b8bdf8bb65d33862144a03610fd57e3f;hp=637fd074ff152d816baec39862170974e8021bb5;hpb=8fafb0a67faa548cb16e122e214912a17835e369;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c index 637fd074ff1..3b12030ec0c 100644 --- a/src/mesa/drivers/dri/i965/brw_eu_emit.c +++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c @@ -48,7 +48,7 @@ gen6_resolve_implied_move(struct brw_codegen *p, struct brw_reg *src, unsigned msg_reg_nr) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen < 6) return; @@ -78,7 +78,7 @@ gen7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg) * Since we're pretending to have 16 MRFs anyway, we may as well use the * registers required for messages with EOT. */ - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) { reg->file = BRW_GENERAL_REGISTER_FILE; reg->nr += GEN7_MRF_HACK_START; @@ -91,8 +91,8 @@ gen7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg) * The hardware encoding may depend on whether the value is an immediate. */ unsigned -brw_reg_type_to_hw_type(const struct brw_device_info *devinfo, - enum brw_reg_type type, unsigned file) +brw_reg_type_to_hw_type(const struct gen_device_info *devinfo, + enum brw_reg_type type, enum brw_reg_file file) { if (file == BRW_IMMEDIATE_VALUE) { static const int imm_hw_types[] = { @@ -144,10 +144,11 @@ brw_reg_type_to_hw_type(const struct brw_device_info *devinfo, void brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; - if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE && - dest.file != BRW_MESSAGE_REGISTER_FILE) + if (dest.file == BRW_MESSAGE_REGISTER_FILE) + assert((dest.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen)); + else if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE) assert(dest.nr < 128); gen7_convert_mrf_to_grf(p, &dest); @@ -168,10 +169,10 @@ brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) brw_inst_set_dst_hstride(devinfo, inst, dest.hstride); } else { brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16); - brw_inst_set_da16_writemask(devinfo, inst, dest.dw1.bits.writemask); + brw_inst_set_da16_writemask(devinfo, inst, dest.writemask); if (dest.file == BRW_GENERAL_REGISTER_FILE || dest.file == BRW_MESSAGE_REGISTER_FILE) { - assert(dest.dw1.bits.writemask != 0); + assert(dest.writemask != 0); } /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1: * Although Dst.HorzStride is a don't care for Align16, HW needs @@ -186,13 +187,13 @@ brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) */ if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { brw_inst_set_dst_ia1_addr_imm(devinfo, inst, - dest.dw1.bits.indirect_offset); + dest.indirect_offset); if (dest.hstride == BRW_HORIZONTAL_STRIDE_0) dest.hstride = BRW_HORIZONTAL_STRIDE_1; brw_inst_set_dst_hstride(devinfo, inst, dest.hstride); } else { brw_inst_set_dst_ia16_addr_imm(devinfo, inst, - dest.dw1.bits.indirect_offset); + dest.indirect_offset); /* even ignored in da16, still need to set as '01' */ brw_inst_set_dst_hstride(devinfo, inst, 1); } @@ -201,15 +202,27 @@ brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8) * or 16 (SIMD16), as that's normally correct. However, when dealing with * small registers, we automatically reduce it to match the register size. + * + * In platforms that support fp64 we can emit instructions with a width of + * 4 that need two SIMD8 registers and an exec_size of 8 or 16. In these + * cases we need to make sure that these instructions have their exec sizes + * set properly when they are emitted and we can't rely on this code to fix + * it. */ - if (dest.width < BRW_EXECUTE_8) + bool fix_exec_size; + if (devinfo->gen >= 6) + fix_exec_size = dest.width < BRW_EXECUTE_4; + else + fix_exec_size = dest.width < BRW_EXECUTE_8; + + if (fix_exec_size) brw_inst_set_exec_size(devinfo, inst, dest.width); } extern int reg_type_size[]; static void -validate_reg(const struct brw_device_info *devinfo, +validate_reg(const struct gen_device_info *devinfo, brw_inst *inst, struct brw_reg reg) { const int hstride_for_reg[] = {0, 1, 2, 4}; @@ -235,6 +248,15 @@ validate_reg(const struct brw_device_info *devinfo, reg.file == BRW_ARF_NULL) return; + /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5: + * + * "Swizzling is not allowed when an accumulator is used as an implicit + * source or an explicit source in an instruction." + */ + if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE && + reg.nr == BRW_ARF_ACCUMULATOR) + assert(reg.swizzle == BRW_SWIZZLE_XYZW); + assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg)); hstride = hstride_for_reg[reg.hstride]; @@ -298,9 +320,11 @@ is_compactable_immediate(unsigned imm) void brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; - if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE) + if (reg.file == BRW_MESSAGE_REGISTER_FILE) + assert((reg.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen)); + else if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE) assert(reg.nr < 128); gen7_convert_mrf_to_grf(p, ®); @@ -326,7 +350,11 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) brw_inst_set_src0_address_mode(devinfo, inst, reg.address_mode); if (reg.file == BRW_IMMEDIATE_VALUE) { - brw_inst_set_imm_ud(devinfo, inst, reg.dw1.ud); + if (reg.type == BRW_REGISTER_TYPE_DF || + brw_inst_opcode(devinfo, inst) == BRW_OPCODE_DIM) + brw_inst_set_imm_df(devinfo, inst, reg.df); + else + brw_inst_set_imm_ud(devinfo, inst, reg.ud); /* The Bspec's section titled "Non-present Operands" claims that if src0 * is an immediate that src1's type must be the same as that of src0. @@ -349,13 +377,20 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) * The GM45 instruction compaction tables do not contain mapped meanings * so it's not clear whether it has the restriction. We'll assume it was * lifted on SNB. (FINISHME: decode the GM45 tables and check.) + * + * Don't do any of this for 64-bit immediates, since the src1 fields + * overlap with the immediate and setting them would overwrite the + * immediate we set. */ - brw_inst_set_src1_reg_file(devinfo, inst, BRW_ARCHITECTURE_REGISTER_FILE); - if (devinfo->gen < 6) { - brw_inst_set_src1_reg_type(devinfo, inst, - brw_inst_src0_reg_type(devinfo, inst)); - } else { - brw_inst_set_src1_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD); + if (type_sz(reg.type) < 8) { + brw_inst_set_src1_reg_file(devinfo, inst, + BRW_ARCHITECTURE_REGISTER_FILE); + if (devinfo->gen < 6) { + brw_inst_set_src1_reg_type(devinfo, inst, + brw_inst_src0_reg_type(devinfo, inst)); + } else { + brw_inst_set_src1_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD); + } } /* Compacted instructions only have 12-bits (plus 1 for the other 20) @@ -396,9 +431,9 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.subnr); if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { - brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.dw1.bits.indirect_offset); + brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.indirect_offset); } else { - brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.dw1.bits.indirect_offset); + brw_inst_set_src0_ia16_addr_imm(devinfo, inst, reg.indirect_offset); } } @@ -415,13 +450,13 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) } } else { brw_inst_set_src0_da16_swiz_x(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X)); brw_inst_set_src0_da16_swiz_y(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y)); brw_inst_set_src0_da16_swiz_z(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z)); brw_inst_set_src0_da16_swiz_w(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W)); /* This is an oddity of the fact we're using the same * descriptions for registers in align_16 as align_1: @@ -438,11 +473,19 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) void brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE) assert(reg.nr < 128); + /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5: + * + * "Accumulator registers may be accessed explicitly as src0 + * operands only." + */ + assert(reg.file != BRW_ARCHITECTURE_REGISTER_FILE || + reg.nr != BRW_ARF_ACCUMULATOR); + gen7_convert_mrf_to_grf(p, ®); assert(reg.file != BRW_MESSAGE_REGISTER_FILE); @@ -459,7 +502,9 @@ brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) assert(brw_inst_src0_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE); if (reg.file == BRW_IMMEDIATE_VALUE) { - brw_inst_set_imm_ud(devinfo, inst, reg.dw1.ud); + /* two-argument instructions can only use 32-bit immediates */ + assert(type_sz(reg.type) < 8); + brw_inst_set_imm_ud(devinfo, inst, reg.ud); } else { /* This is a hardware restriction, which may or may not be lifted * in the future: @@ -487,13 +532,13 @@ brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) } } else { brw_inst_set_src1_da16_swiz_x(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X)); brw_inst_set_src1_da16_swiz_y(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y)); brw_inst_set_src1_da16_swiz_z(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z)); brw_inst_set_src1_da16_swiz_w(devinfo, inst, - BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W)); + BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W)); /* This is an oddity of the fact we're using the same * descriptions for registers in align_16 as align_1: @@ -514,7 +559,7 @@ brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) * \b before filling out any message-specific data. Callers can * choose not to fill in irrelevant bits; they will be zero. */ -static void +void brw_set_message_descriptor(struct brw_codegen *p, brw_inst *inst, enum brw_message_target sfid, @@ -523,7 +568,7 @@ brw_set_message_descriptor(struct brw_codegen *p, bool header_present, bool end_of_thread) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_src1(p, inst, brw_imm_d(0)); @@ -555,7 +600,7 @@ static void brw_set_math_message( struct brw_codegen *p, bool low_precision, unsigned dataType ) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned msg_length; unsigned response_length; @@ -601,7 +646,7 @@ static void brw_set_ff_sync_message(struct brw_codegen *p, unsigned response_length, bool end_of_thread) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_message_descriptor(p, insn, BRW_SFID_URB, 1, response_length, true, end_of_thread); @@ -622,7 +667,7 @@ static void brw_set_urb_message( struct brw_codegen *p, unsigned offset, unsigned swizzle_control ) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; assert(devinfo->gen < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE); assert(devinfo->gen < 7 || !(flags & BRW_URB_WRITE_ALLOCATE)); @@ -668,7 +713,7 @@ brw_set_dp_write_message(struct brw_codegen *p, unsigned end_of_thread, unsigned send_commit_msg) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned sfid; if (devinfo->gen >= 7) { @@ -707,11 +752,19 @@ brw_set_dp_read_message(struct brw_codegen *p, bool header_present, unsigned response_length) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned sfid; if (devinfo->gen >= 7) { - sfid = GEN7_SFID_DATAPORT_DATA_CACHE; + if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE) + sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; + else if (target_cache == BRW_DATAPORT_READ_TARGET_DATA_CACHE) + sfid = GEN7_SFID_DATAPORT_DATA_CACHE; + else if (target_cache == BRW_DATAPORT_READ_TARGET_SAMPLER_CACHE) + sfid = GEN6_SFID_DATAPORT_SAMPLER_CACHE; + else + unreachable("Invalid target cache"); + } else if (devinfo->gen == 6) { if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE) sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; @@ -743,7 +796,7 @@ brw_set_sampler_message(struct brw_codegen *p, unsigned simd_mode, unsigned return_format) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_message_descriptor(p, inst, BRW_SFID_SAMPLER, msg_length, response_length, header_present, false); @@ -770,16 +823,19 @@ gen7_set_dp_scratch_message(struct brw_codegen *p, unsigned rlen, bool header_present) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; assert(num_regs == 1 || num_regs == 2 || num_regs == 4 || (devinfo->gen >= 8 && num_regs == 8)); + const unsigned block_size = (devinfo->gen >= 8 ? _mesa_logbase2(num_regs) : + num_regs - 1); + brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE, mlen, rlen, header_present, false); brw_inst_set_dp_category(devinfo, inst, 1); /* Scratch Block Read/Write msgs */ brw_inst_set_scratch_read_write(devinfo, inst, write); brw_inst_set_scratch_type(devinfo, inst, dword); brw_inst_set_scratch_invalidate_after_read(devinfo, inst, invalidate_after_read); - brw_inst_set_scratch_block_size(devinfo, inst, ffs(num_regs) - 1); + brw_inst_set_scratch_block_size(devinfo, inst, block_size); brw_inst_set_scratch_addr_offset(devinfo, inst, addr_offset); } @@ -787,7 +843,7 @@ gen7_set_dp_scratch_message(struct brw_codegen *p, brw_inst * brw_next_insn(struct brw_codegen *p, unsigned opcode) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; if (p->nr_insn + 1 > p->store_size) { @@ -817,6 +873,10 @@ static brw_inst * brw_alu2(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, struct brw_reg src0, struct brw_reg src1) { + /* 64-bit immediates are only supported on 1-src instructions */ + assert(src0.file != BRW_IMMEDIATE_VALUE || type_sz(src0.type) <= 4); + assert(src1.file != BRW_IMMEDIATE_VALUE || type_sz(src1.type) <= 4); + brw_inst *insn = next_insn(p, opcode); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src0); @@ -827,19 +887,18 @@ brw_alu2(struct brw_codegen *p, unsigned opcode, static int get_3src_subreg_nr(struct brw_reg reg) { - if (reg.vstride == BRW_VERTICAL_STRIDE_0) { - assert(brw_is_single_value_swizzle(reg.dw1.bits.swizzle)); - return reg.subnr / 4 + BRW_GET_SWZ(reg.dw1.bits.swizzle, 0); - } else { - return reg.subnr / 4; - } + /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions + * use 32-bit units (components 0..7). Since they only support F/D/UD + * types, this doesn't lose any flexibility, but uses fewer bits. + */ + return reg.subnr / 4; } static brw_inst * brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, struct brw_reg src0, struct brw_reg src1, struct brw_reg src2) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *inst = next_insn(p, opcode); gen7_convert_mrf_to_grf(p, &dest); @@ -850,8 +909,9 @@ brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, dest.file == BRW_MESSAGE_REGISTER_FILE); assert(dest.nr < 128); assert(dest.address_mode == BRW_ADDRESS_DIRECT); - assert(dest.type == BRW_REGISTER_TYPE_F || - dest.type == BRW_REGISTER_TYPE_D || + assert(dest.type == BRW_REGISTER_TYPE_F || + dest.type == BRW_REGISTER_TYPE_DF || + dest.type == BRW_REGISTER_TYPE_D || dest.type == BRW_REGISTER_TYPE_UD); if (devinfo->gen == 6) { brw_inst_set_3src_dst_reg_file(devinfo, inst, @@ -859,12 +919,12 @@ brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, } brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr); brw_inst_set_3src_dst_subreg_nr(devinfo, inst, dest.subnr / 16); - brw_inst_set_3src_dst_writemask(devinfo, inst, dest.dw1.bits.writemask); + brw_inst_set_3src_dst_writemask(devinfo, inst, dest.writemask); assert(src0.file == BRW_GENERAL_REGISTER_FILE); assert(src0.address_mode == BRW_ADDRESS_DIRECT); assert(src0.nr < 128); - brw_inst_set_3src_src0_swizzle(devinfo, inst, src0.dw1.bits.swizzle); + brw_inst_set_3src_src0_swizzle(devinfo, inst, src0.swizzle); brw_inst_set_3src_src0_subreg_nr(devinfo, inst, get_3src_subreg_nr(src0)); brw_inst_set_3src_src0_reg_nr(devinfo, inst, src0.nr); brw_inst_set_3src_src0_abs(devinfo, inst, src0.abs); @@ -875,7 +935,7 @@ brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, assert(src1.file == BRW_GENERAL_REGISTER_FILE); assert(src1.address_mode == BRW_ADDRESS_DIRECT); assert(src1.nr < 128); - brw_inst_set_3src_src1_swizzle(devinfo, inst, src1.dw1.bits.swizzle); + brw_inst_set_3src_src1_swizzle(devinfo, inst, src1.swizzle); brw_inst_set_3src_src1_subreg_nr(devinfo, inst, get_3src_subreg_nr(src1)); brw_inst_set_3src_src1_reg_nr(devinfo, inst, src1.nr); brw_inst_set_3src_src1_abs(devinfo, inst, src1.abs); @@ -886,7 +946,7 @@ brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, assert(src2.file == BRW_GENERAL_REGISTER_FILE); assert(src2.address_mode == BRW_ADDRESS_DIRECT); assert(src2.nr < 128); - brw_inst_set_3src_src2_swizzle(devinfo, inst, src2.dw1.bits.swizzle); + brw_inst_set_3src_src2_swizzle(devinfo, inst, src2.swizzle); brw_inst_set_3src_src2_subreg_nr(devinfo, inst, get_3src_subreg_nr(src2)); brw_inst_set_3src_src2_reg_nr(devinfo, inst, src2.nr); brw_inst_set_3src_src2_abs(devinfo, inst, src2.abs); @@ -906,6 +966,10 @@ brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_F); brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_F); break; + case BRW_REGISTER_TYPE_DF: + brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_DF); + brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_DF); + break; case BRW_REGISTER_TYPE_D: brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_D); brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_D); @@ -960,10 +1024,17 @@ brw_inst *brw_##OP(struct brw_codegen *p, \ struct brw_reg src1, \ struct brw_reg src2) \ { \ - assert(dest.type == BRW_REGISTER_TYPE_F); \ - assert(src0.type == BRW_REGISTER_TYPE_F); \ - assert(src1.type == BRW_REGISTER_TYPE_F); \ - assert(src2.type == BRW_REGISTER_TYPE_F); \ + assert(dest.type == BRW_REGISTER_TYPE_F || \ + dest.type == BRW_REGISTER_TYPE_DF); \ + if (dest.type == BRW_REGISTER_TYPE_F) { \ + assert(src0.type == BRW_REGISTER_TYPE_F); \ + assert(src1.type == BRW_REGISTER_TYPE_F); \ + assert(src2.type == BRW_REGISTER_TYPE_F); \ + } else if (dest.type == BRW_REGISTER_TYPE_DF) { \ + assert(src0.type == BRW_REGISTER_TYPE_DF); \ + assert(src1.type == BRW_REGISTER_TYPE_DF); \ + assert(src2.type == BRW_REGISTER_TYPE_DF); \ + } \ return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \ } @@ -979,7 +1050,7 @@ void brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src) \ { \ - const struct brw_device_info *devinfo = p->devinfo; \ + const struct gen_device_info *devinfo = p->devinfo; \ brw_inst *rnd, *add; \ rnd = next_insn(p, BRW_OPCODE_##OP); \ brw_set_dest(p, rnd, dest); \ @@ -1002,6 +1073,7 @@ ALU2(OR) ALU2(XOR) ALU2(SHR) ALU2(SHL) +ALU1(DIM) ALU2(ASR) ALU1(FRC) ALU1(RNDD) @@ -1131,7 +1203,7 @@ brw_PLN(struct brw_codegen *p, struct brw_reg dest, brw_inst * brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16; /* The F32TO16 instruction doesn't support 32-bit destination types in * Align1 mode, and neither does the Gen8 implementation in terms of a @@ -1178,7 +1250,7 @@ brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) brw_inst * brw_F16TO32(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16; if (align16) { @@ -1210,8 +1282,9 @@ brw_F16TO32(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) void brw_NOP(struct brw_codegen *p) { brw_inst *insn = next_insn(p, BRW_OPCODE_NOP); - brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD)); - brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD)); + brw_inst_set_exec_size(p->devinfo, insn, BRW_EXECUTE_1); + brw_set_dest(p, insn, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD)); + brw_set_src0(p, insn, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD)); brw_set_src1(p, insn, brw_imm_ud(0x0)); } @@ -1227,7 +1300,7 @@ brw_inst * brw_JMPI(struct brw_codegen *p, struct brw_reg index, unsigned predicate_control) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_reg ip = brw_ip_reg(); brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index); @@ -1262,7 +1335,7 @@ pop_if_stack(struct brw_codegen *p) static void push_loop_stack(struct brw_codegen *p, brw_inst *inst) { - if (p->loop_stack_array_size < p->loop_stack_depth) { + if (p->loop_stack_array_size <= (p->loop_stack_depth + 1)) { p->loop_stack_array_size *= 2; p->loop_stack = reralloc(p->mem_ctx, p->loop_stack, int, p->loop_stack_array_size); @@ -1297,7 +1370,7 @@ get_inner_do_insn(struct brw_codegen *p) brw_inst * brw_IF(struct brw_codegen *p, unsigned execute_size) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_IF); @@ -1345,14 +1418,14 @@ brw_inst * gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional, struct brw_reg src0, struct brw_reg src1) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_IF); brw_set_dest(p, insn, brw_imm_w(0)); - brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 - : BRW_EXECUTE_8); + brw_inst_set_exec_size(devinfo, insn, + brw_inst_exec_size(devinfo, p->current)); brw_inst_set_gen6_jump_count(devinfo, insn, 0); brw_set_src0(p, insn, src0); brw_set_src1(p, insn, src1); @@ -1372,7 +1445,7 @@ static void convert_IF_ELSE_to_ADD(struct brw_codegen *p, brw_inst *if_inst, brw_inst *else_inst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* The next instruction (where the ENDIF would be, if it existed) */ brw_inst *next_inst = &p->store[p->nr_insn]; @@ -1413,7 +1486,7 @@ static void patch_IF_ELSE(struct brw_codegen *p, brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* We shouldn't be patching IF and ELSE instructions in single program flow * mode when gen < 6, because in single program flow mode on those @@ -1500,7 +1573,7 @@ patch_IF_ELSE(struct brw_codegen *p, void brw_ELSE(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_ELSE); @@ -1538,7 +1611,7 @@ brw_ELSE(struct brw_codegen *p) void brw_ENDIF(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = NULL; brw_inst *else_inst = NULL; brw_inst *if_inst = NULL; @@ -1619,7 +1692,7 @@ brw_ENDIF(struct brw_codegen *p) brw_inst * brw_BREAK(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_BREAK); @@ -1638,8 +1711,8 @@ brw_BREAK(struct brw_codegen *p) p->if_depth_in_loop[p->loop_stack_depth]); } brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); - brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 - : BRW_EXECUTE_8); + brw_inst_set_exec_size(devinfo, insn, + brw_inst_exec_size(devinfo, p->current)); return insn; } @@ -1647,7 +1720,7 @@ brw_BREAK(struct brw_codegen *p) brw_inst * brw_CONT(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_CONTINUE); @@ -1664,15 +1737,15 @@ brw_CONT(struct brw_codegen *p) p->if_depth_in_loop[p->loop_stack_depth]); } brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); - brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 - : BRW_EXECUTE_8); + brw_inst_set_exec_size(devinfo, insn, + brw_inst_exec_size(devinfo, p->current)); return insn; } brw_inst * gen6_HALT(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_HALT); @@ -1684,12 +1757,9 @@ gen6_HALT(struct brw_codegen *p) brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */ } - if (p->compressed) { - brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_16); - } else { - brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); - brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_8); - } + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_exec_size(devinfo, insn, + brw_inst_exec_size(devinfo, p->current)); return insn; } @@ -1712,7 +1782,7 @@ gen6_HALT(struct brw_codegen *p) brw_inst * brw_DO(struct brw_codegen *p, unsigned execute_size) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen >= 6 || p->single_program_flow) { push_loop_stack(p, &p->store[p->nr_insn]); @@ -1746,7 +1816,7 @@ brw_DO(struct brw_codegen *p, unsigned execute_size) static void brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *do_inst = get_inner_do_insn(p); brw_inst *inst; unsigned br = brw_jump_scale(devinfo); @@ -1771,7 +1841,7 @@ brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst) brw_inst * brw_WHILE(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn, *do_insn; unsigned br = brw_jump_scale(devinfo); @@ -1795,8 +1865,9 @@ brw_WHILE(struct brw_codegen *p) brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); } - brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 - : BRW_EXECUTE_8); + brw_inst_set_exec_size(devinfo, insn, + brw_inst_exec_size(devinfo, p->current)); + } else { if (p->single_program_flow) { insn = next_insn(p, BRW_OPCODE_ADD); @@ -1834,7 +1905,7 @@ brw_WHILE(struct brw_codegen *p) */ void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *jmp_insn = &p->store[jmp_insn_idx]; unsigned jmpi = 1; @@ -1858,7 +1929,7 @@ void brw_CMP(struct brw_codegen *p, struct brw_reg src0, struct brw_reg src1) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_CMP); brw_inst_set_cond_modifier(devinfo, insn, conditional); @@ -1894,7 +1965,7 @@ void gen4_math(struct brw_codegen *p, struct brw_reg src, unsigned precision ) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); unsigned data_type; if (has_scalar_region(src)) { @@ -1927,7 +1998,7 @@ void gen6_math(struct brw_codegen *p, struct brw_reg src0, struct brw_reg src1) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_MATH); assert(devinfo->gen >= 6); @@ -1977,6 +2048,19 @@ void gen6_math(struct brw_codegen *p, brw_set_src1(p, insn, src1); } +/** + * Return the right surface index to access the thread scratch space using + * stateless dataport messages. + */ +unsigned +brw_scratch_surface_idx(const struct brw_codegen *p) +{ + /* The scratch space is thread-local so IA coherency is unnecessary. */ + if (p->devinfo->gen >= 8) + return GEN8_BTI_STATELESS_NON_COHERENT; + else + return BRW_BTI_STATELESS; +} /** * Write a block of OWORDs (half a GRF each) from the scratch buffer, @@ -1990,22 +2074,20 @@ void brw_oword_block_write_scratch(struct brw_codegen *p, int num_regs, unsigned offset) { - const struct brw_device_info *devinfo = p->devinfo; - uint32_t msg_control, msg_type; - int mlen; + const struct gen_device_info *devinfo = p->devinfo; + uint32_t msg_type; if (devinfo->gen >= 6) offset /= 16; mrf = retype(mrf, BRW_REGISTER_TYPE_UD); - if (num_regs == 1) { - msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS; - mlen = 2; - } else { - msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS; - mlen = 3; - } + const unsigned mlen = 1 + num_regs; + const unsigned msg_control = + (num_regs == 1 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS : + num_regs == 2 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS : + num_regs == 4 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : 0); + assert(msg_control); /* Set up the message header. This is g0, with g0.2 filled with * the offset. We don't want to leave our offset around in g0 or @@ -2037,10 +2119,11 @@ void brw_oword_block_write_scratch(struct brw_codegen *p, struct brw_reg src_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW); - if (brw_inst_qtr_control(devinfo, insn) != BRW_COMPRESSION_NONE) { - brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_compression(devinfo, insn, false); + + if (brw_inst_exec_size(devinfo, insn) >= 16) src_header = vec16(src_header); - } + assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE); if (devinfo->gen < 6) brw_inst_set_base_mrf(devinfo, insn, mrf.nr); @@ -2077,7 +2160,7 @@ void brw_oword_block_write_scratch(struct brw_codegen *p, brw_set_dp_write_message(p, insn, - 255, /* binding table index (255=stateless) */ + brw_scratch_surface_idx(p), msg_control, msg_type, mlen, @@ -2104,9 +2187,7 @@ brw_oword_block_read_scratch(struct brw_codegen *p, int num_regs, unsigned offset) { - const struct brw_device_info *devinfo = p->devinfo; - uint32_t msg_control; - int rlen; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen >= 6) offset /= 16; @@ -2125,13 +2206,15 @@ brw_oword_block_read_scratch(struct brw_codegen *p, } dest = retype(dest, BRW_REGISTER_TYPE_UW); - if (num_regs == 1) { - msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS; - rlen = 1; - } else { - msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS; - rlen = 2; - } + const unsigned rlen = num_regs; + const unsigned msg_control = + (num_regs == 1 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS : + num_regs == 2 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS : + num_regs == 4 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : 0); + assert(msg_control); + const unsigned target_cache = devinfo->gen >= 7 ? + BRW_DATAPORT_READ_TARGET_DATA_CACHE : + BRW_DATAPORT_READ_TARGET_RENDER_CACHE; { brw_push_insn_state(p); @@ -2151,7 +2234,7 @@ brw_oword_block_read_scratch(struct brw_codegen *p, brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); assert(brw_inst_pred_control(devinfo, insn) == 0); - brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_compression(devinfo, insn, false); brw_set_dest(p, insn, dest); /* UW? */ if (devinfo->gen >= 6) { @@ -2163,10 +2246,10 @@ brw_oword_block_read_scratch(struct brw_codegen *p, brw_set_dp_read_message(p, insn, - 255, /* binding table index (255=stateless) */ + brw_scratch_surface_idx(p), msg_control, BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ, /* msg_type */ - BRW_DATAPORT_READ_TARGET_RENDER_CACHE, + target_cache, 1, /* msg_length */ true, /* header_present */ rlen); @@ -2179,11 +2262,10 @@ gen7_block_read_scratch(struct brw_codegen *p, int num_regs, unsigned offset) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE); - brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW)); /* The HW requires that the header is present; this is to get the g0.5 @@ -2220,7 +2302,7 @@ void brw_oword_block_read(struct brw_codegen *p, uint32_t offset, uint32_t bind_table_index) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* On newer hardware, offset is in units of owords. */ if (devinfo->gen >= 6) @@ -2271,7 +2353,6 @@ void brw_oword_block_read(struct brw_codegen *p, void brw_fb_WRITE(struct brw_codegen *p, - int dispatch_width, struct brw_reg payload, struct brw_reg implied_header, unsigned msg_control, @@ -2282,12 +2363,12 @@ void brw_fb_WRITE(struct brw_codegen *p, bool last_render_target, bool header_present) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; unsigned msg_type; struct brw_reg dest, src0; - if (dispatch_width == 16) + if (brw_inst_exec_size(devinfo, p->current) >= BRW_EXECUTE_16) dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW); else dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW); @@ -2297,7 +2378,7 @@ void brw_fb_WRITE(struct brw_codegen *p, } else { insn = next_insn(p, BRW_OPCODE_SEND); } - brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_compression(devinfo, insn, false); if (devinfo->gen >= 6) { /* headerless version, just submit color payload */ @@ -2327,6 +2408,34 @@ void brw_fb_WRITE(struct brw_codegen *p, 0 /* send_commit_msg */); } +brw_inst * +gen9_fb_READ(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg payload, + unsigned binding_table_index, + unsigned msg_length, + unsigned response_length, + bool per_sample) +{ + const struct gen_device_info *devinfo = p->devinfo; + assert(devinfo->gen >= 9); + const unsigned msg_subtype = + brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16 ? 0 : 1; + brw_inst *insn = next_insn(p, BRW_OPCODE_SENDC); + + brw_set_dest(p, insn, dst); + brw_set_src0(p, insn, payload); + brw_set_dp_read_message(p, insn, binding_table_index, + per_sample << 5 | msg_subtype, + GEN9_DATAPORT_RC_RENDER_TARGET_READ, + BRW_DATAPORT_READ_TARGET_RENDER_CACHE, + msg_length, true /* header_present */, + response_length); + brw_inst_set_rt_slot_group(devinfo, insn, + brw_inst_qtr_control(devinfo, p->current) / 2); + + return insn; +} /** * Texture sample instruction. @@ -2346,7 +2455,7 @@ void brw_SAMPLE(struct brw_codegen *p, unsigned simd_mode, unsigned return_format) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; if (msg_reg_nr != -1) @@ -2367,8 +2476,7 @@ void brw_SAMPLE(struct brw_codegen *p, * are allowed in SIMD16 mode and they could not work without SecHalf. For * these reasons, we allow BRW_COMPRESSION_2NDHALF here. */ - if (brw_inst_qtr_control(devinfo, insn) != BRW_COMPRESSION_2NDHALF) - brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_compression(devinfo, insn, false); if (devinfo->gen < 6) brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr); @@ -2402,11 +2510,11 @@ void brw_adjust_sampler_state_pointer(struct brw_codegen *p, * exclusively use the offset - we have to use both. */ - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (sampler_index.file == BRW_IMMEDIATE_VALUE) { const int sampler_state_size = 16; /* 16 bytes */ - uint32_t sampler = sampler_index.dw1.ud; + uint32_t sampler = sampler_index.ud; if (sampler >= 16) { assert(devinfo->is_haswell || devinfo->gen >= 8); @@ -2446,7 +2554,7 @@ void brw_urb_WRITE(struct brw_codegen *p, unsigned offset, unsigned swizzle) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); @@ -2465,7 +2573,7 @@ void brw_urb_WRITE(struct brw_codegen *p, insn = next_insn(p, BRW_OPCODE_SEND); - assert(msg_length < BRW_MAX_MRF); + assert(msg_length < BRW_MAX_MRF(devinfo->gen)); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src0); @@ -2490,13 +2598,23 @@ brw_send_indirect_message(struct brw_codegen *p, struct brw_reg payload, struct brw_reg desc) { - const struct brw_device_info *devinfo = p->devinfo; - struct brw_inst *send, *setup; + const struct gen_device_info *devinfo = p->devinfo; + struct brw_inst *send; + int setup; + + dst = retype(dst, BRW_REGISTER_TYPE_UW); assert(desc.type == BRW_REGISTER_TYPE_UD); + /* We hold on to the setup instruction (the SEND in the direct case, the OR + * in the indirect case) by its index in the instruction store. The + * pointer returned by next_insn() may become invalid if emitting the SEND + * in the indirect case reallocs the store. + */ + if (desc.file == BRW_IMMEDIATE_VALUE) { - setup = send = next_insn(p, BRW_OPCODE_SEND); + setup = p->nr_insn; + send = next_insn(p, BRW_OPCODE_SEND); brw_set_src1(p, send, desc); } else { @@ -2511,7 +2629,8 @@ brw_send_indirect_message(struct brw_codegen *p, * caller can specify additional descriptor bits with the usual * brw_set_*_message() helper functions. */ - setup = brw_OR(p, addr, desc, brw_imm_ud(0)); + setup = p->nr_insn; + brw_OR(p, addr, desc, brw_imm_ud(0)); brw_pop_insn_state(p); @@ -2519,11 +2638,14 @@ brw_send_indirect_message(struct brw_codegen *p, brw_set_src1(p, send, addr); } + if (dst.width < BRW_EXECUTE_8) + brw_inst_set_exec_size(devinfo, send, dst.width); + brw_set_dest(p, send, dst); brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD)); brw_inst_set_sfid(devinfo, send, sfid); - return setup; + return &p->store[setup]; } static struct brw_inst * @@ -2536,7 +2658,7 @@ brw_send_indirect_surface_message(struct brw_codegen *p, unsigned response_len, bool header_present) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *insn; if (surface.file != BRW_IMMEDIATE_VALUE) { @@ -2552,7 +2674,7 @@ brw_send_indirect_surface_message(struct brw_codegen *p, */ insn = brw_AND(p, addr, suboffset(vec1(retype(surface, BRW_REGISTER_TYPE_UD)), - BRW_GET_SWZ(surface.dw1.bits.swizzle, 0)), + BRW_GET_SWZ(surface.swizzle, 0)), brw_imm_ud(0xff)); brw_pop_insn_state(p); @@ -2568,12 +2690,25 @@ brw_send_indirect_surface_message(struct brw_codegen *p, return insn; } +static bool +while_jumps_before_offset(const struct gen_device_info *devinfo, + brw_inst *insn, int while_offset, int start_offset) +{ + int scale = 16 / brw_jump_scale(devinfo); + int jip = devinfo->gen == 6 ? brw_inst_gen6_jump_count(devinfo, insn) + : brw_inst_jip(devinfo, insn); + return while_offset + jip * scale <= start_offset; +} + + static int brw_find_next_block_end(struct brw_codegen *p, int start_offset) { int offset; void *store = p->store; - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; + + int depth = 0; for (offset = next_offset(devinfo, store, start_offset); offset < p->next_insn_offset; @@ -2581,11 +2716,25 @@ brw_find_next_block_end(struct brw_codegen *p, int start_offset) brw_inst *insn = store + offset; switch (brw_inst_opcode(devinfo, insn)) { + case BRW_OPCODE_IF: + depth++; + break; case BRW_OPCODE_ENDIF: - case BRW_OPCODE_ELSE: + if (depth == 0) + return offset; + depth--; + break; case BRW_OPCODE_WHILE: + /* If the while doesn't jump before our instruction, it's the end + * of a sibling do...while loop. Ignore it. + */ + if (!while_jumps_before_offset(devinfo, insn, offset, start_offset)) + continue; + /* fallthrough */ + case BRW_OPCODE_ELSE: case BRW_OPCODE_HALT: - return offset; + if (depth == 0) + return offset; } } @@ -2599,9 +2748,8 @@ brw_find_next_block_end(struct brw_codegen *p, int start_offset) static int brw_find_loop_end(struct brw_codegen *p, int start_offset) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; int offset; - int scale = 16 / brw_jump_scale(devinfo); void *store = p->store; assert(devinfo->gen >= 6); @@ -2615,9 +2763,7 @@ brw_find_loop_end(struct brw_codegen *p, int start_offset) brw_inst *insn = store + offset; if (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_WHILE) { - int jip = devinfo->gen == 6 ? brw_inst_gen6_jump_count(devinfo, insn) - : brw_inst_jip(devinfo, insn); - if (offset + jip * scale <= start_offset) + if (while_jumps_before_offset(devinfo, insn, offset, start_offset)) return offset; } } @@ -2629,9 +2775,9 @@ brw_find_loop_end(struct brw_codegen *p, int start_offset) * BREAK, CONT, and HALT instructions to their correct locations. */ void -brw_set_uip_jip(struct brw_codegen *p) +brw_set_uip_jip(struct brw_codegen *p, int start_offset) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; int offset; int br = brw_jump_scale(devinfo); int scale = 16 / br; @@ -2640,17 +2786,9 @@ brw_set_uip_jip(struct brw_codegen *p) if (devinfo->gen < 6) return; - for (offset = 0; offset < p->next_insn_offset; - offset = next_offset(devinfo, store, offset)) { + for (offset = start_offset; offset < p->next_insn_offset; offset += 16) { brw_inst *insn = store + offset; - - if (brw_inst_cmpt_control(devinfo, insn)) { - /* Fixups for compacted BREAK/CONTINUE not supported yet. */ - assert(brw_inst_opcode(devinfo, insn) != BRW_OPCODE_BREAK && - brw_inst_opcode(devinfo, insn) != BRW_OPCODE_CONTINUE && - brw_inst_opcode(devinfo, insn) != BRW_OPCODE_HALT); - continue; - } + assert(brw_inst_cmpt_control(devinfo, insn) == 0); int block_end_offset = brw_find_next_block_end(p, offset); switch (brw_inst_opcode(devinfo, insn)) { @@ -2714,7 +2852,7 @@ void brw_ff_sync(struct brw_codegen *p, unsigned response_length, bool eot) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); @@ -2779,9 +2917,11 @@ brw_surface_payload_size(struct brw_codegen *p, bool has_simd4x2, bool has_simd16) { - if (has_simd4x2 && brw_inst_access_mode(p->devinfo, p->current) == BRW_ALIGN_16) + if (has_simd4x2 && + brw_inst_access_mode(p->devinfo, p->current) == BRW_ALIGN_16) return 1; - else if (has_simd16 && p->compressed) + else if (has_simd16 && + brw_inst_exec_size(p->devinfo, p->current) == BRW_EXECUTE_16) return 2 * num_channels; else return num_channels; @@ -2793,14 +2933,14 @@ brw_set_dp_untyped_atomic_message(struct brw_codegen *p, unsigned atomic_op, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned msg_control = atomic_op | /* Atomic Operation Type: BRW_AOP_* */ (response_expected ? 1 << 5 : 0); /* Return data expected */ if (devinfo->gen >= 8 || devinfo->is_haswell) { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (!p->compressed) + if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16) msg_control |= 1 << 4; /* SIMD8 mode */ brw_inst_set_dp_msg_type(devinfo, insn, @@ -2813,7 +2953,7 @@ brw_set_dp_untyped_atomic_message(struct brw_codegen *p, brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP); - if (!p->compressed) + if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16) msg_control |= 1 << 4; /* SIMD8 mode */ } @@ -2829,7 +2969,7 @@ brw_untyped_atomic(struct brw_codegen *p, unsigned msg_length, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN7_SFID_DATAPORT_DATA_CACHE); @@ -2856,12 +2996,12 @@ brw_set_dp_untyped_surface_read_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of 32-bit channels to drop. */ unsigned msg_control = 0xf & (0xf << num_channels); if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (p->compressed) + if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16) msg_control |= 1 << 4; /* SIMD16 mode */ else msg_control |= 2 << 4; /* SIMD8 mode */ @@ -2882,15 +3022,14 @@ brw_untyped_surface_read(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN7_SFID_DATAPORT_DATA_CACHE); - const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1); struct brw_inst *insn = brw_send_indirect_surface_message( p, sfid, dst, payload, surface, msg_length, brw_surface_payload_size(p, num_channels, true, true), - align1); + false); brw_set_dp_untyped_surface_read_message( p, insn, num_channels); @@ -2901,12 +3040,12 @@ brw_set_dp_untyped_surface_write_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of 32-bit channels to drop. */ unsigned msg_control = 0xf & (0xf << num_channels); if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (p->compressed) + if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16) msg_control |= 1 << 4; /* SIMD16 mode */ else msg_control |= 2 << 4; /* SIMD8 mode */ @@ -2931,7 +3070,7 @@ brw_untyped_surface_write(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN7_SFID_DATAPORT_DATA_CACHE); @@ -2953,14 +3092,14 @@ brw_set_dp_typed_atomic_message(struct brw_codegen *p, unsigned atomic_op, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned msg_control = atomic_op | /* Atomic Operation Type: BRW_AOP_* */ (response_expected ? 1 << 5 : 0); /* Return data expected */ if (devinfo->gen >= 8 || devinfo->is_haswell) { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1) msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */ brw_inst_set_dp_msg_type(devinfo, insn, @@ -2974,7 +3113,7 @@ brw_set_dp_typed_atomic_message(struct brw_codegen *p, brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_RC_TYPED_ATOMIC_OP); - if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1) msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */ } @@ -2989,7 +3128,7 @@ brw_typed_atomic(struct brw_codegen *p, unsigned atomic_op, unsigned msg_length, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN6_SFID_DATAPORT_RENDER_CACHE); @@ -3011,13 +3150,13 @@ brw_set_dp_typed_surface_read_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of unused channels. */ unsigned msg_control = 0xf & (0xf << num_channels); if (devinfo->gen >= 8 || devinfo->is_haswell) { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1) msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */ else msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */ @@ -3027,7 +3166,7 @@ brw_set_dp_typed_surface_read_message(struct brw_codegen *p, HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ); } else { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1) msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */ } @@ -3046,7 +3185,7 @@ brw_typed_surface_read(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN6_SFID_DATAPORT_RENDER_CACHE); @@ -3065,13 +3204,13 @@ brw_set_dp_typed_surface_write_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of unused channels. */ unsigned msg_control = 0xf & (0xf << num_channels); if (devinfo->gen >= 8 || devinfo->is_haswell) { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1) msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */ else msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */ @@ -3082,7 +3221,7 @@ brw_set_dp_typed_surface_write_message(struct brw_codegen *p, } else { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1) msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */ } @@ -3100,7 +3239,7 @@ brw_typed_surface_write(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN6_SFID_DATAPORT_RENDER_CACHE); @@ -3122,7 +3261,7 @@ brw_set_memory_fence_message(struct brw_codegen *p, enum brw_message_target sfid, bool commit_enable) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_message_descriptor(p, insn, sfid, 1 /* message length */, @@ -3149,14 +3288,20 @@ void brw_memory_fence(struct brw_codegen *p, struct brw_reg dst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const bool commit_enable = devinfo->gen == 7 && !devinfo->is_haswell; struct brw_inst *insn; + brw_push_insn_state(p); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_exec_size(p, BRW_EXECUTE_1); + dst = vec1(dst); + /* Set dst as destination for dependency tracking, the MEMORY_FENCE * message doesn't write anything back. */ insn = next_insn(p, BRW_OPCODE_SEND); + dst = retype(dst, BRW_REGISTER_TYPE_UW); brw_set_dest(p, insn, dst); brw_set_src0(p, insn, dst); brw_set_memory_fence_message(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE, @@ -3178,12 +3323,10 @@ brw_memory_fence(struct brw_codegen *p, * cache messages will be properly ordered with respect to past data and * render cache messages. */ - brw_push_insn_state(p); - brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); - brw_set_default_mask_control(p, BRW_MASK_DISABLE); brw_MOV(p, dst, offset(dst, 1)); - brw_pop_insn_state(p); } + + brw_pop_insn_state(p); } void @@ -3192,32 +3335,37 @@ brw_pixel_interpolator_query(struct brw_codegen *p, struct brw_reg mrf, bool noperspective, unsigned mode, - unsigned data, + struct brw_reg data, unsigned msg_length, unsigned response_length) { - const struct brw_device_info *devinfo = p->devinfo; - struct brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); - - brw_set_dest(p, insn, dest); - brw_set_src0(p, insn, mrf); - brw_set_message_descriptor(p, insn, GEN7_SFID_PIXEL_INTERPOLATOR, - msg_length, response_length, - false /* header is never present for PI */, - false); + const struct gen_device_info *devinfo = p->devinfo; + struct brw_inst *insn; + const uint16_t exec_size = brw_inst_exec_size(devinfo, p->current); - brw_inst_set_pi_simd_mode( - devinfo, insn, brw_inst_exec_size(devinfo, insn) == BRW_EXECUTE_16); + /* brw_send_indirect_message will automatically use a direct send message + * if data is actually immediate. + */ + insn = brw_send_indirect_message(p, + GEN7_SFID_PIXEL_INTERPOLATOR, + dest, + mrf, + vec1(data)); + brw_inst_set_mlen(devinfo, insn, msg_length); + brw_inst_set_rlen(devinfo, insn, response_length); + + brw_inst_set_pi_simd_mode(devinfo, insn, exec_size == BRW_EXECUTE_16); brw_inst_set_pi_slot_group(devinfo, insn, 0); /* zero unless 32/64px dispatch */ brw_inst_set_pi_nopersp(devinfo, insn, noperspective); brw_inst_set_pi_message_type(devinfo, insn, mode); - brw_inst_set_pi_message_data(devinfo, insn, data); } void brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; + const unsigned exec_size = 1 << brw_inst_exec_size(devinfo, p->current); + const unsigned qtr_control = brw_inst_qtr_control(devinfo, p->current); brw_inst *inst; assert(devinfo->gen >= 7); @@ -3238,26 +3386,38 @@ brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst) retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD)); /* Quarter control has the effect of magically shifting the value of - * this register. Make sure it's set to zero. + * this register so you'll get the first active channel relative to + * the specified quarter control as result. */ - brw_inst_set_qtr_control(devinfo, inst, GEN6_COMPRESSION_1Q); } else { - const struct brw_reg flag = retype(brw_flag_reg(1, 0), - BRW_REGISTER_TYPE_UD); + const struct brw_reg flag = brw_flag_reg(1, 0); - brw_MOV(p, flag, brw_imm_ud(0)); + brw_MOV(p, retype(flag, BRW_REGISTER_TYPE_UD), brw_imm_ud(0)); - /* Run a 16-wide instruction returning zero with execution masking - * and a conditional modifier enabled in order to get the current - * execution mask in f1.0. + /* Run enough instructions returning zero with execution masking and + * a conditional modifier enabled in order to get the full execution + * mask in f1.0. We could use a single 32-wide move here if it + * weren't because of the hardware bug that causes channel enables to + * be applied incorrectly to the second half of 32-wide instructions + * on Gen7. */ - inst = brw_MOV(p, brw_null_reg(), brw_imm_ud(0)); - brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_16); - brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); - brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z); - brw_inst_set_flag_reg_nr(devinfo, inst, 1); + const unsigned lower_size = MIN2(16, exec_size); + for (unsigned i = 0; i < exec_size / lower_size; i++) { + inst = brw_MOV(p, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW), + brw_imm_uw(0)); + brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); + brw_inst_set_group(devinfo, inst, lower_size * i + 8 * qtr_control); + brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z); + brw_inst_set_flag_reg_nr(devinfo, inst, 1); + brw_inst_set_exec_size(devinfo, inst, cvt(lower_size) - 1); + } - brw_FBL(p, vec1(dst), flag); + /* Find the first bit set in the exec_size-wide portion of the flag + * register that was updated by the last sequence of MOV + * instructions. + */ + const enum brw_reg_type type = brw_int_type(exec_size / 8, false); + brw_FBL(p, vec1(dst), byte_offset(retype(flag, type), qtr_control)); } } else { brw_set_default_mask_control(p, BRW_MASK_DISABLE); @@ -3274,11 +3434,14 @@ brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst) /* Overwrite the destination without and with execution masking to * find out which of the channels is active. */ + brw_push_insn_state(p); + brw_set_default_exec_size(p, BRW_EXECUTE_4); brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X), brw_imm_ud(1)); inst = brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X), brw_imm_ud(0)); + brw_pop_insn_state(p); brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); } } @@ -3292,10 +3455,14 @@ brw_broadcast(struct brw_codegen *p, struct brw_reg src, struct brw_reg idx) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1; brw_inst *inst; + brw_push_insn_state(p); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_exec_size(p, align1 ? BRW_EXECUTE_1 : BRW_EXECUTE_4); + assert(src.file == BRW_GENERAL_REGISTER_FILE && src.address_mode == BRW_ADDRESS_DIRECT); @@ -3305,7 +3472,7 @@ brw_broadcast(struct brw_codegen *p, * We will typically not get here if the optimizer is doing its job, but * asserting would be mean. */ - const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.dw1.ud : 0; + const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.ud : 0; brw_MOV(p, dst, (align1 ? stride(suboffset(src, i), 0, 1, 0) : stride(suboffset(src, 4 * i), 0, 4, 1))); @@ -3346,19 +3513,21 @@ brw_broadcast(struct brw_codegen *p, */ inst = brw_MOV(p, brw_null_reg(), - stride(brw_swizzle1(idx, 0), 0, 4, 1)); + stride(brw_swizzle(idx, BRW_SWIZZLE_XXXX), 4, 4, 1)); brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NONE); brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_NZ); brw_inst_set_flag_reg_nr(devinfo, inst, 1); /* and use predicated SEL to pick the right channel. */ inst = brw_SEL(p, dst, - stride(suboffset(src, 4), 0, 4, 1), - stride(src, 0, 4, 1)); + stride(suboffset(src, 4), 4, 4, 1), + stride(src, 4, 4, 1)); brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NORMAL); brw_inst_set_flag_reg_nr(devinfo, inst, 1); } } + + brw_pop_insn_state(p); } /** @@ -3414,13 +3583,13 @@ void brw_shader_time_add(struct brw_codegen *p, void brw_barrier(struct brw_codegen *p, struct brw_reg src) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *inst; assert(devinfo->gen >= 7); inst = next_insn(p, BRW_OPCODE_SEND); - brw_set_dest(p, inst, brw_null_reg()); + brw_set_dest(p, inst, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW)); brw_set_src0(p, inst, src); brw_set_src1(p, inst, brw_null_reg()); @@ -3444,7 +3613,7 @@ brw_barrier(struct brw_codegen *p, struct brw_reg src) void brw_WAIT(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *insn; struct brw_reg src = brw_notification_reg();