X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fbrw_eu_emit.c;h=3b12030ec0c1f66562b3c8e4b4ae73b293e2f4d5;hb=69fdf13c215c2970feaca76f178a5c2c11ba8fec;hp=9dba818b1b30ab7b46be70cce957656d28e3ff27;hpb=fc7107de1d7cac6be817e8951e53f997c248c277;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c index 9dba818b1b3..3b12030ec0c 100644 --- a/src/mesa/drivers/dri/i965/brw_eu_emit.c +++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c @@ -48,7 +48,7 @@ gen6_resolve_implied_move(struct brw_codegen *p, struct brw_reg *src, unsigned msg_reg_nr) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen < 6) return; @@ -78,7 +78,7 @@ gen7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg) * Since we're pretending to have 16 MRFs anyway, we may as well use the * registers required for messages with EOT. */ - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) { reg->file = BRW_GENERAL_REGISTER_FILE; reg->nr += GEN7_MRF_HACK_START; @@ -91,7 +91,7 @@ gen7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg) * The hardware encoding may depend on whether the value is an immediate. */ unsigned -brw_reg_type_to_hw_type(const struct brw_device_info *devinfo, +brw_reg_type_to_hw_type(const struct gen_device_info *devinfo, enum brw_reg_type type, enum brw_reg_file file) { if (file == BRW_IMMEDIATE_VALUE) { @@ -144,7 +144,7 @@ brw_reg_type_to_hw_type(const struct brw_device_info *devinfo, void brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (dest.file == BRW_MESSAGE_REGISTER_FILE) assert((dest.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen)); @@ -222,7 +222,7 @@ brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) extern int reg_type_size[]; static void -validate_reg(const struct brw_device_info *devinfo, +validate_reg(const struct gen_device_info *devinfo, brw_inst *inst, struct brw_reg reg) { const int hstride_for_reg[] = {0, 1, 2, 4}; @@ -320,7 +320,7 @@ is_compactable_immediate(unsigned imm) void brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (reg.file == BRW_MESSAGE_REGISTER_FILE) assert((reg.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen)); @@ -350,7 +350,8 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) brw_inst_set_src0_address_mode(devinfo, inst, reg.address_mode); if (reg.file == BRW_IMMEDIATE_VALUE) { - if (reg.type == BRW_REGISTER_TYPE_DF) + if (reg.type == BRW_REGISTER_TYPE_DF || + brw_inst_opcode(devinfo, inst) == BRW_OPCODE_DIM) brw_inst_set_imm_df(devinfo, inst, reg.df); else brw_inst_set_imm_ud(devinfo, inst, reg.ud); @@ -472,7 +473,7 @@ brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) void brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE) assert(reg.nr < 128); @@ -567,7 +568,7 @@ brw_set_message_descriptor(struct brw_codegen *p, bool header_present, bool end_of_thread) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_src1(p, inst, brw_imm_d(0)); @@ -599,7 +600,7 @@ static void brw_set_math_message( struct brw_codegen *p, bool low_precision, unsigned dataType ) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned msg_length; unsigned response_length; @@ -645,7 +646,7 @@ static void brw_set_ff_sync_message(struct brw_codegen *p, unsigned response_length, bool end_of_thread) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_message_descriptor(p, insn, BRW_SFID_URB, 1, response_length, true, end_of_thread); @@ -666,7 +667,7 @@ static void brw_set_urb_message( struct brw_codegen *p, unsigned offset, unsigned swizzle_control ) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; assert(devinfo->gen < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE); assert(devinfo->gen < 7 || !(flags & BRW_URB_WRITE_ALLOCATE)); @@ -712,7 +713,7 @@ brw_set_dp_write_message(struct brw_codegen *p, unsigned end_of_thread, unsigned send_commit_msg) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned sfid; if (devinfo->gen >= 7) { @@ -751,11 +752,19 @@ brw_set_dp_read_message(struct brw_codegen *p, bool header_present, unsigned response_length) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned sfid; if (devinfo->gen >= 7) { - sfid = GEN7_SFID_DATAPORT_DATA_CACHE; + if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE) + sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; + else if (target_cache == BRW_DATAPORT_READ_TARGET_DATA_CACHE) + sfid = GEN7_SFID_DATAPORT_DATA_CACHE; + else if (target_cache == BRW_DATAPORT_READ_TARGET_SAMPLER_CACHE) + sfid = GEN6_SFID_DATAPORT_SAMPLER_CACHE; + else + unreachable("Invalid target cache"); + } else if (devinfo->gen == 6) { if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE) sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; @@ -787,7 +796,7 @@ brw_set_sampler_message(struct brw_codegen *p, unsigned simd_mode, unsigned return_format) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_message_descriptor(p, inst, BRW_SFID_SAMPLER, msg_length, response_length, header_present, false); @@ -814,16 +823,19 @@ gen7_set_dp_scratch_message(struct brw_codegen *p, unsigned rlen, bool header_present) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; assert(num_regs == 1 || num_regs == 2 || num_regs == 4 || (devinfo->gen >= 8 && num_regs == 8)); + const unsigned block_size = (devinfo->gen >= 8 ? _mesa_logbase2(num_regs) : + num_regs - 1); + brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE, mlen, rlen, header_present, false); brw_inst_set_dp_category(devinfo, inst, 1); /* Scratch Block Read/Write msgs */ brw_inst_set_scratch_read_write(devinfo, inst, write); brw_inst_set_scratch_type(devinfo, inst, dword); brw_inst_set_scratch_invalidate_after_read(devinfo, inst, invalidate_after_read); - brw_inst_set_scratch_block_size(devinfo, inst, ffs(num_regs) - 1); + brw_inst_set_scratch_block_size(devinfo, inst, block_size); brw_inst_set_scratch_addr_offset(devinfo, inst, addr_offset); } @@ -831,7 +843,7 @@ gen7_set_dp_scratch_message(struct brw_codegen *p, brw_inst * brw_next_insn(struct brw_codegen *p, unsigned opcode) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; if (p->nr_insn + 1 > p->store_size) { @@ -886,7 +898,7 @@ static brw_inst * brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, struct brw_reg src0, struct brw_reg src1, struct brw_reg src2) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *inst = next_insn(p, opcode); gen7_convert_mrf_to_grf(p, &dest); @@ -1038,7 +1050,7 @@ void brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src) \ { \ - const struct brw_device_info *devinfo = p->devinfo; \ + const struct gen_device_info *devinfo = p->devinfo; \ brw_inst *rnd, *add; \ rnd = next_insn(p, BRW_OPCODE_##OP); \ brw_set_dest(p, rnd, dest); \ @@ -1061,6 +1073,7 @@ ALU2(OR) ALU2(XOR) ALU2(SHR) ALU2(SHL) +ALU1(DIM) ALU2(ASR) ALU1(FRC) ALU1(RNDD) @@ -1190,7 +1203,7 @@ brw_PLN(struct brw_codegen *p, struct brw_reg dest, brw_inst * brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16; /* The F32TO16 instruction doesn't support 32-bit destination types in * Align1 mode, and neither does the Gen8 implementation in terms of a @@ -1237,7 +1250,7 @@ brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) brw_inst * brw_F16TO32(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16; if (align16) { @@ -1287,7 +1300,7 @@ brw_inst * brw_JMPI(struct brw_codegen *p, struct brw_reg index, unsigned predicate_control) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_reg ip = brw_ip_reg(); brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index); @@ -1357,7 +1370,7 @@ get_inner_do_insn(struct brw_codegen *p) brw_inst * brw_IF(struct brw_codegen *p, unsigned execute_size) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_IF); @@ -1405,7 +1418,7 @@ brw_inst * gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional, struct brw_reg src0, struct brw_reg src1) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_IF); @@ -1432,7 +1445,7 @@ static void convert_IF_ELSE_to_ADD(struct brw_codegen *p, brw_inst *if_inst, brw_inst *else_inst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* The next instruction (where the ENDIF would be, if it existed) */ brw_inst *next_inst = &p->store[p->nr_insn]; @@ -1473,7 +1486,7 @@ static void patch_IF_ELSE(struct brw_codegen *p, brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* We shouldn't be patching IF and ELSE instructions in single program flow * mode when gen < 6, because in single program flow mode on those @@ -1560,7 +1573,7 @@ patch_IF_ELSE(struct brw_codegen *p, void brw_ELSE(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_ELSE); @@ -1598,7 +1611,7 @@ brw_ELSE(struct brw_codegen *p) void brw_ENDIF(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = NULL; brw_inst *else_inst = NULL; brw_inst *if_inst = NULL; @@ -1679,7 +1692,7 @@ brw_ENDIF(struct brw_codegen *p) brw_inst * brw_BREAK(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_BREAK); @@ -1707,7 +1720,7 @@ brw_BREAK(struct brw_codegen *p) brw_inst * brw_CONT(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_CONTINUE); @@ -1732,7 +1745,7 @@ brw_CONT(struct brw_codegen *p) brw_inst * gen6_HALT(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; insn = next_insn(p, BRW_OPCODE_HALT); @@ -1769,7 +1782,7 @@ gen6_HALT(struct brw_codegen *p) brw_inst * brw_DO(struct brw_codegen *p, unsigned execute_size) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen >= 6 || p->single_program_flow) { push_loop_stack(p, &p->store[p->nr_insn]); @@ -1803,7 +1816,7 @@ brw_DO(struct brw_codegen *p, unsigned execute_size) static void brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *do_inst = get_inner_do_insn(p); brw_inst *inst; unsigned br = brw_jump_scale(devinfo); @@ -1828,7 +1841,7 @@ brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst) brw_inst * brw_WHILE(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn, *do_insn; unsigned br = brw_jump_scale(devinfo); @@ -1892,7 +1905,7 @@ brw_WHILE(struct brw_codegen *p) */ void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *jmp_insn = &p->store[jmp_insn_idx]; unsigned jmpi = 1; @@ -1916,7 +1929,7 @@ void brw_CMP(struct brw_codegen *p, struct brw_reg src0, struct brw_reg src1) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_CMP); brw_inst_set_cond_modifier(devinfo, insn, conditional); @@ -1952,7 +1965,7 @@ void gen4_math(struct brw_codegen *p, struct brw_reg src, unsigned precision ) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); unsigned data_type; if (has_scalar_region(src)) { @@ -1985,7 +1998,7 @@ void gen6_math(struct brw_codegen *p, struct brw_reg src0, struct brw_reg src1) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_MATH); assert(devinfo->gen >= 6); @@ -2061,22 +2074,20 @@ void brw_oword_block_write_scratch(struct brw_codegen *p, int num_regs, unsigned offset) { - const struct brw_device_info *devinfo = p->devinfo; - uint32_t msg_control, msg_type; - int mlen; + const struct gen_device_info *devinfo = p->devinfo; + uint32_t msg_type; if (devinfo->gen >= 6) offset /= 16; mrf = retype(mrf, BRW_REGISTER_TYPE_UD); - if (num_regs == 1) { - msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS; - mlen = 2; - } else { - msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS; - mlen = 3; - } + const unsigned mlen = 1 + num_regs; + const unsigned msg_control = + (num_regs == 1 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS : + num_regs == 2 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS : + num_regs == 4 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : 0); + assert(msg_control); /* Set up the message header. This is g0, with g0.2 filled with * the offset. We don't want to leave our offset around in g0 or @@ -2176,9 +2187,7 @@ brw_oword_block_read_scratch(struct brw_codegen *p, int num_regs, unsigned offset) { - const struct brw_device_info *devinfo = p->devinfo; - uint32_t msg_control; - int rlen; + const struct gen_device_info *devinfo = p->devinfo; if (devinfo->gen >= 6) offset /= 16; @@ -2197,13 +2206,15 @@ brw_oword_block_read_scratch(struct brw_codegen *p, } dest = retype(dest, BRW_REGISTER_TYPE_UW); - if (num_regs == 1) { - msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS; - rlen = 1; - } else { - msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS; - rlen = 2; - } + const unsigned rlen = num_regs; + const unsigned msg_control = + (num_regs == 1 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS : + num_regs == 2 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS : + num_regs == 4 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : 0); + assert(msg_control); + const unsigned target_cache = devinfo->gen >= 7 ? + BRW_DATAPORT_READ_TARGET_DATA_CACHE : + BRW_DATAPORT_READ_TARGET_RENDER_CACHE; { brw_push_insn_state(p); @@ -2238,7 +2249,7 @@ brw_oword_block_read_scratch(struct brw_codegen *p, brw_scratch_surface_idx(p), msg_control, BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ, /* msg_type */ - BRW_DATAPORT_READ_TARGET_RENDER_CACHE, + target_cache, 1, /* msg_length */ true, /* header_present */ rlen); @@ -2251,7 +2262,7 @@ gen7_block_read_scratch(struct brw_codegen *p, int num_regs, unsigned offset) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE); @@ -2291,7 +2302,7 @@ void brw_oword_block_read(struct brw_codegen *p, uint32_t offset, uint32_t bind_table_index) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* On newer hardware, offset is in units of owords. */ if (devinfo->gen >= 6) @@ -2352,7 +2363,7 @@ void brw_fb_WRITE(struct brw_codegen *p, bool last_render_target, bool header_present) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; unsigned msg_type; struct brw_reg dest, src0; @@ -2397,6 +2408,34 @@ void brw_fb_WRITE(struct brw_codegen *p, 0 /* send_commit_msg */); } +brw_inst * +gen9_fb_READ(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg payload, + unsigned binding_table_index, + unsigned msg_length, + unsigned response_length, + bool per_sample) +{ + const struct gen_device_info *devinfo = p->devinfo; + assert(devinfo->gen >= 9); + const unsigned msg_subtype = + brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16 ? 0 : 1; + brw_inst *insn = next_insn(p, BRW_OPCODE_SENDC); + + brw_set_dest(p, insn, dst); + brw_set_src0(p, insn, payload); + brw_set_dp_read_message(p, insn, binding_table_index, + per_sample << 5 | msg_subtype, + GEN9_DATAPORT_RC_RENDER_TARGET_READ, + BRW_DATAPORT_READ_TARGET_RENDER_CACHE, + msg_length, true /* header_present */, + response_length); + brw_inst_set_rt_slot_group(devinfo, insn, + brw_inst_qtr_control(devinfo, p->current) / 2); + + return insn; +} /** * Texture sample instruction. @@ -2416,7 +2455,7 @@ void brw_SAMPLE(struct brw_codegen *p, unsigned simd_mode, unsigned return_format) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; if (msg_reg_nr != -1) @@ -2471,7 +2510,7 @@ void brw_adjust_sampler_state_pointer(struct brw_codegen *p, * exclusively use the offset - we have to use both. */ - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; if (sampler_index.file == BRW_IMMEDIATE_VALUE) { const int sampler_state_size = 16; /* 16 bytes */ @@ -2515,7 +2554,7 @@ void brw_urb_WRITE(struct brw_codegen *p, unsigned offset, unsigned swizzle) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); @@ -2559,7 +2598,7 @@ brw_send_indirect_message(struct brw_codegen *p, struct brw_reg payload, struct brw_reg desc) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *send; int setup; @@ -2619,7 +2658,7 @@ brw_send_indirect_surface_message(struct brw_codegen *p, unsigned response_len, bool header_present) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *insn; if (surface.file != BRW_IMMEDIATE_VALUE) { @@ -2652,7 +2691,7 @@ brw_send_indirect_surface_message(struct brw_codegen *p, } static bool -while_jumps_before_offset(const struct brw_device_info *devinfo, +while_jumps_before_offset(const struct gen_device_info *devinfo, brw_inst *insn, int while_offset, int start_offset) { int scale = 16 / brw_jump_scale(devinfo); @@ -2667,7 +2706,7 @@ brw_find_next_block_end(struct brw_codegen *p, int start_offset) { int offset; void *store = p->store; - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; int depth = 0; @@ -2709,7 +2748,7 @@ brw_find_next_block_end(struct brw_codegen *p, int start_offset) static int brw_find_loop_end(struct brw_codegen *p, int start_offset) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; int offset; void *store = p->store; @@ -2736,9 +2775,9 @@ brw_find_loop_end(struct brw_codegen *p, int start_offset) * BREAK, CONT, and HALT instructions to their correct locations. */ void -brw_set_uip_jip(struct brw_codegen *p) +brw_set_uip_jip(struct brw_codegen *p, int start_offset) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; int offset; int br = brw_jump_scale(devinfo); int scale = 16 / br; @@ -2747,17 +2786,9 @@ brw_set_uip_jip(struct brw_codegen *p) if (devinfo->gen < 6) return; - for (offset = 0; offset < p->next_insn_offset; - offset = next_offset(devinfo, store, offset)) { + for (offset = start_offset; offset < p->next_insn_offset; offset += 16) { brw_inst *insn = store + offset; - - if (brw_inst_cmpt_control(devinfo, insn)) { - /* Fixups for compacted BREAK/CONTINUE not supported yet. */ - assert(brw_inst_opcode(devinfo, insn) != BRW_OPCODE_BREAK && - brw_inst_opcode(devinfo, insn) != BRW_OPCODE_CONTINUE && - brw_inst_opcode(devinfo, insn) != BRW_OPCODE_HALT); - continue; - } + assert(brw_inst_cmpt_control(devinfo, insn) == 0); int block_end_offset = brw_find_next_block_end(p, offset); switch (brw_inst_opcode(devinfo, insn)) { @@ -2821,7 +2852,7 @@ void brw_ff_sync(struct brw_codegen *p, unsigned response_length, bool eot) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); @@ -2902,14 +2933,14 @@ brw_set_dp_untyped_atomic_message(struct brw_codegen *p, unsigned atomic_op, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned msg_control = atomic_op | /* Atomic Operation Type: BRW_AOP_* */ (response_expected ? 1 << 5 : 0); /* Return data expected */ if (devinfo->gen >= 8 || devinfo->is_haswell) { if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { - if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_8) + if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16) msg_control |= 1 << 4; /* SIMD8 mode */ brw_inst_set_dp_msg_type(devinfo, insn, @@ -2922,7 +2953,7 @@ brw_set_dp_untyped_atomic_message(struct brw_codegen *p, brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP); - if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_8) + if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16) msg_control |= 1 << 4; /* SIMD8 mode */ } @@ -2938,7 +2969,7 @@ brw_untyped_atomic(struct brw_codegen *p, unsigned msg_length, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN7_SFID_DATAPORT_DATA_CACHE); @@ -2965,7 +2996,7 @@ brw_set_dp_untyped_surface_read_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of 32-bit channels to drop. */ unsigned msg_control = 0xf & (0xf << num_channels); @@ -2991,7 +3022,7 @@ brw_untyped_surface_read(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN7_SFID_DATAPORT_DATA_CACHE); @@ -3009,7 +3040,7 @@ brw_set_dp_untyped_surface_write_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of 32-bit channels to drop. */ unsigned msg_control = 0xf & (0xf << num_channels); @@ -3039,7 +3070,7 @@ brw_untyped_surface_write(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN7_SFID_DATAPORT_DATA_CACHE); @@ -3061,7 +3092,7 @@ brw_set_dp_typed_atomic_message(struct brw_codegen *p, unsigned atomic_op, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; unsigned msg_control = atomic_op | /* Atomic Operation Type: BRW_AOP_* */ (response_expected ? 1 << 5 : 0); /* Return data expected */ @@ -3097,7 +3128,7 @@ brw_typed_atomic(struct brw_codegen *p, unsigned atomic_op, unsigned msg_length, bool response_expected) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN6_SFID_DATAPORT_RENDER_CACHE); @@ -3119,7 +3150,7 @@ brw_set_dp_typed_surface_read_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of unused channels. */ unsigned msg_control = 0xf & (0xf << num_channels); @@ -3154,7 +3185,7 @@ brw_typed_surface_read(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN6_SFID_DATAPORT_RENDER_CACHE); @@ -3173,7 +3204,7 @@ brw_set_dp_typed_surface_write_message(struct brw_codegen *p, struct brw_inst *insn, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; /* Set mask of unused channels. */ unsigned msg_control = 0xf & (0xf << num_channels); @@ -3208,7 +3239,7 @@ brw_typed_surface_write(struct brw_codegen *p, unsigned msg_length, unsigned num_channels) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? HSW_SFID_DATAPORT_DATA_CACHE_1 : GEN6_SFID_DATAPORT_RENDER_CACHE); @@ -3230,7 +3261,7 @@ brw_set_memory_fence_message(struct brw_codegen *p, enum brw_message_target sfid, bool commit_enable) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; brw_set_message_descriptor(p, insn, sfid, 1 /* message length */, @@ -3257,7 +3288,7 @@ void brw_memory_fence(struct brw_codegen *p, struct brw_reg dst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const bool commit_enable = devinfo->gen == 7 && !devinfo->is_haswell; struct brw_inst *insn; @@ -3308,7 +3339,7 @@ brw_pixel_interpolator_query(struct brw_codegen *p, unsigned msg_length, unsigned response_length) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *insn; const uint16_t exec_size = brw_inst_exec_size(devinfo, p->current); @@ -3332,7 +3363,9 @@ brw_pixel_interpolator_query(struct brw_codegen *p, void brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; + const unsigned exec_size = 1 << brw_inst_exec_size(devinfo, p->current); + const unsigned qtr_control = brw_inst_qtr_control(devinfo, p->current); brw_inst *inst; assert(devinfo->gen >= 7); @@ -3353,26 +3386,38 @@ brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst) retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD)); /* Quarter control has the effect of magically shifting the value of - * this register. Make sure it's set to zero. + * this register so you'll get the first active channel relative to + * the specified quarter control as result. */ - brw_inst_set_qtr_control(devinfo, inst, GEN6_COMPRESSION_1Q); } else { - const struct brw_reg flag = retype(brw_flag_reg(1, 0), - BRW_REGISTER_TYPE_UD); + const struct brw_reg flag = brw_flag_reg(1, 0); - brw_MOV(p, flag, brw_imm_ud(0)); + brw_MOV(p, retype(flag, BRW_REGISTER_TYPE_UD), brw_imm_ud(0)); - /* Run a 16-wide instruction returning zero with execution masking - * and a conditional modifier enabled in order to get the current - * execution mask in f1.0. + /* Run enough instructions returning zero with execution masking and + * a conditional modifier enabled in order to get the full execution + * mask in f1.0. We could use a single 32-wide move here if it + * weren't because of the hardware bug that causes channel enables to + * be applied incorrectly to the second half of 32-wide instructions + * on Gen7. */ - inst = brw_MOV(p, brw_null_reg(), brw_imm_ud(0)); - brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_16); - brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); - brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z); - brw_inst_set_flag_reg_nr(devinfo, inst, 1); + const unsigned lower_size = MIN2(16, exec_size); + for (unsigned i = 0; i < exec_size / lower_size; i++) { + inst = brw_MOV(p, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW), + brw_imm_uw(0)); + brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); + brw_inst_set_group(devinfo, inst, lower_size * i + 8 * qtr_control); + brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z); + brw_inst_set_flag_reg_nr(devinfo, inst, 1); + brw_inst_set_exec_size(devinfo, inst, cvt(lower_size) - 1); + } - brw_FBL(p, vec1(dst), flag); + /* Find the first bit set in the exec_size-wide portion of the flag + * register that was updated by the last sequence of MOV + * instructions. + */ + const enum brw_reg_type type = brw_int_type(exec_size / 8, false); + brw_FBL(p, vec1(dst), byte_offset(retype(flag, type), qtr_control)); } } else { brw_set_default_mask_control(p, BRW_MASK_DISABLE); @@ -3410,10 +3455,14 @@ brw_broadcast(struct brw_codegen *p, struct brw_reg src, struct brw_reg idx) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1; brw_inst *inst; + brw_push_insn_state(p); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_exec_size(p, align1 ? BRW_EXECUTE_1 : BRW_EXECUTE_4); + assert(src.file == BRW_GENERAL_REGISTER_FILE && src.address_mode == BRW_ADDRESS_DIRECT); @@ -3464,19 +3513,21 @@ brw_broadcast(struct brw_codegen *p, */ inst = brw_MOV(p, brw_null_reg(), - stride(brw_swizzle(idx, BRW_SWIZZLE_XXXX), 0, 4, 1)); + stride(brw_swizzle(idx, BRW_SWIZZLE_XXXX), 4, 4, 1)); brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NONE); brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_NZ); brw_inst_set_flag_reg_nr(devinfo, inst, 1); /* and use predicated SEL to pick the right channel. */ inst = brw_SEL(p, dst, - stride(suboffset(src, 4), 0, 4, 1), - stride(src, 0, 4, 1)); + stride(suboffset(src, 4), 4, 4, 1), + stride(src, 4, 4, 1)); brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NORMAL); brw_inst_set_flag_reg_nr(devinfo, inst, 1); } } + + brw_pop_insn_state(p); } /** @@ -3532,7 +3583,7 @@ void brw_shader_time_add(struct brw_codegen *p, void brw_barrier(struct brw_codegen *p, struct brw_reg src) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *inst; assert(devinfo->gen >= 7); @@ -3562,7 +3613,7 @@ brw_barrier(struct brw_codegen *p, struct brw_reg src) void brw_WAIT(struct brw_codegen *p) { - const struct brw_device_info *devinfo = p->devinfo; + const struct gen_device_info *devinfo = p->devinfo; struct brw_inst *insn; struct brw_reg src = brw_notification_reg();