X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fdrivers%2Fdri%2Fi965%2Fbrw_eu_emit.c;h=4d397622fc122f8e3e6125541f8821d40fd6d7a8;hb=8765f1d7ddfb00dc5b202e4e679ebe640a547d50;hp=5213f5ab54143370264c4ee163836954c8c8f2e5;hpb=d542c45c75671e93a1cfae1f0eaf9c12f082f4f1;p=mesa.git diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c index 5213f5ab541..4d397622fc1 100644 --- a/src/mesa/drivers/dri/i965/brw_eu_emit.c +++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c @@ -1,6 +1,6 @@ /* Copyright (C) Intel Corp. 2006. All Rights Reserved. - Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to + Intel funded Tungsten Graphics to develop this 3D driver. Permission is hereby granted, free of charge, to any person obtaining @@ -26,7 +26,7 @@ **********************************************************************/ /* * Authors: - * Keith Whitwell + * Keith Whitwell */ @@ -34,22 +34,7 @@ #include "brw_defines.h" #include "brw_eu.h" -#include "glsl/ralloc.h" - -/*********************************************************************** - * Internal helper for constructing instructions - */ - -static void guess_execution_size(struct brw_compile *p, - struct brw_instruction *insn, - struct brw_reg reg) -{ - if (reg.width == BRW_WIDTH_8 && p->compressed) - insn->header.execution_size = BRW_EXECUTE_16; - else - insn->header.execution_size = reg.width; /* note - definitions are compatible */ -} - +#include "util/ralloc.h" /** * Prior to Sandybridge, the SEND instruction accepted non-MRF source @@ -59,12 +44,12 @@ static void guess_execution_size(struct brw_compile *p, * explicit move; it should be called before emitting a SEND instruction. */ void -gen6_resolve_implied_move(struct brw_compile *p, +gen6_resolve_implied_move(struct brw_codegen *p, struct brw_reg *src, - GLuint msg_reg_nr) + unsigned msg_reg_nr) { - struct brw_context *brw = p->brw; - if (brw->gen < 6) + const struct brw_device_info *devinfo = p->devinfo; + if (devinfo->gen < 6) return; if (src->file == BRW_MESSAGE_REGISTER_FILE) @@ -72,8 +57,9 @@ gen6_resolve_implied_move(struct brw_compile *p, if (src->file != BRW_ARCHITECTURE_REGISTER_FILE || src->nr != BRW_ARF_NULL) { brw_push_insn_state(p); - brw_set_mask_control(p, BRW_MASK_DISABLE); - brw_set_compression_control(p, BRW_COMPRESSION_NONE); + brw_set_default_exec_size(p, BRW_EXECUTE_8); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); brw_MOV(p, retype(brw_message_reg(msg_reg_nr), BRW_REGISTER_TYPE_UD), retype(*src, BRW_REGISTER_TYPE_UD)); brw_pop_insn_state(p); @@ -82,7 +68,7 @@ gen6_resolve_implied_move(struct brw_compile *p, } static void -gen7_convert_mrf_to_grf(struct brw_compile *p, struct brw_reg *reg) +gen7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg) { /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"): * "The send with EOT should use register space R112-R127 for . This is @@ -92,40 +78,97 @@ gen7_convert_mrf_to_grf(struct brw_compile *p, struct brw_reg *reg) * Since we're pretending to have 16 MRFs anyway, we may as well use the * registers required for messages with EOT. */ - struct brw_context *brw = p->brw; - if (brw->gen == 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) { + const struct brw_device_info *devinfo = p->devinfo; + if (devinfo->gen >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) { reg->file = BRW_GENERAL_REGISTER_FILE; reg->nr += GEN7_MRF_HACK_START; } } +/** + * Convert a brw_reg_type enumeration value into the hardware representation. + * + * The hardware encoding may depend on whether the value is an immediate. + */ +unsigned +brw_reg_type_to_hw_type(const struct brw_device_info *devinfo, + enum brw_reg_type type, unsigned file) +{ + if (file == BRW_IMMEDIATE_VALUE) { + const static int imm_hw_types[] = { + [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD, + [BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D, + [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW, + [BRW_REGISTER_TYPE_W] = BRW_HW_REG_TYPE_W, + [BRW_REGISTER_TYPE_F] = BRW_HW_REG_TYPE_F, + [BRW_REGISTER_TYPE_UB] = -1, + [BRW_REGISTER_TYPE_B] = -1, + [BRW_REGISTER_TYPE_UV] = BRW_HW_REG_IMM_TYPE_UV, + [BRW_REGISTER_TYPE_VF] = BRW_HW_REG_IMM_TYPE_VF, + [BRW_REGISTER_TYPE_V] = BRW_HW_REG_IMM_TYPE_V, + [BRW_REGISTER_TYPE_DF] = GEN8_HW_REG_IMM_TYPE_DF, + [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_IMM_TYPE_HF, + [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ, + [BRW_REGISTER_TYPE_Q] = GEN8_HW_REG_TYPE_Q, + }; + assert(type < ARRAY_SIZE(imm_hw_types)); + assert(imm_hw_types[type] != -1); + assert(devinfo->gen >= 8 || type < BRW_REGISTER_TYPE_DF); + return imm_hw_types[type]; + } else { + /* Non-immediate registers */ + const static int hw_types[] = { + [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD, + [BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D, + [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW, + [BRW_REGISTER_TYPE_W] = BRW_HW_REG_TYPE_W, + [BRW_REGISTER_TYPE_UB] = BRW_HW_REG_NON_IMM_TYPE_UB, + [BRW_REGISTER_TYPE_B] = BRW_HW_REG_NON_IMM_TYPE_B, + [BRW_REGISTER_TYPE_F] = BRW_HW_REG_TYPE_F, + [BRW_REGISTER_TYPE_UV] = -1, + [BRW_REGISTER_TYPE_VF] = -1, + [BRW_REGISTER_TYPE_V] = -1, + [BRW_REGISTER_TYPE_DF] = GEN7_HW_REG_NON_IMM_TYPE_DF, + [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_NON_IMM_TYPE_HF, + [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ, + [BRW_REGISTER_TYPE_Q] = GEN8_HW_REG_TYPE_Q, + }; + assert(type < ARRAY_SIZE(hw_types)); + assert(hw_types[type] != -1); + assert(devinfo->gen >= 7 || type < BRW_REGISTER_TYPE_DF); + assert(devinfo->gen >= 8 || type < BRW_REGISTER_TYPE_HF); + return hw_types[type]; + } +} void -brw_set_dest(struct brw_compile *p, struct brw_instruction *insn, - struct brw_reg dest) +brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest) { + const struct brw_device_info *devinfo = p->devinfo; + if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE && dest.file != BRW_MESSAGE_REGISTER_FILE) assert(dest.nr < 128); gen7_convert_mrf_to_grf(p, &dest); - insn->bits1.da1.dest_reg_file = dest.file; - insn->bits1.da1.dest_reg_type = dest.type; - insn->bits1.da1.dest_address_mode = dest.address_mode; + brw_inst_set_dst_reg_file(devinfo, inst, dest.file); + brw_inst_set_dst_reg_type(devinfo, inst, + brw_reg_type_to_hw_type(devinfo, dest.type, + dest.file)); + brw_inst_set_dst_address_mode(devinfo, inst, dest.address_mode); if (dest.address_mode == BRW_ADDRESS_DIRECT) { - insn->bits1.da1.dest_reg_nr = dest.nr; + brw_inst_set_dst_da_reg_nr(devinfo, inst, dest.nr); - if (insn->header.access_mode == BRW_ALIGN_1) { - insn->bits1.da1.dest_subreg_nr = dest.subnr; + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { + brw_inst_set_dst_da1_subreg_nr(devinfo, inst, dest.subnr); if (dest.hstride == BRW_HORIZONTAL_STRIDE_0) dest.hstride = BRW_HORIZONTAL_STRIDE_1; - insn->bits1.da1.dest_horiz_stride = dest.hstride; - } - else { - insn->bits1.da16.dest_subreg_nr = dest.subnr / 16; - insn->bits1.da16.dest_writemask = dest.dw1.bits.writemask; + brw_inst_set_dst_hstride(devinfo, inst, dest.hstride); + } else { + brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16); + brw_inst_set_da16_writemask(devinfo, inst, dest.dw1.bits.writemask); if (dest.file == BRW_GENERAL_REGISTER_FILE || dest.file == BRW_MESSAGE_REGISTER_FILE) { assert(dest.dw1.bits.writemask != 0); @@ -134,42 +177,45 @@ brw_set_dest(struct brw_compile *p, struct brw_instruction *insn, * Although Dst.HorzStride is a don't care for Align16, HW needs * this to be programmed as "01". */ - insn->bits1.da16.dest_horiz_stride = 1; + brw_inst_set_dst_hstride(devinfo, inst, 1); } - } - else { - insn->bits1.ia1.dest_subreg_nr = dest.subnr; + } else { + brw_inst_set_dst_ia_subreg_nr(devinfo, inst, dest.subnr); /* These are different sizes in align1 vs align16: */ - if (insn->header.access_mode == BRW_ALIGN_1) { - insn->bits1.ia1.dest_indirect_offset = dest.dw1.bits.indirect_offset; + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { + brw_inst_set_dst_ia1_addr_imm(devinfo, inst, + dest.dw1.bits.indirect_offset); if (dest.hstride == BRW_HORIZONTAL_STRIDE_0) dest.hstride = BRW_HORIZONTAL_STRIDE_1; - insn->bits1.ia1.dest_horiz_stride = dest.hstride; - } - else { - insn->bits1.ia16.dest_indirect_offset = dest.dw1.bits.indirect_offset; + brw_inst_set_dst_hstride(devinfo, inst, dest.hstride); + } else { + brw_inst_set_dst_ia16_addr_imm(devinfo, inst, + dest.dw1.bits.indirect_offset); /* even ignored in da16, still need to set as '01' */ - insn->bits1.ia16.dest_horiz_stride = 1; + brw_inst_set_dst_hstride(devinfo, inst, 1); } } - /* NEW: Set the execution size based on dest.width and - * insn->compression_control: + /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8) + * or 16 (SIMD16), as that's normally correct. However, when dealing with + * small registers, we automatically reduce it to match the register size. */ - guess_execution_size(p, insn, dest); + if (dest.width < BRW_EXECUTE_8) + brw_inst_set_exec_size(devinfo, inst, dest.width); } extern int reg_type_size[]; static void -validate_reg(struct brw_instruction *insn, struct brw_reg reg) +validate_reg(const struct brw_device_info *devinfo, + brw_inst *inst, struct brw_reg reg) { - int hstride_for_reg[] = {0, 1, 2, 4}; - int vstride_for_reg[] = {0, 1, 2, 4, 8, 16, 32, 64, 128, 256}; - int width_for_reg[] = {1, 2, 4, 8, 16}; - int execsize_for_reg[] = {1, 2, 4, 8, 16}; + const int hstride_for_reg[] = {0, 1, 2, 4}; + const int vstride_for_reg[] = {0, 1, 2, 4, 8, 16, 32}; + const int width_for_reg[] = {1, 2, 4, 8, 16}; + const int execsize_for_reg[] = {1, 2, 4, 8, 16, 32}; int width, hstride, vstride, execsize; if (reg.file == BRW_IMMEDIATE_VALUE) { @@ -178,8 +224,8 @@ validate_reg(struct brw_instruction *insn, struct brw_reg reg) * destination horiz stride has to be a word. */ if (reg.type == BRW_REGISTER_TYPE_V) { - assert(hstride_for_reg[insn->bits1.da1.dest_horiz_stride] * - reg_type_size[insn->bits1.da1.dest_reg_type] == 2); + assert(hstride_for_reg[brw_inst_dst_hstride(devinfo, inst)] * + reg_type_size[brw_inst_dst_reg_type(devinfo, inst)] == 2); } return; @@ -189,22 +235,22 @@ validate_reg(struct brw_instruction *insn, struct brw_reg reg) reg.file == BRW_ARF_NULL) return; - assert(reg.hstride >= 0 && reg.hstride < Elements(hstride_for_reg)); + assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg)); hstride = hstride_for_reg[reg.hstride]; if (reg.vstride == 0xf) { vstride = -1; } else { - assert(reg.vstride >= 0 && reg.vstride < Elements(vstride_for_reg)); + assert(reg.vstride >= 0 && reg.vstride < ARRAY_SIZE(vstride_for_reg)); vstride = vstride_for_reg[reg.vstride]; } - assert(reg.width >= 0 && reg.width < Elements(width_for_reg)); + assert(reg.width >= 0 && reg.width < ARRAY_SIZE(width_for_reg)); width = width_for_reg[reg.width]; - assert(insn->header.execution_size >= 0 && - insn->header.execution_size < Elements(execsize_for_reg)); - execsize = execsize_for_reg[insn->header.execution_size]; + assert(brw_inst_exec_size(devinfo, inst) >= 0 && + brw_inst_exec_size(devinfo, inst) < ARRAY_SIZE(execsize_for_reg)); + execsize = execsize_for_reg[brw_inst_exec_size(devinfo, inst)]; /* Restrictions from 3.3.10: Register Region Restrictions. */ /* 3. */ @@ -239,19 +285,28 @@ validate_reg(struct brw_instruction *insn, struct brw_reg reg) /* 10. Check destination issues. */ } +static bool +is_compactable_immediate(unsigned imm) +{ + /* We get the low 12 bits as-is. */ + imm &= ~0xfff; + + /* We get one bit replicated through the top 20 bits. */ + return imm == 0 || imm == 0xfffff000; +} + void -brw_set_src0(struct brw_compile *p, struct brw_instruction *insn, - struct brw_reg reg) +brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; - if (reg.type != BRW_ARCHITECTURE_REGISTER_FILE) + if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE) assert(reg.nr < 128); gen7_convert_mrf_to_grf(p, ®); - if (brw->gen >= 6 && (insn->header.opcode == BRW_OPCODE_SEND || - insn->header.opcode == BRW_OPCODE_SENDC)) { + if (devinfo->gen >= 6 && (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND || + brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC)) { /* Any source modifiers or regions will be ignored, since this just * identifies the MRF/GRF to start reading the message contents from. * Check for some likely failures. @@ -261,143 +316,192 @@ brw_set_src0(struct brw_compile *p, struct brw_instruction *insn, assert(reg.address_mode == BRW_ADDRESS_DIRECT); } - validate_reg(insn, reg); + validate_reg(devinfo, inst, reg); - insn->bits1.da1.src0_reg_file = reg.file; - insn->bits1.da1.src0_reg_type = reg.type; - insn->bits2.da1.src0_abs = reg.abs; - insn->bits2.da1.src0_negate = reg.negate; - insn->bits2.da1.src0_address_mode = reg.address_mode; + brw_inst_set_src0_reg_file(devinfo, inst, reg.file); + brw_inst_set_src0_reg_type(devinfo, inst, + brw_reg_type_to_hw_type(devinfo, reg.type, reg.file)); + brw_inst_set_src0_abs(devinfo, inst, reg.abs); + brw_inst_set_src0_negate(devinfo, inst, reg.negate); + brw_inst_set_src0_address_mode(devinfo, inst, reg.address_mode); if (reg.file == BRW_IMMEDIATE_VALUE) { - insn->bits3.ud = reg.dw1.ud; + brw_inst_set_imm_ud(devinfo, inst, reg.dw1.ud); + + /* The Bspec's section titled "Non-present Operands" claims that if src0 + * is an immediate that src1's type must be the same as that of src0. + * + * The SNB+ DataTypeIndex instruction compaction tables contain mappings + * that do not follow this rule. E.g., from the IVB/HSW table: + * + * DataTypeIndex 18-Bit Mapping Mapped Meaning + * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir | + * + * And from the SNB table: + * + * DataTypeIndex 18-Bit Mapping Mapped Meaning + * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir | + * + * Neither of these cause warnings from the simulator when used, + * compacted or otherwise. In fact, all compaction mappings that have an + * immediate in src0 use a:ud for src1. + * + * The GM45 instruction compaction tables do not contain mapped meanings + * so it's not clear whether it has the restriction. We'll assume it was + * lifted on SNB. (FINISHME: decode the GM45 tables and check.) + */ + brw_inst_set_src1_reg_file(devinfo, inst, BRW_ARCHITECTURE_REGISTER_FILE); + if (devinfo->gen < 6) { + brw_inst_set_src1_reg_type(devinfo, inst, + brw_inst_src0_reg_type(devinfo, inst)); + } else { + brw_inst_set_src1_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD); + } - /* Required to set some fields in src1 as well: + /* Compacted instructions only have 12-bits (plus 1 for the other 20) + * for immediate values. Presumably the hardware engineers realized + * that the only useful floating-point value that could be represented + * in this format is 0.0, which can also be represented as a VF-typed + * immediate, so they gave us the previously mentioned mapping on IVB+. + * + * Strangely, we do have a mapping for imm:f in src1, so we don't need + * to do this there. + * + * If we see a 0.0:F, change the type to VF so that it can be compacted. */ - insn->bits1.da1.src1_reg_file = 0; /* arf */ - insn->bits1.da1.src1_reg_type = reg.type; - } - else - { - if (reg.address_mode == BRW_ADDRESS_DIRECT) { - if (insn->header.access_mode == BRW_ALIGN_1) { - insn->bits2.da1.src0_subreg_nr = reg.subnr; - insn->bits2.da1.src0_reg_nr = reg.nr; - } - else { - insn->bits2.da16.src0_subreg_nr = reg.subnr / 16; - insn->bits2.da16.src0_reg_nr = reg.nr; - } + if (brw_inst_imm_ud(devinfo, inst) == 0x0 && + brw_inst_src0_reg_type(devinfo, inst) == BRW_HW_REG_TYPE_F) { + brw_inst_set_src0_reg_type(devinfo, inst, BRW_HW_REG_IMM_TYPE_VF); } - else { - insn->bits2.ia1.src0_subreg_nr = reg.subnr; - if (insn->header.access_mode == BRW_ALIGN_1) { - insn->bits2.ia1.src0_indirect_offset = reg.dw1.bits.indirect_offset; + /* There are no mappings for dst:d | i:d, so if the immediate is suitable + * set the types to :UD so the instruction can be compacted. + */ + if (is_compactable_immediate(brw_inst_imm_ud(devinfo, inst)) && + brw_inst_cond_modifier(devinfo, inst) == BRW_CONDITIONAL_NONE && + brw_inst_src0_reg_type(devinfo, inst) == BRW_HW_REG_TYPE_D && + brw_inst_dst_reg_type(devinfo, inst) == BRW_HW_REG_TYPE_D) { + brw_inst_set_src0_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD); + brw_inst_set_dst_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD); + } + } else { + if (reg.address_mode == BRW_ADDRESS_DIRECT) { + brw_inst_set_src0_da_reg_nr(devinfo, inst, reg.nr); + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { + brw_inst_set_src0_da1_subreg_nr(devinfo, inst, reg.subnr); + } else { + brw_inst_set_src0_da16_subreg_nr(devinfo, inst, reg.subnr / 16); } - else { - insn->bits2.ia16.src0_subreg_nr = reg.dw1.bits.indirect_offset; + } else { + brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.subnr); + + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { + brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.dw1.bits.indirect_offset); + } else { + brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.dw1.bits.indirect_offset); } } - if (insn->header.access_mode == BRW_ALIGN_1) { + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { if (reg.width == BRW_WIDTH_1 && - insn->header.execution_size == BRW_EXECUTE_1) { - insn->bits2.da1.src0_horiz_stride = BRW_HORIZONTAL_STRIDE_0; - insn->bits2.da1.src0_width = BRW_WIDTH_1; - insn->bits2.da1.src0_vert_stride = BRW_VERTICAL_STRIDE_0; - } - else { - insn->bits2.da1.src0_horiz_stride = reg.hstride; - insn->bits2.da1.src0_width = reg.width; - insn->bits2.da1.src0_vert_stride = reg.vstride; + brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1) { + brw_inst_set_src0_hstride(devinfo, inst, BRW_HORIZONTAL_STRIDE_0); + brw_inst_set_src0_width(devinfo, inst, BRW_WIDTH_1); + brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_0); + } else { + brw_inst_set_src0_hstride(devinfo, inst, reg.hstride); + brw_inst_set_src0_width(devinfo, inst, reg.width); + brw_inst_set_src0_vstride(devinfo, inst, reg.vstride); } - } - else { - insn->bits2.da16.src0_swz_x = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X); - insn->bits2.da16.src0_swz_y = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y); - insn->bits2.da16.src0_swz_z = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z); - insn->bits2.da16.src0_swz_w = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W); + } else { + brw_inst_set_src0_da16_swiz_x(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X)); + brw_inst_set_src0_da16_swiz_y(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y)); + brw_inst_set_src0_da16_swiz_z(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z)); + brw_inst_set_src0_da16_swiz_w(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W)); /* This is an oddity of the fact we're using the same * descriptions for registers in align_16 as align_1: */ if (reg.vstride == BRW_VERTICAL_STRIDE_8) - insn->bits2.da16.src0_vert_stride = BRW_VERTICAL_STRIDE_4; + brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4); else - insn->bits2.da16.src0_vert_stride = reg.vstride; + brw_inst_set_src0_vstride(devinfo, inst, reg.vstride); } } } -void brw_set_src1(struct brw_compile *p, - struct brw_instruction *insn, - struct brw_reg reg) +void +brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg) { - assert(reg.file != BRW_MESSAGE_REGISTER_FILE); + const struct brw_device_info *devinfo = p->devinfo; - if (reg.type != BRW_ARCHITECTURE_REGISTER_FILE) + if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE) assert(reg.nr < 128); gen7_convert_mrf_to_grf(p, ®); + assert(reg.file != BRW_MESSAGE_REGISTER_FILE); - validate_reg(insn, reg); + validate_reg(devinfo, inst, reg); - insn->bits1.da1.src1_reg_file = reg.file; - insn->bits1.da1.src1_reg_type = reg.type; - insn->bits3.da1.src1_abs = reg.abs; - insn->bits3.da1.src1_negate = reg.negate; + brw_inst_set_src1_reg_file(devinfo, inst, reg.file); + brw_inst_set_src1_reg_type(devinfo, inst, + brw_reg_type_to_hw_type(devinfo, reg.type, reg.file)); + brw_inst_set_src1_abs(devinfo, inst, reg.abs); + brw_inst_set_src1_negate(devinfo, inst, reg.negate); /* Only src1 can be immediate in two-argument instructions. */ - assert(insn->bits1.da1.src0_reg_file != BRW_IMMEDIATE_VALUE); + assert(brw_inst_src0_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE); if (reg.file == BRW_IMMEDIATE_VALUE) { - insn->bits3.ud = reg.dw1.ud; - } - else { + brw_inst_set_imm_ud(devinfo, inst, reg.dw1.ud); + } else { /* This is a hardware restriction, which may or may not be lifted * in the future: */ assert (reg.address_mode == BRW_ADDRESS_DIRECT); /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */ - if (insn->header.access_mode == BRW_ALIGN_1) { - insn->bits3.da1.src1_subreg_nr = reg.subnr; - insn->bits3.da1.src1_reg_nr = reg.nr; - } - else { - insn->bits3.da16.src1_subreg_nr = reg.subnr / 16; - insn->bits3.da16.src1_reg_nr = reg.nr; + brw_inst_set_src1_da_reg_nr(devinfo, inst, reg.nr); + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { + brw_inst_set_src1_da1_subreg_nr(devinfo, inst, reg.subnr); + } else { + brw_inst_set_src1_da16_subreg_nr(devinfo, inst, reg.subnr / 16); } - if (insn->header.access_mode == BRW_ALIGN_1) { + if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) { if (reg.width == BRW_WIDTH_1 && - insn->header.execution_size == BRW_EXECUTE_1) { - insn->bits3.da1.src1_horiz_stride = BRW_HORIZONTAL_STRIDE_0; - insn->bits3.da1.src1_width = BRW_WIDTH_1; - insn->bits3.da1.src1_vert_stride = BRW_VERTICAL_STRIDE_0; - } - else { - insn->bits3.da1.src1_horiz_stride = reg.hstride; - insn->bits3.da1.src1_width = reg.width; - insn->bits3.da1.src1_vert_stride = reg.vstride; + brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1) { + brw_inst_set_src1_hstride(devinfo, inst, BRW_HORIZONTAL_STRIDE_0); + brw_inst_set_src1_width(devinfo, inst, BRW_WIDTH_1); + brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_0); + } else { + brw_inst_set_src1_hstride(devinfo, inst, reg.hstride); + brw_inst_set_src1_width(devinfo, inst, reg.width); + brw_inst_set_src1_vstride(devinfo, inst, reg.vstride); } - } - else { - insn->bits3.da16.src1_swz_x = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X); - insn->bits3.da16.src1_swz_y = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y); - insn->bits3.da16.src1_swz_z = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z); - insn->bits3.da16.src1_swz_w = BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W); + } else { + brw_inst_set_src1_da16_swiz_x(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X)); + brw_inst_set_src1_da16_swiz_y(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y)); + brw_inst_set_src1_da16_swiz_z(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z)); + brw_inst_set_src1_da16_swiz_w(devinfo, inst, + BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W)); /* This is an oddity of the fact we're using the same * descriptions for registers in align_16 as align_1: */ if (reg.vstride == BRW_VERTICAL_STRIDE_8) - insn->bits3.da16.src1_vert_stride = BRW_VERTICAL_STRIDE_4; + brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4); else - insn->bits3.da16.src1_vert_stride = reg.vstride; + brw_inst_set_src1_vstride(devinfo, inst, reg.vstride); } } } @@ -411,48 +515,47 @@ void brw_set_src1(struct brw_compile *p, * choose not to fill in irrelevant bits; they will be zero. */ static void -brw_set_message_descriptor(struct brw_compile *p, - struct brw_instruction *inst, +brw_set_message_descriptor(struct brw_codegen *p, + brw_inst *inst, enum brw_message_target sfid, unsigned msg_length, unsigned response_length, bool header_present, bool end_of_thread) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; brw_set_src1(p, inst, brw_imm_d(0)); - if (brw->gen >= 5) { - inst->bits3.generic_gen5.header_present = header_present; - inst->bits3.generic_gen5.response_length = response_length; - inst->bits3.generic_gen5.msg_length = msg_length; - inst->bits3.generic_gen5.end_of_thread = end_of_thread; + /* For indirect sends, `inst` will not be the SEND/SENDC instruction + * itself; instead, it will be a MOV/OR into the address register. + * + * In this case, we avoid setting the extended message descriptor bits, + * since they go on the later SEND/SENDC instead and if set here would + * instead clobber the conditionalmod bits. + */ + unsigned opcode = brw_inst_opcode(devinfo, inst); + if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC) { + brw_inst_set_sfid(devinfo, inst, sfid); + } - if (brw->gen >= 6) { - /* On Gen6+ Message target/SFID goes in bits 27:24 of the header */ - inst->header.destreg__conditionalmod = sfid; - } else { - /* Set Extended Message Descriptor (ex_desc) */ - inst->bits2.send_gen5.sfid = sfid; - inst->bits2.send_gen5.end_of_thread = end_of_thread; - } - } else { - inst->bits3.generic.response_length = response_length; - inst->bits3.generic.msg_length = msg_length; - inst->bits3.generic.msg_target = sfid; - inst->bits3.generic.end_of_thread = end_of_thread; + brw_inst_set_mlen(devinfo, inst, msg_length); + brw_inst_set_rlen(devinfo, inst, response_length); + brw_inst_set_eot(devinfo, inst, end_of_thread); + + if (devinfo->gen >= 5) { + brw_inst_set_header_present(devinfo, inst, header_present); } } -static void brw_set_math_message( struct brw_compile *p, - struct brw_instruction *insn, - GLuint function, - GLuint integer_type, +static void brw_set_math_message( struct brw_codegen *p, + brw_inst *inst, + unsigned function, + unsigned integer_type, bool low_precision, - GLuint dataType ) + unsigned dataType ) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; unsigned msg_length; unsigned response_length; @@ -481,108 +584,100 @@ static void brw_set_math_message( struct brw_compile *p, } - brw_set_message_descriptor(p, insn, BRW_SFID_MATH, + brw_set_message_descriptor(p, inst, BRW_SFID_MATH, msg_length, response_length, false, false); - if (brw->gen == 5) { - insn->bits3.math_gen5.function = function; - insn->bits3.math_gen5.int_type = integer_type; - insn->bits3.math_gen5.precision = low_precision; - insn->bits3.math_gen5.saturate = insn->header.saturate; - insn->bits3.math_gen5.data_type = dataType; - insn->bits3.math_gen5.snapshot = 0; - } else { - insn->bits3.math.function = function; - insn->bits3.math.int_type = integer_type; - insn->bits3.math.precision = low_precision; - insn->bits3.math.saturate = insn->header.saturate; - insn->bits3.math.data_type = dataType; - } - insn->header.saturate = 0; + brw_inst_set_math_msg_function(devinfo, inst, function); + brw_inst_set_math_msg_signed_int(devinfo, inst, integer_type); + brw_inst_set_math_msg_precision(devinfo, inst, low_precision); + brw_inst_set_math_msg_saturate(devinfo, inst, brw_inst_saturate(devinfo, inst)); + brw_inst_set_math_msg_data_type(devinfo, inst, dataType); + brw_inst_set_saturate(devinfo, inst, 0); } -static void brw_set_ff_sync_message(struct brw_compile *p, - struct brw_instruction *insn, +static void brw_set_ff_sync_message(struct brw_codegen *p, + brw_inst *insn, bool allocate, - GLuint response_length, + unsigned response_length, bool end_of_thread) { + const struct brw_device_info *devinfo = p->devinfo; + brw_set_message_descriptor(p, insn, BRW_SFID_URB, 1, response_length, true, end_of_thread); - insn->bits3.urb_gen5.opcode = 1; /* FF_SYNC */ - insn->bits3.urb_gen5.offset = 0; /* Not used by FF_SYNC */ - insn->bits3.urb_gen5.swizzle_control = 0; /* Not used by FF_SYNC */ - insn->bits3.urb_gen5.allocate = allocate; - insn->bits3.urb_gen5.used = 0; /* Not used by FF_SYNC */ - insn->bits3.urb_gen5.complete = 0; /* Not used by FF_SYNC */ + brw_inst_set_urb_opcode(devinfo, insn, 1); /* FF_SYNC */ + brw_inst_set_urb_allocate(devinfo, insn, allocate); + /* The following fields are not used by FF_SYNC: */ + brw_inst_set_urb_global_offset(devinfo, insn, 0); + brw_inst_set_urb_swizzle_control(devinfo, insn, 0); + brw_inst_set_urb_used(devinfo, insn, 0); + brw_inst_set_urb_complete(devinfo, insn, 0); } -static void brw_set_urb_message( struct brw_compile *p, - struct brw_instruction *insn, +static void brw_set_urb_message( struct brw_codegen *p, + brw_inst *insn, enum brw_urb_write_flags flags, - GLuint msg_length, - GLuint response_length, - GLuint offset, - GLuint swizzle_control ) + unsigned msg_length, + unsigned response_length, + unsigned offset, + unsigned swizzle_control ) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; + + assert(devinfo->gen < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE); + assert(devinfo->gen < 7 || !(flags & BRW_URB_WRITE_ALLOCATE)); + assert(devinfo->gen >= 7 || !(flags & BRW_URB_WRITE_PER_SLOT_OFFSET)); brw_set_message_descriptor(p, insn, BRW_SFID_URB, msg_length, response_length, true, flags & BRW_URB_WRITE_EOT); - if (brw->gen == 7) { - if (flags & BRW_URB_WRITE_OWORD) { - assert(msg_length == 2); /* header + one OWORD of data */ - insn->bits3.urb_gen7.opcode = BRW_URB_OPCODE_WRITE_OWORD; - } else { - insn->bits3.urb_gen7.opcode = BRW_URB_OPCODE_WRITE_HWORD; - } - insn->bits3.urb_gen7.offset = offset; - assert(swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE); - insn->bits3.urb_gen7.swizzle_control = swizzle_control; - insn->bits3.urb_gen7.per_slot_offset = - flags & BRW_URB_WRITE_PER_SLOT_OFFSET ? 1 : 0; - insn->bits3.urb_gen7.complete = flags & BRW_URB_WRITE_COMPLETE ? 1 : 0; - } else if (brw->gen >= 5) { - insn->bits3.urb_gen5.opcode = 0; /* URB_WRITE */ - insn->bits3.urb_gen5.offset = offset; - insn->bits3.urb_gen5.swizzle_control = swizzle_control; - insn->bits3.urb_gen5.allocate = flags & BRW_URB_WRITE_ALLOCATE ? 1 : 0; - insn->bits3.urb_gen5.used = flags & BRW_URB_WRITE_UNUSED ? 0 : 1; - insn->bits3.urb_gen5.complete = flags & BRW_URB_WRITE_COMPLETE ? 1 : 0; + + if (flags & BRW_URB_WRITE_OWORD) { + assert(msg_length == 2); /* header + one OWORD of data */ + brw_inst_set_urb_opcode(devinfo, insn, BRW_URB_OPCODE_WRITE_OWORD); + } else { + brw_inst_set_urb_opcode(devinfo, insn, BRW_URB_OPCODE_WRITE_HWORD); + } + + brw_inst_set_urb_global_offset(devinfo, insn, offset); + brw_inst_set_urb_swizzle_control(devinfo, insn, swizzle_control); + + if (devinfo->gen < 8) { + brw_inst_set_urb_complete(devinfo, insn, !!(flags & BRW_URB_WRITE_COMPLETE)); + } + + if (devinfo->gen < 7) { + brw_inst_set_urb_allocate(devinfo, insn, !!(flags & BRW_URB_WRITE_ALLOCATE)); + brw_inst_set_urb_used(devinfo, insn, !(flags & BRW_URB_WRITE_UNUSED)); } else { - insn->bits3.urb.opcode = 0; /* ? */ - insn->bits3.urb.offset = offset; - insn->bits3.urb.swizzle_control = swizzle_control; - insn->bits3.urb.allocate = flags & BRW_URB_WRITE_ALLOCATE ? 1 : 0; - insn->bits3.urb.used = flags & BRW_URB_WRITE_UNUSED ? 0 : 1; - insn->bits3.urb.complete = flags & BRW_URB_WRITE_COMPLETE ? 1 : 0; + brw_inst_set_urb_per_slot_offset(devinfo, insn, + !!(flags & BRW_URB_WRITE_PER_SLOT_OFFSET)); } } void -brw_set_dp_write_message(struct brw_compile *p, - struct brw_instruction *insn, - GLuint binding_table_index, - GLuint msg_control, - GLuint msg_type, - GLuint msg_length, +brw_set_dp_write_message(struct brw_codegen *p, + brw_inst *insn, + unsigned binding_table_index, + unsigned msg_control, + unsigned msg_type, + unsigned msg_length, bool header_present, - GLuint last_render_target, - GLuint response_length, - GLuint end_of_thread, - GLuint send_commit_msg) + unsigned last_render_target, + unsigned response_length, + unsigned end_of_thread, + unsigned send_commit_msg) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; unsigned sfid; - if (brw->gen >= 7) { + if (devinfo->gen >= 7) { /* Use the Render Cache for RT writes; otherwise use the Data Cache */ if (msg_type == GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE) sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; else sfid = GEN7_SFID_DATAPORT_DATA_CACHE; - } else if (brw->gen == 6) { + } else if (devinfo->gen == 6) { /* Use the render cache for all write messages. */ sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; } else { @@ -592,49 +687,32 @@ brw_set_dp_write_message(struct brw_compile *p, brw_set_message_descriptor(p, insn, sfid, msg_length, response_length, header_present, end_of_thread); - if (brw->gen >= 7) { - insn->bits3.gen7_dp.binding_table_index = binding_table_index; - insn->bits3.gen7_dp.msg_control = msg_control; - insn->bits3.gen7_dp.last_render_target = last_render_target; - insn->bits3.gen7_dp.msg_type = msg_type; - } else if (brw->gen == 6) { - insn->bits3.gen6_dp.binding_table_index = binding_table_index; - insn->bits3.gen6_dp.msg_control = msg_control; - insn->bits3.gen6_dp.last_render_target = last_render_target; - insn->bits3.gen6_dp.msg_type = msg_type; - insn->bits3.gen6_dp.send_commit_msg = send_commit_msg; - } else if (brw->gen == 5) { - insn->bits3.dp_write_gen5.binding_table_index = binding_table_index; - insn->bits3.dp_write_gen5.msg_control = msg_control; - insn->bits3.dp_write_gen5.last_render_target = last_render_target; - insn->bits3.dp_write_gen5.msg_type = msg_type; - insn->bits3.dp_write_gen5.send_commit_msg = send_commit_msg; - } else { - insn->bits3.dp_write.binding_table_index = binding_table_index; - insn->bits3.dp_write.msg_control = msg_control; - insn->bits3.dp_write.last_render_target = last_render_target; - insn->bits3.dp_write.msg_type = msg_type; - insn->bits3.dp_write.send_commit_msg = send_commit_msg; + brw_inst_set_binding_table_index(devinfo, insn, binding_table_index); + brw_inst_set_dp_write_msg_type(devinfo, insn, msg_type); + brw_inst_set_dp_write_msg_control(devinfo, insn, msg_control); + brw_inst_set_rt_last(devinfo, insn, last_render_target); + if (devinfo->gen < 7) { + brw_inst_set_dp_write_commit(devinfo, insn, send_commit_msg); } } void -brw_set_dp_read_message(struct brw_compile *p, - struct brw_instruction *insn, - GLuint binding_table_index, - GLuint msg_control, - GLuint msg_type, - GLuint target_cache, - GLuint msg_length, +brw_set_dp_read_message(struct brw_codegen *p, + brw_inst *insn, + unsigned binding_table_index, + unsigned msg_control, + unsigned msg_type, + unsigned target_cache, + unsigned msg_length, bool header_present, - GLuint response_length) + unsigned response_length) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; unsigned sfid; - if (brw->gen >= 7) { + if (devinfo->gen >= 7) { sfid = GEN7_SFID_DATAPORT_DATA_CACHE; - } else if (brw->gen == 6) { + } else if (devinfo->gen == 6) { if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE) sfid = GEN6_SFID_DATAPORT_RENDER_CACHE; else @@ -646,125 +724,100 @@ brw_set_dp_read_message(struct brw_compile *p, brw_set_message_descriptor(p, insn, sfid, msg_length, response_length, header_present, false); - if (brw->gen >= 7) { - insn->bits3.gen7_dp.binding_table_index = binding_table_index; - insn->bits3.gen7_dp.msg_control = msg_control; - insn->bits3.gen7_dp.last_render_target = 0; - insn->bits3.gen7_dp.msg_type = msg_type; - } else if (brw->gen == 6) { - insn->bits3.gen6_dp.binding_table_index = binding_table_index; - insn->bits3.gen6_dp.msg_control = msg_control; - insn->bits3.gen6_dp.last_render_target = 0; - insn->bits3.gen6_dp.msg_type = msg_type; - insn->bits3.gen6_dp.send_commit_msg = 0; - } else if (brw->gen == 5) { - insn->bits3.dp_read_gen5.binding_table_index = binding_table_index; - insn->bits3.dp_read_gen5.msg_control = msg_control; - insn->bits3.dp_read_gen5.msg_type = msg_type; - insn->bits3.dp_read_gen5.target_cache = target_cache; - } else if (brw->is_g4x) { - insn->bits3.dp_read_g4x.binding_table_index = binding_table_index; /*0:7*/ - insn->bits3.dp_read_g4x.msg_control = msg_control; /*8:10*/ - insn->bits3.dp_read_g4x.msg_type = msg_type; /*11:13*/ - insn->bits3.dp_read_g4x.target_cache = target_cache; /*14:15*/ - } else { - insn->bits3.dp_read.binding_table_index = binding_table_index; /*0:7*/ - insn->bits3.dp_read.msg_control = msg_control; /*8:11*/ - insn->bits3.dp_read.msg_type = msg_type; /*12:13*/ - insn->bits3.dp_read.target_cache = target_cache; /*14:15*/ - } + brw_inst_set_binding_table_index(devinfo, insn, binding_table_index); + brw_inst_set_dp_read_msg_type(devinfo, insn, msg_type); + brw_inst_set_dp_read_msg_control(devinfo, insn, msg_control); + if (devinfo->gen < 6) + brw_inst_set_dp_read_target_cache(devinfo, insn, target_cache); } void -brw_set_sampler_message(struct brw_compile *p, - struct brw_instruction *insn, - GLuint binding_table_index, - GLuint sampler, - GLuint msg_type, - GLuint response_length, - GLuint msg_length, - GLuint header_present, - GLuint simd_mode, - GLuint return_format) -{ - struct brw_context *brw = p->brw; - - brw_set_message_descriptor(p, insn, BRW_SFID_SAMPLER, msg_length, +brw_set_sampler_message(struct brw_codegen *p, + brw_inst *inst, + unsigned binding_table_index, + unsigned sampler, + unsigned msg_type, + unsigned response_length, + unsigned msg_length, + unsigned header_present, + unsigned simd_mode, + unsigned return_format) +{ + const struct brw_device_info *devinfo = p->devinfo; + + brw_set_message_descriptor(p, inst, BRW_SFID_SAMPLER, msg_length, response_length, header_present, false); - if (brw->gen >= 7) { - insn->bits3.sampler_gen7.binding_table_index = binding_table_index; - insn->bits3.sampler_gen7.sampler = sampler; - insn->bits3.sampler_gen7.msg_type = msg_type; - insn->bits3.sampler_gen7.simd_mode = simd_mode; - } else if (brw->gen >= 5) { - insn->bits3.sampler_gen5.binding_table_index = binding_table_index; - insn->bits3.sampler_gen5.sampler = sampler; - insn->bits3.sampler_gen5.msg_type = msg_type; - insn->bits3.sampler_gen5.simd_mode = simd_mode; - } else if (brw->is_g4x) { - insn->bits3.sampler_g4x.binding_table_index = binding_table_index; - insn->bits3.sampler_g4x.sampler = sampler; - insn->bits3.sampler_g4x.msg_type = msg_type; - } else { - insn->bits3.sampler.binding_table_index = binding_table_index; - insn->bits3.sampler.sampler = sampler; - insn->bits3.sampler.msg_type = msg_type; - insn->bits3.sampler.return_format = return_format; + brw_inst_set_binding_table_index(devinfo, inst, binding_table_index); + brw_inst_set_sampler(devinfo, inst, sampler); + brw_inst_set_sampler_msg_type(devinfo, inst, msg_type); + if (devinfo->gen >= 5) { + brw_inst_set_sampler_simd_mode(devinfo, inst, simd_mode); + } else if (devinfo->gen == 4 && !devinfo->is_g4x) { + brw_inst_set_sampler_return_format(devinfo, inst, return_format); } } +static void +gen7_set_dp_scratch_message(struct brw_codegen *p, + brw_inst *inst, + bool write, + bool dword, + bool invalidate_after_read, + unsigned num_regs, + unsigned addr_offset, + unsigned mlen, + unsigned rlen, + bool header_present) +{ + const struct brw_device_info *devinfo = p->devinfo; + assert(num_regs == 1 || num_regs == 2 || num_regs == 4 || + (devinfo->gen >= 8 && num_regs == 8)); + brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE, + mlen, rlen, header_present, false); + brw_inst_set_dp_category(devinfo, inst, 1); /* Scratch Block Read/Write msgs */ + brw_inst_set_scratch_read_write(devinfo, inst, write); + brw_inst_set_scratch_type(devinfo, inst, dword); + brw_inst_set_scratch_invalidate_after_read(devinfo, inst, invalidate_after_read); + brw_inst_set_scratch_block_size(devinfo, inst, ffs(num_regs) - 1); + brw_inst_set_scratch_addr_offset(devinfo, inst, addr_offset); +} #define next_insn brw_next_insn -struct brw_instruction * -brw_next_insn(struct brw_compile *p, GLuint opcode) +brw_inst * +brw_next_insn(struct brw_codegen *p, unsigned opcode) { - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; if (p->nr_insn + 1 > p->store_size) { - if (0) - printf("incresing the store size to %d\n", p->store_size << 1); p->store_size <<= 1; - p->store = reralloc(p->mem_ctx, p->store, - struct brw_instruction, p->store_size); - if (!p->store) - assert(!"realloc eu store memeory failed"); + p->store = reralloc(p->mem_ctx, p->store, brw_inst, p->store_size); } p->next_insn_offset += 16; insn = &p->store[p->nr_insn++]; memcpy(insn, p->current, sizeof(*insn)); - /* Reset this one-shot flag: - */ - - if (p->current->header.destreg__conditionalmod) { - p->current->header.destreg__conditionalmod = 0; - p->current->header.predicate_control = BRW_PREDICATE_NORMAL; - } - - insn->header.opcode = opcode; + brw_inst_set_opcode(devinfo, insn, opcode); return insn; } -static struct brw_instruction *brw_alu1( struct brw_compile *p, - GLuint opcode, - struct brw_reg dest, - struct brw_reg src ) +static brw_inst * +brw_alu1(struct brw_codegen *p, unsigned opcode, + struct brw_reg dest, struct brw_reg src) { - struct brw_instruction *insn = next_insn(p, opcode); + brw_inst *insn = next_insn(p, opcode); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src); return insn; } -static struct brw_instruction *brw_alu2(struct brw_compile *p, - GLuint opcode, - struct brw_reg dest, - struct brw_reg src0, - struct brw_reg src1 ) +static brw_inst * +brw_alu2(struct brw_codegen *p, unsigned opcode, + struct brw_reg dest, struct brw_reg src0, struct brw_reg src1) { - struct brw_instruction *insn = next_insn(p, opcode); + brw_inst *insn = next_insn(p, opcode); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src0); brw_set_src1(p, insn, src1); @@ -782,19 +835,16 @@ get_3src_subreg_nr(struct brw_reg reg) } } -static struct brw_instruction *brw_alu3(struct brw_compile *p, - GLuint opcode, - struct brw_reg dest, - struct brw_reg src0, - struct brw_reg src1, - struct brw_reg src2) +static brw_inst * +brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest, + struct brw_reg src0, struct brw_reg src1, struct brw_reg src2) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn = next_insn(p, opcode); + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *inst = next_insn(p, opcode); gen7_convert_mrf_to_grf(p, &dest); - assert(insn->header.access_mode == BRW_ALIGN_16); + assert(brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_16); assert(dest.file == BRW_GENERAL_REGISTER_FILE || dest.file == BRW_MESSAGE_REGISTER_FILE); @@ -803,44 +853,48 @@ static struct brw_instruction *brw_alu3(struct brw_compile *p, assert(dest.type == BRW_REGISTER_TYPE_F || dest.type == BRW_REGISTER_TYPE_D || dest.type == BRW_REGISTER_TYPE_UD); - insn->bits1.da3src.dest_reg_file = (dest.file == BRW_MESSAGE_REGISTER_FILE); - insn->bits1.da3src.dest_reg_nr = dest.nr; - insn->bits1.da3src.dest_subreg_nr = dest.subnr / 16; - insn->bits1.da3src.dest_writemask = dest.dw1.bits.writemask; - guess_execution_size(p, insn, dest); + if (devinfo->gen == 6) { + brw_inst_set_3src_dst_reg_file(devinfo, inst, + dest.file == BRW_MESSAGE_REGISTER_FILE); + } + brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr); + brw_inst_set_3src_dst_subreg_nr(devinfo, inst, dest.subnr / 16); + brw_inst_set_3src_dst_writemask(devinfo, inst, dest.dw1.bits.writemask); assert(src0.file == BRW_GENERAL_REGISTER_FILE); assert(src0.address_mode == BRW_ADDRESS_DIRECT); assert(src0.nr < 128); - insn->bits2.da3src.src0_swizzle = src0.dw1.bits.swizzle; - insn->bits2.da3src.src0_subreg_nr = get_3src_subreg_nr(src0); - insn->bits2.da3src.src0_reg_nr = src0.nr; - insn->bits1.da3src.src0_abs = src0.abs; - insn->bits1.da3src.src0_negate = src0.negate; - insn->bits2.da3src.src0_rep_ctrl = src0.vstride == BRW_VERTICAL_STRIDE_0; + brw_inst_set_3src_src0_swizzle(devinfo, inst, src0.dw1.bits.swizzle); + brw_inst_set_3src_src0_subreg_nr(devinfo, inst, get_3src_subreg_nr(src0)); + brw_inst_set_3src_src0_reg_nr(devinfo, inst, src0.nr); + brw_inst_set_3src_src0_abs(devinfo, inst, src0.abs); + brw_inst_set_3src_src0_negate(devinfo, inst, src0.negate); + brw_inst_set_3src_src0_rep_ctrl(devinfo, inst, + src0.vstride == BRW_VERTICAL_STRIDE_0); assert(src1.file == BRW_GENERAL_REGISTER_FILE); assert(src1.address_mode == BRW_ADDRESS_DIRECT); assert(src1.nr < 128); - insn->bits2.da3src.src1_swizzle = src1.dw1.bits.swizzle; - insn->bits2.da3src.src1_subreg_nr_low = get_3src_subreg_nr(src1) & 0x3; - insn->bits3.da3src.src1_subreg_nr_high = get_3src_subreg_nr(src1) >> 2; - insn->bits2.da3src.src1_rep_ctrl = src1.vstride == BRW_VERTICAL_STRIDE_0; - insn->bits3.da3src.src1_reg_nr = src1.nr; - insn->bits1.da3src.src1_abs = src1.abs; - insn->bits1.da3src.src1_negate = src1.negate; + brw_inst_set_3src_src1_swizzle(devinfo, inst, src1.dw1.bits.swizzle); + brw_inst_set_3src_src1_subreg_nr(devinfo, inst, get_3src_subreg_nr(src1)); + brw_inst_set_3src_src1_reg_nr(devinfo, inst, src1.nr); + brw_inst_set_3src_src1_abs(devinfo, inst, src1.abs); + brw_inst_set_3src_src1_negate(devinfo, inst, src1.negate); + brw_inst_set_3src_src1_rep_ctrl(devinfo, inst, + src1.vstride == BRW_VERTICAL_STRIDE_0); assert(src2.file == BRW_GENERAL_REGISTER_FILE); assert(src2.address_mode == BRW_ADDRESS_DIRECT); assert(src2.nr < 128); - insn->bits3.da3src.src2_swizzle = src2.dw1.bits.swizzle; - insn->bits3.da3src.src2_subreg_nr = get_3src_subreg_nr(src2); - insn->bits3.da3src.src2_rep_ctrl = src2.vstride == BRW_VERTICAL_STRIDE_0; - insn->bits3.da3src.src2_reg_nr = src2.nr; - insn->bits1.da3src.src2_abs = src2.abs; - insn->bits1.da3src.src2_negate = src2.negate; - - if (brw->gen >= 7) { + brw_inst_set_3src_src2_swizzle(devinfo, inst, src2.dw1.bits.swizzle); + brw_inst_set_3src_src2_subreg_nr(devinfo, inst, get_3src_subreg_nr(src2)); + brw_inst_set_3src_src2_reg_nr(devinfo, inst, src2.nr); + brw_inst_set_3src_src2_abs(devinfo, inst, src2.abs); + brw_inst_set_3src_src2_negate(devinfo, inst, src2.negate); + brw_inst_set_3src_src2_rep_ctrl(devinfo, inst, + src2.vstride == BRW_VERTICAL_STRIDE_0); + + if (devinfo->gen >= 7) { /* Set both the source and destination types based on dest.type, * ignoring the source register types. The MAD and LRP emitters ensure * that all four types are float. The BFE and BFI2 emitters, however, @@ -849,21 +903,23 @@ static struct brw_instruction *brw_alu3(struct brw_compile *p, */ switch (dest.type) { case BRW_REGISTER_TYPE_F: - insn->bits1.da3src.src_type = BRW_3SRC_TYPE_F; - insn->bits1.da3src.dst_type = BRW_3SRC_TYPE_F; + brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_F); + brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_F); break; case BRW_REGISTER_TYPE_D: - insn->bits1.da3src.src_type = BRW_3SRC_TYPE_D; - insn->bits1.da3src.dst_type = BRW_3SRC_TYPE_D; + brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_D); + brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_D); break; case BRW_REGISTER_TYPE_UD: - insn->bits1.da3src.src_type = BRW_3SRC_TYPE_UD; - insn->bits1.da3src.dst_type = BRW_3SRC_TYPE_UD; + brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_UD); + brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_UD); break; + default: + unreachable("not reached"); } } - return insn; + return inst; } @@ -871,7 +927,7 @@ static struct brw_instruction *brw_alu3(struct brw_compile *p, * Convenience routines. */ #define ALU1(OP) \ -struct brw_instruction *brw_##OP(struct brw_compile *p, \ +brw_inst *brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src0) \ { \ @@ -879,7 +935,7 @@ struct brw_instruction *brw_##OP(struct brw_compile *p, \ } #define ALU2(OP) \ -struct brw_instruction *brw_##OP(struct brw_compile *p, \ +brw_inst *brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src0, \ struct brw_reg src1) \ @@ -888,7 +944,7 @@ struct brw_instruction *brw_##OP(struct brw_compile *p, \ } #define ALU3(OP) \ -struct brw_instruction *brw_##OP(struct brw_compile *p, \ +brw_inst *brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src0, \ struct brw_reg src1, \ @@ -898,7 +954,7 @@ struct brw_instruction *brw_##OP(struct brw_compile *p, \ } #define ALU3F(OP) \ -struct brw_instruction *brw_##OP(struct brw_compile *p, \ +brw_inst *brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src0, \ struct brw_reg src1, \ @@ -919,20 +975,21 @@ struct brw_instruction *brw_##OP(struct brw_compile *p, \ * Sandybridge and later appear to round correctly without an ADD. */ #define ROUND(OP) \ -void brw_##OP(struct brw_compile *p, \ +void brw_##OP(struct brw_codegen *p, \ struct brw_reg dest, \ struct brw_reg src) \ { \ - struct brw_instruction *rnd, *add; \ + const struct brw_device_info *devinfo = p->devinfo; \ + brw_inst *rnd, *add; \ rnd = next_insn(p, BRW_OPCODE_##OP); \ brw_set_dest(p, rnd, dest); \ brw_set_src0(p, rnd, src); \ \ - if (p->brw->gen < 6) { \ + if (devinfo->gen < 6) { \ /* turn on round-increments */ \ - rnd->header.destreg__conditionalmod = BRW_CONDITIONAL_R; \ + brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \ add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \ - add->header.predicate_control = BRW_PREDICATE_NORMAL; \ + brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \ } \ } @@ -946,8 +1003,6 @@ ALU2(XOR) ALU2(SHR) ALU2(SHL) ALU2(ASR) -ALU1(F32TO16) -ALU1(F16TO32) ALU1(FRC) ALU1(RNDD) ALU2(MAC) @@ -957,8 +1012,6 @@ ALU2(DP4) ALU2(DPH) ALU2(DP3) ALU2(DP2) -ALU2(LINE) -ALU2(PLN) ALU3F(MAD) ALU3F(LRP) ALU1(BFREV) @@ -975,10 +1028,9 @@ ROUND(RNDZ) ROUND(RNDE) -struct brw_instruction *brw_ADD(struct brw_compile *p, - struct brw_reg dest, - struct brw_reg src0, - struct brw_reg src1) +brw_inst * +brw_ADD(struct brw_codegen *p, struct brw_reg dest, + struct brw_reg src0, struct brw_reg src1) { /* 6.2.2: add */ if (src0.type == BRW_REGISTER_TYPE_F || @@ -998,10 +1050,9 @@ struct brw_instruction *brw_ADD(struct brw_compile *p, return brw_alu2(p, BRW_OPCODE_ADD, dest, src0, src1); } -struct brw_instruction *brw_AVG(struct brw_compile *p, - struct brw_reg dest, - struct brw_reg src0, - struct brw_reg src1) +brw_inst * +brw_AVG(struct brw_codegen *p, struct brw_reg dest, + struct brw_reg src0, struct brw_reg src1) { assert(dest.type == src0.type); assert(src0.type == src1.type); @@ -1014,16 +1065,15 @@ struct brw_instruction *brw_AVG(struct brw_compile *p, case BRW_REGISTER_TYPE_UD: break; default: - assert(!"Bad type for brw_AVG"); + unreachable("Bad type for brw_AVG"); } return brw_alu2(p, BRW_OPCODE_AVG, dest, src0, src1); } -struct brw_instruction *brw_MUL(struct brw_compile *p, - struct brw_reg dest, - struct brw_reg src0, - struct brw_reg src1) +brw_inst * +brw_MUL(struct brw_codegen *p, struct brw_reg dest, + struct brw_reg src0, struct brw_reg src1) { /* 6.32.38: mul */ if (src0.type == BRW_REGISTER_TYPE_D || @@ -1055,10 +1105,111 @@ struct brw_instruction *brw_MUL(struct brw_compile *p, return brw_alu2(p, BRW_OPCODE_MUL, dest, src0, src1); } +brw_inst * +brw_LINE(struct brw_codegen *p, struct brw_reg dest, + struct brw_reg src0, struct brw_reg src1) +{ + src0.vstride = BRW_VERTICAL_STRIDE_0; + src0.width = BRW_WIDTH_1; + src0.hstride = BRW_HORIZONTAL_STRIDE_0; + return brw_alu2(p, BRW_OPCODE_LINE, dest, src0, src1); +} + +brw_inst * +brw_PLN(struct brw_codegen *p, struct brw_reg dest, + struct brw_reg src0, struct brw_reg src1) +{ + src0.vstride = BRW_VERTICAL_STRIDE_0; + src0.width = BRW_WIDTH_1; + src0.hstride = BRW_HORIZONTAL_STRIDE_0; + src1.vstride = BRW_VERTICAL_STRIDE_8; + src1.width = BRW_WIDTH_8; + src1.hstride = BRW_HORIZONTAL_STRIDE_1; + return brw_alu2(p, BRW_OPCODE_PLN, dest, src0, src1); +} + +brw_inst * +brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) +{ + const struct brw_device_info *devinfo = p->devinfo; + const bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16; + /* The F32TO16 instruction doesn't support 32-bit destination types in + * Align1 mode, and neither does the Gen8 implementation in terms of a + * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as + * an undocumented feature. + */ + const bool needs_zero_fill = (dst.type == BRW_REGISTER_TYPE_UD && + (!align16 || devinfo->gen >= 8)); + brw_inst *inst; + + if (align16) { + assert(dst.type == BRW_REGISTER_TYPE_UD); + } else { + assert(dst.type == BRW_REGISTER_TYPE_UD || + dst.type == BRW_REGISTER_TYPE_W || + dst.type == BRW_REGISTER_TYPE_UW || + dst.type == BRW_REGISTER_TYPE_HF); + } + + brw_push_insn_state(p); + + if (needs_zero_fill) { + brw_set_default_access_mode(p, BRW_ALIGN_1); + dst = spread(retype(dst, BRW_REGISTER_TYPE_W), 2); + } + + if (devinfo->gen >= 8) { + inst = brw_MOV(p, retype(dst, BRW_REGISTER_TYPE_HF), src); + } else { + assert(devinfo->gen == 7); + inst = brw_alu1(p, BRW_OPCODE_F32TO16, dst, src); + } + + if (needs_zero_fill) { + brw_inst_set_no_dd_clear(devinfo, inst, true); + inst = brw_MOV(p, suboffset(dst, 1), brw_imm_ud(0u)); + brw_inst_set_no_dd_check(devinfo, inst, true); + } + + brw_pop_insn_state(p); + return inst; +} + +brw_inst * +brw_F16TO32(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src) +{ + const struct brw_device_info *devinfo = p->devinfo; + bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16; + + if (align16) { + assert(src.type == BRW_REGISTER_TYPE_UD); + } else { + /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32: + * + * Because this instruction does not have a 16-bit floating-point + * type, the source data type must be Word (W). The destination type + * must be F (Float). + */ + if (src.type == BRW_REGISTER_TYPE_UD) + src = spread(retype(src, BRW_REGISTER_TYPE_W), 2); -void brw_NOP(struct brw_compile *p) + assert(src.type == BRW_REGISTER_TYPE_W || + src.type == BRW_REGISTER_TYPE_UW || + src.type == BRW_REGISTER_TYPE_HF); + } + + if (devinfo->gen >= 8) { + return brw_MOV(p, dst, retype(src, BRW_REGISTER_TYPE_HF)); + } else { + assert(devinfo->gen == 7); + return brw_alu1(p, BRW_OPCODE_F16TO32, dst, src); + } +} + + +void brw_NOP(struct brw_codegen *p) { - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_NOP); + brw_inst *insn = next_insn(p, BRW_OPCODE_NOP); brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD)); brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD)); brw_set_src1(p, insn, brw_imm_ud(0x0)); @@ -1072,24 +1223,24 @@ void brw_NOP(struct brw_compile *p) * Comparisons, if/else/endif */ -struct brw_instruction *brw_JMPI(struct brw_compile *p, - struct brw_reg dest, - struct brw_reg src0, - struct brw_reg src1) +brw_inst * +brw_JMPI(struct brw_codegen *p, struct brw_reg index, + unsigned predicate_control) { - struct brw_instruction *insn = brw_alu2(p, BRW_OPCODE_JMPI, dest, src0, src1); + const struct brw_device_info *devinfo = p->devinfo; + struct brw_reg ip = brw_ip_reg(); + brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index); - insn->header.execution_size = 1; - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.mask_control = BRW_MASK_DISABLE; + brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_2); + brw_inst_set_qtr_control(devinfo, inst, BRW_COMPRESSION_NONE); + brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE); + brw_inst_set_pred_control(devinfo, inst, predicate_control); - p->current->header.predicate_control = BRW_PREDICATE_NONE; - - return insn; + return inst; } static void -push_if_stack(struct brw_compile *p, struct brw_instruction *inst) +push_if_stack(struct brw_codegen *p, brw_inst *inst) { p->if_stack[p->if_stack_depth] = inst - p->store; @@ -1101,15 +1252,15 @@ push_if_stack(struct brw_compile *p, struct brw_instruction *inst) } } -static struct brw_instruction * -pop_if_stack(struct brw_compile *p) +static brw_inst * +pop_if_stack(struct brw_codegen *p) { p->if_stack_depth--; return &p->store[p->if_stack[p->if_stack_depth]]; } static void -push_loop_stack(struct brw_compile *p, struct brw_instruction *inst) +push_loop_stack(struct brw_codegen *p, brw_inst *inst) { if (p->loop_stack_array_size < p->loop_stack_depth) { p->loop_stack_array_size *= 2; @@ -1124,8 +1275,8 @@ push_loop_stack(struct brw_compile *p, struct brw_instruction *inst) p->if_depth_in_loop[p->loop_stack_depth] = 0; } -static struct brw_instruction * -get_inner_do_insn(struct brw_compile *p) +static brw_inst * +get_inner_do_insn(struct brw_codegen *p) { return &p->store[p->loop_stack[p->loop_stack_depth - 1]]; } @@ -1138,46 +1289,49 @@ get_inner_do_insn(struct brw_compile *p) * * When the matching 'else' instruction is reached (presumably by * countdown of the instruction count patched in by our ELSE/ENDIF - * functions), the relevent flags are inverted. + * functions), the relevant flags are inverted. * * When the matching 'endif' instruction is reached, the flags are * popped off. If the stack is now empty, normal execution resumes. */ -struct brw_instruction * -brw_IF(struct brw_compile *p, GLuint execute_size) +brw_inst * +brw_IF(struct brw_codegen *p, unsigned execute_size) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; insn = next_insn(p, BRW_OPCODE_IF); /* Override the defaults for this instruction: */ - if (brw->gen < 6) { + if (devinfo->gen < 6) { brw_set_dest(p, insn, brw_ip_reg()); brw_set_src0(p, insn, brw_ip_reg()); brw_set_src1(p, insn, brw_imm_d(0x0)); - } else if (brw->gen == 6) { + } else if (devinfo->gen == 6) { brw_set_dest(p, insn, brw_imm_w(0)); - insn->bits1.branch_gen6.jump_count = 0; + brw_inst_set_gen6_jump_count(devinfo, insn, 0); brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D))); brw_set_src1(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D))); - } else { + } else if (devinfo->gen == 7) { brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D))); brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D))); - brw_set_src1(p, insn, brw_imm_ud(0)); - insn->bits3.break_cont.jip = 0; - insn->bits3.break_cont.uip = 0; + brw_set_src1(p, insn, brw_imm_w(0)); + brw_inst_set_jip(devinfo, insn, 0); + brw_inst_set_uip(devinfo, insn, 0); + } else { + brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D))); + brw_set_src0(p, insn, brw_imm_d(0)); + brw_inst_set_jip(devinfo, insn, 0); + brw_inst_set_uip(devinfo, insn, 0); } - insn->header.execution_size = execute_size; - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.predicate_control = BRW_PREDICATE_NORMAL; - insn->header.mask_control = BRW_MASK_ENABLE; - if (!p->single_program_flow) - insn->header.thread_control = BRW_THREAD_SWITCH; - - p->current->header.predicate_control = BRW_PREDICATE_NONE; + brw_inst_set_exec_size(devinfo, insn, execute_size); + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NORMAL); + brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE); + if (!p->single_program_flow && devinfo->gen < 6) + brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH); push_if_stack(p, insn); p->if_depth_in_loop[p->loop_stack_depth]++; @@ -1187,30 +1341,25 @@ brw_IF(struct brw_compile *p, GLuint execute_size) /* This function is only used for gen6-style IF instructions with an * embedded comparison (conditional modifier). It is not used on gen7. */ -struct brw_instruction * -gen6_IF(struct brw_compile *p, uint32_t conditional, +brw_inst * +gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional, struct brw_reg src0, struct brw_reg src1) { - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; insn = next_insn(p, BRW_OPCODE_IF); brw_set_dest(p, insn, brw_imm_w(0)); - if (p->compressed) { - insn->header.execution_size = BRW_EXECUTE_16; - } else { - insn->header.execution_size = BRW_EXECUTE_8; - } - insn->bits1.branch_gen6.jump_count = 0; + brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 + : BRW_EXECUTE_8); + brw_inst_set_gen6_jump_count(devinfo, insn, 0); brw_set_src0(p, insn, src0); brw_set_src1(p, insn, src1); - assert(insn->header.compression_control == BRW_COMPRESSION_NONE); - assert(insn->header.predicate_control == BRW_PREDICATE_NONE); - insn->header.destreg__conditionalmod = conditional; - - if (!p->single_program_flow) - insn->header.thread_control = BRW_THREAD_SWITCH; + assert(brw_inst_qtr_control(devinfo, insn) == BRW_COMPRESSION_NONE); + assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE); + brw_inst_set_cond_modifier(devinfo, insn, conditional); push_if_stack(p, insn); return insn; @@ -1220,17 +1369,18 @@ gen6_IF(struct brw_compile *p, uint32_t conditional, * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs. */ static void -convert_IF_ELSE_to_ADD(struct brw_compile *p, - struct brw_instruction *if_inst, - struct brw_instruction *else_inst) +convert_IF_ELSE_to_ADD(struct brw_codegen *p, + brw_inst *if_inst, brw_inst *else_inst) { + const struct brw_device_info *devinfo = p->devinfo; + /* The next instruction (where the ENDIF would be, if it existed) */ - struct brw_instruction *next_inst = &p->store[p->nr_insn]; + brw_inst *next_inst = &p->store[p->nr_insn]; assert(p->single_program_flow); - assert(if_inst != NULL && if_inst->header.opcode == BRW_OPCODE_IF); - assert(else_inst == NULL || else_inst->header.opcode == BRW_OPCODE_ELSE); - assert(if_inst->header.execution_size == BRW_EXECUTE_1); + assert(if_inst != NULL && brw_inst_opcode(devinfo, if_inst) == BRW_OPCODE_IF); + assert(else_inst == NULL || brw_inst_opcode(devinfo, else_inst) == BRW_OPCODE_ELSE); + assert(brw_inst_exec_size(devinfo, if_inst) == BRW_EXECUTE_1); /* Convert IF to an ADD instruction that moves the instruction pointer * to the first instruction of the ELSE block. If there is no ELSE @@ -1240,19 +1390,19 @@ convert_IF_ELSE_to_ADD(struct brw_compile *p, * stack operations, and if we're currently executing, we just want to * continue normally. */ - if_inst->header.opcode = BRW_OPCODE_ADD; - if_inst->header.predicate_inverse = 1; + brw_inst_set_opcode(devinfo, if_inst, BRW_OPCODE_ADD); + brw_inst_set_pred_inv(devinfo, if_inst, true); if (else_inst != NULL) { /* Convert ELSE to an ADD instruction that points where the ENDIF * would be. */ - else_inst->header.opcode = BRW_OPCODE_ADD; + brw_inst_set_opcode(devinfo, else_inst, BRW_OPCODE_ADD); - if_inst->bits3.ud = (else_inst - if_inst + 1) * 16; - else_inst->bits3.ud = (next_inst - else_inst) * 16; + brw_inst_set_imm_ud(devinfo, if_inst, (else_inst - if_inst + 1) * 16); + brw_inst_set_imm_ud(devinfo, else_inst, (next_inst - else_inst) * 16); } else { - if_inst->bits3.ud = (next_inst - if_inst) * 16; + brw_inst_set_imm_ud(devinfo, if_inst, (next_inst - if_inst) * 16); } } @@ -1260,12 +1410,10 @@ convert_IF_ELSE_to_ADD(struct brw_compile *p, * Patch IF and ELSE instructions with appropriate jump targets. */ static void -patch_IF_ELSE(struct brw_compile *p, - struct brw_instruction *if_inst, - struct brw_instruction *else_inst, - struct brw_instruction *endif_inst) +patch_IF_ELSE(struct brw_codegen *p, + brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; /* We shouldn't be patching IF and ELSE instructions in single program flow * mode when gen < 6, because in single program flow mode on those @@ -1279,114 +1427,122 @@ patch_IF_ELSE(struct brw_compile *p, * instructions to conditional ADDs. So we do patch IF and ELSE * instructions in single program flow mode on those platforms. */ - if (brw->gen < 6) + if (devinfo->gen < 6) assert(!p->single_program_flow); - assert(if_inst != NULL && if_inst->header.opcode == BRW_OPCODE_IF); + assert(if_inst != NULL && brw_inst_opcode(devinfo, if_inst) == BRW_OPCODE_IF); assert(endif_inst != NULL); - assert(else_inst == NULL || else_inst->header.opcode == BRW_OPCODE_ELSE); + assert(else_inst == NULL || brw_inst_opcode(devinfo, else_inst) == BRW_OPCODE_ELSE); - unsigned br = 1; - /* Jump count is for 64bit data chunk each, so one 128bit instruction - * requires 2 chunks. - */ - if (brw->gen >= 5) - br = 2; + unsigned br = brw_jump_scale(devinfo); - assert(endif_inst->header.opcode == BRW_OPCODE_ENDIF); - endif_inst->header.execution_size = if_inst->header.execution_size; + assert(brw_inst_opcode(devinfo, endif_inst) == BRW_OPCODE_ENDIF); + brw_inst_set_exec_size(devinfo, endif_inst, brw_inst_exec_size(devinfo, if_inst)); if (else_inst == NULL) { /* Patch IF -> ENDIF */ - if (brw->gen < 6) { + if (devinfo->gen < 6) { /* Turn it into an IFF, which means no mask stack operations for * all-false and jumping past the ENDIF. */ - if_inst->header.opcode = BRW_OPCODE_IFF; - if_inst->bits3.if_else.jump_count = br * (endif_inst - if_inst + 1); - if_inst->bits3.if_else.pop_count = 0; - if_inst->bits3.if_else.pad0 = 0; - } else if (brw->gen == 6) { + brw_inst_set_opcode(devinfo, if_inst, BRW_OPCODE_IFF); + brw_inst_set_gen4_jump_count(devinfo, if_inst, + br * (endif_inst - if_inst + 1)); + brw_inst_set_gen4_pop_count(devinfo, if_inst, 0); + } else if (devinfo->gen == 6) { /* As of gen6, there is no IFF and IF must point to the ENDIF. */ - if_inst->bits1.branch_gen6.jump_count = br * (endif_inst - if_inst); + brw_inst_set_gen6_jump_count(devinfo, if_inst, br*(endif_inst - if_inst)); } else { - if_inst->bits3.break_cont.uip = br * (endif_inst - if_inst); - if_inst->bits3.break_cont.jip = br * (endif_inst - if_inst); + brw_inst_set_uip(devinfo, if_inst, br * (endif_inst - if_inst)); + brw_inst_set_jip(devinfo, if_inst, br * (endif_inst - if_inst)); } } else { - else_inst->header.execution_size = if_inst->header.execution_size; + brw_inst_set_exec_size(devinfo, else_inst, brw_inst_exec_size(devinfo, if_inst)); /* Patch IF -> ELSE */ - if (brw->gen < 6) { - if_inst->bits3.if_else.jump_count = br * (else_inst - if_inst); - if_inst->bits3.if_else.pop_count = 0; - if_inst->bits3.if_else.pad0 = 0; - } else if (brw->gen == 6) { - if_inst->bits1.branch_gen6.jump_count = br * (else_inst - if_inst + 1); + if (devinfo->gen < 6) { + brw_inst_set_gen4_jump_count(devinfo, if_inst, + br * (else_inst - if_inst)); + brw_inst_set_gen4_pop_count(devinfo, if_inst, 0); + } else if (devinfo->gen == 6) { + brw_inst_set_gen6_jump_count(devinfo, if_inst, + br * (else_inst - if_inst + 1)); } /* Patch ELSE -> ENDIF */ - if (brw->gen < 6) { + if (devinfo->gen < 6) { /* BRW_OPCODE_ELSE pre-gen6 should point just past the * matching ENDIF. */ - else_inst->bits3.if_else.jump_count = br*(endif_inst - else_inst + 1); - else_inst->bits3.if_else.pop_count = 1; - else_inst->bits3.if_else.pad0 = 0; - } else if (brw->gen == 6) { + brw_inst_set_gen4_jump_count(devinfo, else_inst, + br * (endif_inst - else_inst + 1)); + brw_inst_set_gen4_pop_count(devinfo, else_inst, 1); + } else if (devinfo->gen == 6) { /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */ - else_inst->bits1.branch_gen6.jump_count = br*(endif_inst - else_inst); + brw_inst_set_gen6_jump_count(devinfo, else_inst, + br * (endif_inst - else_inst)); } else { /* The IF instruction's JIP should point just past the ELSE */ - if_inst->bits3.break_cont.jip = br * (else_inst - if_inst + 1); + brw_inst_set_jip(devinfo, if_inst, br * (else_inst - if_inst + 1)); /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */ - if_inst->bits3.break_cont.uip = br * (endif_inst - if_inst); - else_inst->bits3.break_cont.jip = br * (endif_inst - else_inst); + brw_inst_set_uip(devinfo, if_inst, br * (endif_inst - if_inst)); + brw_inst_set_jip(devinfo, else_inst, br * (endif_inst - else_inst)); + if (devinfo->gen >= 8) { + /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both + * should point to ENDIF. + */ + brw_inst_set_uip(devinfo, else_inst, br * (endif_inst - else_inst)); + } } } } void -brw_ELSE(struct brw_compile *p) +brw_ELSE(struct brw_codegen *p) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; insn = next_insn(p, BRW_OPCODE_ELSE); - if (brw->gen < 6) { + if (devinfo->gen < 6) { brw_set_dest(p, insn, brw_ip_reg()); brw_set_src0(p, insn, brw_ip_reg()); brw_set_src1(p, insn, brw_imm_d(0x0)); - } else if (brw->gen == 6) { + } else if (devinfo->gen == 6) { brw_set_dest(p, insn, brw_imm_w(0)); - insn->bits1.branch_gen6.jump_count = 0; + brw_inst_set_gen6_jump_count(devinfo, insn, 0); brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - } else { + } else if (devinfo->gen == 7) { brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src1(p, insn, brw_imm_ud(0)); - insn->bits3.break_cont.jip = 0; - insn->bits3.break_cont.uip = 0; + brw_set_src1(p, insn, brw_imm_w(0)); + brw_inst_set_jip(devinfo, insn, 0); + brw_inst_set_uip(devinfo, insn, 0); + } else { + brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src0(p, insn, brw_imm_d(0)); + brw_inst_set_jip(devinfo, insn, 0); + brw_inst_set_uip(devinfo, insn, 0); } - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.mask_control = BRW_MASK_ENABLE; - if (!p->single_program_flow) - insn->header.thread_control = BRW_THREAD_SWITCH; + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE); + if (!p->single_program_flow && devinfo->gen < 6) + brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH); push_if_stack(p, insn); } void -brw_ENDIF(struct brw_compile *p) +brw_ENDIF(struct brw_codegen *p) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn = NULL; - struct brw_instruction *else_inst = NULL; - struct brw_instruction *if_inst = NULL; - struct brw_instruction *tmp; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn = NULL; + brw_inst *else_inst = NULL; + brw_inst *if_inst = NULL; + brw_inst *tmp; bool emit_endif = true; /* In single program flow mode, we can express IF and ELSE instructions @@ -1401,11 +1557,11 @@ brw_ENDIF(struct brw_compile *p) * instructions to conditional ADDs. So we only do this trick on Gen4 and * Gen5. */ - if (brw->gen < 6 && p->single_program_flow) + if (devinfo->gen < 6 && p->single_program_flow) emit_endif = false; /* - * A single next_insn() may change the base adress of instruction store + * A single next_insn() may change the base address of instruction store * memory(p->store), so call it first before referencing the instruction * store pointer from an index */ @@ -1415,7 +1571,7 @@ brw_ENDIF(struct brw_compile *p) /* Pop the IF and (optional) ELSE instructions from the stack */ p->if_depth_in_loop[p->loop_stack_depth]--; tmp = pop_if_stack(p); - if (tmp->header.opcode == BRW_OPCODE_ELSE) { + if (brw_inst_opcode(devinfo, tmp) == BRW_OPCODE_ELSE) { else_inst = tmp; tmp = pop_if_stack(p); } @@ -1427,44 +1583,50 @@ brw_ENDIF(struct brw_compile *p) return; } - if (brw->gen < 6) { - brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD)); - brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD)); + if (devinfo->gen < 6) { + brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src1(p, insn, brw_imm_d(0x0)); - } else if (brw->gen == 6) { + } else if (devinfo->gen == 6) { brw_set_dest(p, insn, brw_imm_w(0)); brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - } else { + } else if (devinfo->gen == 7) { brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src1(p, insn, brw_imm_ud(0)); + brw_set_src1(p, insn, brw_imm_w(0)); + } else { + brw_set_src0(p, insn, brw_imm_d(0)); } - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.mask_control = BRW_MASK_ENABLE; - insn->header.thread_control = BRW_THREAD_SWITCH; + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE); + if (devinfo->gen < 6) + brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH); /* Also pop item off the stack in the endif instruction: */ - if (brw->gen < 6) { - insn->bits3.if_else.jump_count = 0; - insn->bits3.if_else.pop_count = 1; - insn->bits3.if_else.pad0 = 0; - } else if (brw->gen == 6) { - insn->bits1.branch_gen6.jump_count = 2; + if (devinfo->gen < 6) { + brw_inst_set_gen4_jump_count(devinfo, insn, 0); + brw_inst_set_gen4_pop_count(devinfo, insn, 1); + } else if (devinfo->gen == 6) { + brw_inst_set_gen6_jump_count(devinfo, insn, 2); } else { - insn->bits3.break_cont.jip = 2; + brw_inst_set_jip(devinfo, insn, 2); } patch_IF_ELSE(p, if_inst, else_inst, insn); } -struct brw_instruction *brw_BREAK(struct brw_compile *p) +brw_inst * +brw_BREAK(struct brw_codegen *p) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; insn = next_insn(p, BRW_OPCODE_BREAK); - if (brw->gen >= 6) { + if (devinfo->gen >= 8) { + brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src0(p, insn, brw_imm_d(0x0)); + } else if (devinfo->gen >= 6) { brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_src1(p, insn, brw_imm_d(0x0)); @@ -1472,60 +1634,61 @@ struct brw_instruction *brw_BREAK(struct brw_compile *p) brw_set_dest(p, insn, brw_ip_reg()); brw_set_src0(p, insn, brw_ip_reg()); brw_set_src1(p, insn, brw_imm_d(0x0)); - insn->bits3.if_else.pad0 = 0; - insn->bits3.if_else.pop_count = p->if_depth_in_loop[p->loop_stack_depth]; + brw_inst_set_gen4_pop_count(devinfo, insn, + p->if_depth_in_loop[p->loop_stack_depth]); } - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.execution_size = BRW_EXECUTE_8; + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 + : BRW_EXECUTE_8); return insn; } -struct brw_instruction *gen6_CONT(struct brw_compile *p) +brw_inst * +brw_CONT(struct brw_codegen *p) { - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; insn = next_insn(p, BRW_OPCODE_CONTINUE); - brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); brw_set_dest(p, insn, brw_ip_reg()); - brw_set_src0(p, insn, brw_ip_reg()); - brw_set_src1(p, insn, brw_imm_d(0x0)); - - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.execution_size = BRW_EXECUTE_8; - return insn; -} + if (devinfo->gen >= 8) { + brw_set_src0(p, insn, brw_imm_d(0x0)); + } else { + brw_set_src0(p, insn, brw_ip_reg()); + brw_set_src1(p, insn, brw_imm_d(0x0)); + } -struct brw_instruction *brw_CONT(struct brw_compile *p) -{ - struct brw_instruction *insn; - insn = next_insn(p, BRW_OPCODE_CONTINUE); - brw_set_dest(p, insn, brw_ip_reg()); - brw_set_src0(p, insn, brw_ip_reg()); - brw_set_src1(p, insn, brw_imm_d(0x0)); - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.execution_size = BRW_EXECUTE_8; - /* insn->header.mask_control = BRW_MASK_DISABLE; */ - insn->bits3.if_else.pad0 = 0; - insn->bits3.if_else.pop_count = p->if_depth_in_loop[p->loop_stack_depth]; + if (devinfo->gen < 6) { + brw_inst_set_gen4_pop_count(devinfo, insn, + p->if_depth_in_loop[p->loop_stack_depth]); + } + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 + : BRW_EXECUTE_8); return insn; } -struct brw_instruction *gen6_HALT(struct brw_compile *p) +brw_inst * +gen6_HALT(struct brw_codegen *p) { - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; insn = next_insn(p, BRW_OPCODE_HALT); brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */ + if (devinfo->gen >= 8) { + brw_set_src0(p, insn, brw_imm_d(0x0)); + } else { + brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */ + } if (p->compressed) { - insn->header.execution_size = BRW_EXECUTE_16; + brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_16); } else { - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.execution_size = BRW_EXECUTE_8; + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_8); } return insn; } @@ -1546,15 +1709,16 @@ struct brw_instruction *gen6_HALT(struct brw_compile *p) * For gen6, there's no more mask stack, so no need for DO. WHILE * just points back to the first instruction of the loop. */ -struct brw_instruction *brw_DO(struct brw_compile *p, GLuint execute_size) +brw_inst * +brw_DO(struct brw_codegen *p, unsigned execute_size) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; - if (brw->gen >= 6 || p->single_program_flow) { + if (devinfo->gen >= 6 || p->single_program_flow) { push_loop_stack(p, &p->store[p->nr_insn]); return &p->store[p->nr_insn]; } else { - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_DO); + brw_inst *insn = next_insn(p, BRW_OPCODE_DO); push_loop_stack(p, insn); @@ -1564,11 +1728,9 @@ struct brw_instruction *brw_DO(struct brw_compile *p, GLuint execute_size) brw_set_src0(p, insn, brw_null_reg()); brw_set_src1(p, insn, brw_null_reg()); - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.execution_size = execute_size; - insn->header.predicate_control = BRW_PREDICATE_NONE; - /* insn->header.mask_control = BRW_MASK_ENABLE; */ - /* insn->header.mask_control = BRW_MASK_DISABLE; */ + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_inst_set_exec_size(devinfo, insn, execute_size); + brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NONE); return insn; } @@ -1582,57 +1744,59 @@ struct brw_instruction *brw_DO(struct brw_compile *p, GLuint execute_size) * nesting, since it can always just point to the end of the block/current loop. */ static void -brw_patch_break_cont(struct brw_compile *p, struct brw_instruction *while_inst) +brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst) { - struct brw_context *brw = p->brw; - struct brw_instruction *do_inst = get_inner_do_insn(p); - struct brw_instruction *inst; - int br = (brw->gen == 5) ? 2 : 1; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *do_inst = get_inner_do_insn(p); + brw_inst *inst; + unsigned br = brw_jump_scale(devinfo); + + assert(devinfo->gen < 6); for (inst = while_inst - 1; inst != do_inst; inst--) { /* If the jump count is != 0, that means that this instruction has already * been patched because it's part of a loop inside of the one we're * patching. */ - if (inst->header.opcode == BRW_OPCODE_BREAK && - inst->bits3.if_else.jump_count == 0) { - inst->bits3.if_else.jump_count = br * ((while_inst - inst) + 1); - } else if (inst->header.opcode == BRW_OPCODE_CONTINUE && - inst->bits3.if_else.jump_count == 0) { - inst->bits3.if_else.jump_count = br * (while_inst - inst); + if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_BREAK && + brw_inst_gen4_jump_count(devinfo, inst) == 0) { + brw_inst_set_gen4_jump_count(devinfo, inst, br*((while_inst - inst) + 1)); + } else if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_CONTINUE && + brw_inst_gen4_jump_count(devinfo, inst) == 0) { + brw_inst_set_gen4_jump_count(devinfo, inst, br * (while_inst - inst)); } } } -struct brw_instruction *brw_WHILE(struct brw_compile *p) +brw_inst * +brw_WHILE(struct brw_codegen *p) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn, *do_insn; - GLuint br = 1; - - if (brw->gen >= 5) - br = 2; - - if (brw->gen >= 7) { - insn = next_insn(p, BRW_OPCODE_WHILE); - do_insn = get_inner_do_insn(p); - - brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src1(p, insn, brw_imm_ud(0)); - insn->bits3.break_cont.jip = br * (do_insn - insn); + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn, *do_insn; + unsigned br = brw_jump_scale(devinfo); - insn->header.execution_size = BRW_EXECUTE_8; - } else if (brw->gen == 6) { + if (devinfo->gen >= 6) { insn = next_insn(p, BRW_OPCODE_WHILE); do_insn = get_inner_do_insn(p); - brw_set_dest(p, insn, brw_imm_w(0)); - insn->bits1.branch_gen6.jump_count = br * (do_insn - insn); - brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); - brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + if (devinfo->gen >= 8) { + brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src0(p, insn, brw_imm_d(0)); + brw_inst_set_jip(devinfo, insn, br * (do_insn - insn)); + } else if (devinfo->gen == 7) { + brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src1(p, insn, brw_imm_w(0)); + brw_inst_set_jip(devinfo, insn, br * (do_insn - insn)); + } else { + brw_set_dest(p, insn, brw_imm_w(0)); + brw_inst_set_gen6_jump_count(devinfo, insn, br * (do_insn - insn)); + brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D)); + } - insn->header.execution_size = BRW_EXECUTE_8; + brw_inst_set_exec_size(devinfo, insn, p->compressed ? BRW_EXECUTE_16 + : BRW_EXECUTE_8); } else { if (p->single_program_flow) { insn = next_insn(p, BRW_OPCODE_ADD); @@ -1641,84 +1805,67 @@ struct brw_instruction *brw_WHILE(struct brw_compile *p) brw_set_dest(p, insn, brw_ip_reg()); brw_set_src0(p, insn, brw_ip_reg()); brw_set_src1(p, insn, brw_imm_d((do_insn - insn) * 16)); - insn->header.execution_size = BRW_EXECUTE_1; + brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1); } else { insn = next_insn(p, BRW_OPCODE_WHILE); do_insn = get_inner_do_insn(p); - assert(do_insn->header.opcode == BRW_OPCODE_DO); + assert(brw_inst_opcode(devinfo, do_insn) == BRW_OPCODE_DO); brw_set_dest(p, insn, brw_ip_reg()); brw_set_src0(p, insn, brw_ip_reg()); brw_set_src1(p, insn, brw_imm_d(0)); - insn->header.execution_size = do_insn->header.execution_size; - insn->bits3.if_else.jump_count = br * (do_insn - insn + 1); - insn->bits3.if_else.pop_count = 0; - insn->bits3.if_else.pad0 = 0; + brw_inst_set_exec_size(devinfo, insn, brw_inst_exec_size(devinfo, do_insn)); + brw_inst_set_gen4_jump_count(devinfo, insn, br * (do_insn - insn + 1)); + brw_inst_set_gen4_pop_count(devinfo, insn, 0); brw_patch_break_cont(p, insn); } } - insn->header.compression_control = BRW_COMPRESSION_NONE; - p->current->header.predicate_control = BRW_PREDICATE_NONE; + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); p->loop_stack_depth--; return insn; } - /* FORWARD JUMPS: */ -void brw_land_fwd_jump(struct brw_compile *p, int jmp_insn_idx) +void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx) { - struct brw_context *brw = p->brw; - struct brw_instruction *jmp_insn = &p->store[jmp_insn_idx]; - GLuint jmpi = 1; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *jmp_insn = &p->store[jmp_insn_idx]; + unsigned jmpi = 1; - if (brw->gen >= 5) + if (devinfo->gen >= 5) jmpi = 2; - assert(jmp_insn->header.opcode == BRW_OPCODE_JMPI); - assert(jmp_insn->bits1.da1.src1_reg_file == BRW_IMMEDIATE_VALUE); + assert(brw_inst_opcode(devinfo, jmp_insn) == BRW_OPCODE_JMPI); + assert(brw_inst_src1_reg_file(devinfo, jmp_insn) == BRW_IMMEDIATE_VALUE); - jmp_insn->bits3.ud = jmpi * (p->nr_insn - jmp_insn_idx - 1); + brw_inst_set_gen4_jump_count(devinfo, jmp_insn, + jmpi * (p->nr_insn - jmp_insn_idx - 1)); } - - /* To integrate with the above, it makes sense that the comparison * instruction should populate the flag register. It might be simpler * just to use the flag reg for most WM tasks? */ -void brw_CMP(struct brw_compile *p, +void brw_CMP(struct brw_codegen *p, struct brw_reg dest, - GLuint conditional, + unsigned conditional, struct brw_reg src0, struct brw_reg src1) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_CMP); + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn = next_insn(p, BRW_OPCODE_CMP); - insn->header.destreg__conditionalmod = conditional; + brw_inst_set_cond_modifier(devinfo, insn, conditional); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src0); brw_set_src1(p, insn, src1); -/* guess_execution_size(insn, src0); */ - - - /* Make it so that future instructions will use the computed flag - * value until brw_set_predicate_control_flag_value() is called - * again. - */ - if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE && - dest.nr == 0) { - p->current->header.predicate_control = BRW_PREDICATE_NORMAL; - p->flag_value = 0xff; - } - /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds * page says: * "Any CMP instruction with a null destination must use a {switch}." @@ -1726,117 +1873,72 @@ void brw_CMP(struct brw_compile *p, * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't * mentioned on their work-arounds pages. */ - if (brw->gen == 7) { + if (devinfo->gen == 7) { if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE && dest.nr == BRW_ARF_NULL) { - insn->header.thread_control = BRW_THREAD_SWITCH; + brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH); } } } -/* Issue 'wait' instruction for n1, host could program MMIO - to wake up thread. */ -void brw_WAIT (struct brw_compile *p) -{ - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_WAIT); - struct brw_reg src = brw_notification_1_reg(); - - brw_set_dest(p, insn, src); - brw_set_src0(p, insn, src); - brw_set_src1(p, insn, brw_null_reg()); - insn->header.execution_size = 0; /* must */ - insn->header.predicate_control = 0; - insn->header.compression_control = 0; -} - - /*********************************************************************** * Helpers for the various SEND message types: */ /** Extended math function, float[8]. */ -void brw_math( struct brw_compile *p, +void gen4_math(struct brw_codegen *p, struct brw_reg dest, - GLuint function, - GLuint msg_reg_nr, + unsigned function, + unsigned msg_reg_nr, struct brw_reg src, - GLuint data_type, - GLuint precision ) + unsigned precision ) { - struct brw_context *brw = p->brw; - - if (brw->gen >= 6) { - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_MATH); - - assert(dest.file == BRW_GENERAL_REGISTER_FILE || - (brw->gen >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE)); - assert(src.file == BRW_GENERAL_REGISTER_FILE); - - assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1); - if (brw->gen == 6) - assert(src.hstride == BRW_HORIZONTAL_STRIDE_1); - - /* Source modifiers are ignored for extended math instructions on Gen6. */ - if (brw->gen == 6) { - assert(!src.negate); - assert(!src.abs); - } - - if (function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT || - function == BRW_MATH_FUNCTION_INT_DIV_REMAINDER || - function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER) { - assert(src.type != BRW_REGISTER_TYPE_F); - } else { - assert(src.type == BRW_REGISTER_TYPE_F); - } - - /* Math is the same ISA format as other opcodes, except that CondModifier - * becomes FC[3:0] and ThreadCtrl becomes FC[5:4]. - */ - insn->header.destreg__conditionalmod = function; - - brw_set_dest(p, insn, dest); - brw_set_src0(p, insn, src); - brw_set_src1(p, insn, brw_null_reg()); + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); + unsigned data_type; + if (has_scalar_region(src)) { + data_type = BRW_MATH_DATA_SCALAR; } else { - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_SEND); + data_type = BRW_MATH_DATA_VECTOR; + } - /* Example code doesn't set predicate_control for send - * instructions. - */ - insn->header.predicate_control = 0; - insn->header.destreg__conditionalmod = msg_reg_nr; + assert(devinfo->gen < 6); - brw_set_dest(p, insn, dest); - brw_set_src0(p, insn, src); - brw_set_math_message(p, - insn, - function, - src.type == BRW_REGISTER_TYPE_D, - precision, - data_type); - } + /* Example code doesn't set predicate_control for send + * instructions. + */ + brw_inst_set_pred_control(devinfo, insn, 0); + brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr); + + brw_set_dest(p, insn, dest); + brw_set_src0(p, insn, src); + brw_set_math_message(p, + insn, + function, + src.type == BRW_REGISTER_TYPE_D, + precision, + data_type); } -/** Extended math function, float[8]. - */ -void brw_math2(struct brw_compile *p, +void gen6_math(struct brw_codegen *p, struct brw_reg dest, - GLuint function, + unsigned function, struct brw_reg src0, struct brw_reg src1) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_MATH); + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn = next_insn(p, BRW_OPCODE_MATH); + + assert(devinfo->gen >= 6); assert(dest.file == BRW_GENERAL_REGISTER_FILE || - (brw->gen >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE)); - assert(src0.file == BRW_GENERAL_REGISTER_FILE); - assert(src1.file == BRW_GENERAL_REGISTER_FILE); + (devinfo->gen >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE)); + assert(src0.file == BRW_GENERAL_REGISTER_FILE || + (devinfo->gen >= 8 && src0.file == BRW_IMMEDIATE_VALUE)); assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1); - if (brw->gen == 6) { + if (devinfo->gen == 6) { assert(src0.hstride == BRW_HORIZONTAL_STRIDE_1); assert(src1.hstride == BRW_HORIZONTAL_STRIDE_1); } @@ -1846,23 +1948,29 @@ void brw_math2(struct brw_compile *p, function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER) { assert(src0.type != BRW_REGISTER_TYPE_F); assert(src1.type != BRW_REGISTER_TYPE_F); + assert(src1.file == BRW_GENERAL_REGISTER_FILE || + (devinfo->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE)); } else { assert(src0.type == BRW_REGISTER_TYPE_F); assert(src1.type == BRW_REGISTER_TYPE_F); + if (function == BRW_MATH_FUNCTION_POW) { + assert(src1.file == BRW_GENERAL_REGISTER_FILE || + (devinfo->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE)); + } else { + assert(src1.file == BRW_ARCHITECTURE_REGISTER_FILE && + src1.nr == BRW_ARF_NULL); + } } /* Source modifiers are ignored for extended math instructions on Gen6. */ - if (brw->gen == 6) { + if (devinfo->gen == 6) { assert(!src0.negate); assert(!src0.abs); assert(!src1.negate); assert(!src1.abs); } - /* Math is the same ISA format as other opcodes, except that CondModifier - * becomes FC[3:0] and ThreadCtrl becomes FC[5:4]. - */ - insn->header.destreg__conditionalmod = function; + brw_inst_set_math_function(devinfo, insn, function); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src0); @@ -1877,16 +1985,16 @@ void brw_math2(struct brw_compile *p, * The offset must be aligned to oword size (16 bytes). Used for * register spilling. */ -void brw_oword_block_write_scratch(struct brw_compile *p, +void brw_oword_block_write_scratch(struct brw_codegen *p, struct brw_reg mrf, int num_regs, - GLuint offset) + unsigned offset) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; uint32_t msg_control, msg_type; int mlen; - if (brw->gen >= 6) + if (devinfo->gen >= 6) offset /= 16; mrf = retype(mrf, BRW_REGISTER_TYPE_UD); @@ -1906,8 +2014,9 @@ void brw_oword_block_write_scratch(struct brw_compile *p, */ { brw_push_insn_state(p); - brw_set_mask_control(p, BRW_MASK_DISABLE); - brw_set_compression_control(p, BRW_COMPRESSION_NONE); + brw_set_default_exec_size(p, BRW_EXECUTE_8); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)); @@ -1923,17 +2032,18 @@ void brw_oword_block_write_scratch(struct brw_compile *p, { struct brw_reg dest; - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_SEND); + brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); int send_commit_msg; struct brw_reg src_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW); - if (insn->header.compression_control != BRW_COMPRESSION_NONE) { - insn->header.compression_control = BRW_COMPRESSION_NONE; + if (brw_inst_qtr_control(devinfo, insn) != BRW_COMPRESSION_NONE) { + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); src_header = vec16(src_header); } - assert(insn->header.predicate_control == BRW_PREDICATE_NONE); - insn->header.destreg__conditionalmod = mrf.nr; + assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE); + if (devinfo->gen < 6) + brw_inst_set_base_mrf(devinfo, insn, mrf.nr); /* Until gen6, writes followed by reads from the same location * are not guaranteed to be ordered unless write_commit is set. @@ -1945,7 +2055,7 @@ void brw_oword_block_write_scratch(struct brw_compile *p, * protection. Our use of DP writes is all about register * spilling within a thread. */ - if (brw->gen >= 6) { + if (devinfo->gen >= 6) { dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW); send_commit_msg = 0; } else { @@ -1954,13 +2064,13 @@ void brw_oword_block_write_scratch(struct brw_compile *p, } brw_set_dest(p, insn, dest); - if (brw->gen >= 6) { + if (devinfo->gen >= 6) { brw_set_src0(p, insn, mrf); } else { brw_set_src0(p, insn, brw_null_reg()); } - if (brw->gen >= 6) + if (devinfo->gen >= 6) msg_type = GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE; else msg_type = BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE; @@ -1988,20 +2098,31 @@ void brw_oword_block_write_scratch(struct brw_compile *p, * spilling. */ void -brw_oword_block_read_scratch(struct brw_compile *p, +brw_oword_block_read_scratch(struct brw_codegen *p, struct brw_reg dest, struct brw_reg mrf, int num_regs, - GLuint offset) + unsigned offset) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; uint32_t msg_control; int rlen; - if (brw->gen >= 6) + if (devinfo->gen >= 6) offset /= 16; - mrf = retype(mrf, BRW_REGISTER_TYPE_UD); + if (p->devinfo->gen >= 7) { + /* On gen 7 and above, we no longer have message registers and we can + * send from any register we want. By using the destination register + * for the message, we guarantee that the implied message write won't + * accidentally overwrite anything. This has been a problem because + * the MRF registers and source for the final FB write are both fixed + * and may overlap. + */ + mrf = retype(dest, BRW_REGISTER_TYPE_UD); + } else { + mrf = retype(mrf, BRW_REGISTER_TYPE_UD); + } dest = retype(dest, BRW_REGISTER_TYPE_UW); if (num_regs == 1) { @@ -2014,33 +2135,30 @@ brw_oword_block_read_scratch(struct brw_compile *p, { brw_push_insn_state(p); - brw_set_compression_control(p, BRW_COMPRESSION_NONE); - brw_set_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_exec_size(p, BRW_EXECUTE_8); + brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)); /* set message header global offset field (reg 0, element 2) */ - brw_MOV(p, - retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE, - mrf.nr, - 2), BRW_REGISTER_TYPE_UD), - brw_imm_ud(offset)); + brw_MOV(p, get_element_ud(mrf, 2), brw_imm_ud(offset)); brw_pop_insn_state(p); } { - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_SEND); + brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); - assert(insn->header.predicate_control == 0); - insn->header.compression_control = BRW_COMPRESSION_NONE; - insn->header.destreg__conditionalmod = mrf.nr; + assert(brw_inst_pred_control(devinfo, insn) == 0); + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); brw_set_dest(p, insn, dest); /* UW? */ - if (brw->gen >= 6) { + if (devinfo->gen >= 6) { brw_set_src0(p, insn, mrf); } else { brw_set_src0(p, insn, brw_null_reg()); + brw_inst_set_base_mrf(devinfo, insn, mrf.nr); } brw_set_dp_read_message(p, @@ -2056,45 +2174,39 @@ brw_oword_block_read_scratch(struct brw_compile *p, } void -gen7_block_read_scratch(struct brw_compile *p, +gen7_block_read_scratch(struct brw_codegen *p, struct brw_reg dest, int num_regs, - GLuint offset) + unsigned offset) { - dest = retype(dest, BRW_REGISTER_TYPE_UW); - - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_SEND); + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); + assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE); - assert(insn->header.predicate_control == BRW_PREDICATE_NONE); - insn->header.compression_control = BRW_COMPRESSION_NONE; - - brw_set_dest(p, insn, dest); + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); + brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW)); /* The HW requires that the header is present; this is to get the g0.5 * scratch offset. */ - bool header_present = true; brw_set_src0(p, insn, brw_vec8_grf(0, 0)); - brw_set_message_descriptor(p, insn, - GEN7_SFID_DATAPORT_DATA_CACHE, - 1, /* mlen: just g0 */ - num_regs, - header_present, - false); - - insn->bits3.ud |= GEN7_DATAPORT_SCRATCH_READ; - - assert(num_regs == 1 || num_regs == 2 || num_regs == 4); - insn->bits3.ud |= (num_regs - 1) << GEN7_DATAPORT_SCRATCH_NUM_REGS_SHIFT; - /* According to the docs, offset is "A 12-bit HWord offset into the memory * Immediate Memory buffer as specified by binding table 0xFF." An HWORD * is 32 bytes, which happens to be the size of a register. */ offset /= REG_SIZE; assert(offset < (1 << 12)); - insn->bits3.ud |= offset; + + gen7_set_dp_scratch_message(p, insn, + false, /* scratch read */ + false, /* OWords */ + false, /* invalidate after read */ + num_regs, + offset, + 1, /* mlen: just g0 */ + num_regs, /* rlen */ + true); /* header present */ } /** @@ -2102,24 +2214,25 @@ gen7_block_read_scratch(struct brw_compile *p, * Location (in buffer) should be a multiple of 16. * Used for fetching shader constants. */ -void brw_oword_block_read(struct brw_compile *p, +void brw_oword_block_read(struct brw_codegen *p, struct brw_reg dest, struct brw_reg mrf, uint32_t offset, uint32_t bind_table_index) { - struct brw_context *brw = p->brw; + const struct brw_device_info *devinfo = p->devinfo; /* On newer hardware, offset is in units of owords. */ - if (brw->gen >= 6) + if (devinfo->gen >= 6) offset /= 16; mrf = retype(mrf, BRW_REGISTER_TYPE_UD); brw_push_insn_state(p); - brw_set_predicate_control(p, BRW_PREDICATE_NONE); - brw_set_compression_control(p, BRW_COMPRESSION_NONE); - brw_set_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_exec_size(p, BRW_EXECUTE_8); + brw_set_default_predicate_control(p, BRW_PREDICATE_NONE); + brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)); @@ -2130,17 +2243,17 @@ void brw_oword_block_read(struct brw_compile *p, 2), BRW_REGISTER_TYPE_UD), brw_imm_ud(offset)); - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_SEND); - insn->header.destreg__conditionalmod = mrf.nr; + brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); /* cast dest to a uword[8] vector */ dest = retype(vec8(dest), BRW_REGISTER_TYPE_UW); brw_set_dest(p, insn, dest); - if (brw->gen >= 6) { + if (devinfo->gen >= 6) { brw_set_src0(p, insn, mrf); } else { brw_set_src0(p, insn, brw_null_reg()); + brw_inst_set_base_mrf(devinfo, insn, mrf.nr); } brw_set_dp_read_message(p, @@ -2157,43 +2270,44 @@ void brw_oword_block_read(struct brw_compile *p, } -void brw_fb_WRITE(struct brw_compile *p, +void brw_fb_WRITE(struct brw_codegen *p, int dispatch_width, - GLuint msg_reg_nr, - struct brw_reg src0, - GLuint msg_control, - GLuint binding_table_index, - GLuint msg_length, - GLuint response_length, + struct brw_reg payload, + struct brw_reg implied_header, + unsigned msg_control, + unsigned binding_table_index, + unsigned msg_length, + unsigned response_length, bool eot, + bool last_render_target, bool header_present) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn; - GLuint msg_type; - struct brw_reg dest; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; + unsigned msg_type; + struct brw_reg dest, src0; if (dispatch_width == 16) dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW); else dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW); - if (brw->gen >= 6) { + if (devinfo->gen >= 6) { insn = next_insn(p, BRW_OPCODE_SENDC); } else { insn = next_insn(p, BRW_OPCODE_SEND); } - /* The execution mask is ignored for render target writes. */ - insn->header.predicate_control = 0; - insn->header.compression_control = BRW_COMPRESSION_NONE; + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); - if (brw->gen >= 6) { + if (devinfo->gen >= 6) { /* headerless version, just submit color payload */ - src0 = brw_message_reg(msg_reg_nr); + src0 = payload; msg_type = GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE; } else { - insn->header.destreg__conditionalmod = msg_reg_nr; + assert(payload.file == BRW_MESSAGE_REGISTER_FILE); + brw_inst_set_base_mrf(devinfo, insn, payload.nr); + src0 = implied_header; msg_type = BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE; } @@ -2207,7 +2321,7 @@ void brw_fb_WRITE(struct brw_compile *p, msg_type, msg_length, header_present, - eot, /* last render target write */ + last_render_target, response_length, eot, 0 /* send_commit_msg */); @@ -2219,27 +2333,27 @@ void brw_fb_WRITE(struct brw_compile *p, * Note: the msg_type plus msg_length values determine exactly what kind * of sampling operation is performed. See volume 4, page 161 of docs. */ -void brw_SAMPLE(struct brw_compile *p, +void brw_SAMPLE(struct brw_codegen *p, struct brw_reg dest, - GLuint msg_reg_nr, + unsigned msg_reg_nr, struct brw_reg src0, - GLuint binding_table_index, - GLuint sampler, - GLuint msg_type, - GLuint response_length, - GLuint msg_length, - GLuint header_present, - GLuint simd_mode, - GLuint return_format) -{ - struct brw_context *brw = p->brw; - struct brw_instruction *insn; + unsigned binding_table_index, + unsigned sampler, + unsigned msg_type, + unsigned response_length, + unsigned msg_length, + unsigned header_present, + unsigned simd_mode, + unsigned return_format) +{ + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; if (msg_reg_nr != -1) gen6_resolve_implied_move(p, &src0, msg_reg_nr); insn = next_insn(p, BRW_OPCODE_SEND); - insn->header.predicate_control = 0; /* XXX */ + brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NONE); /* XXX */ /* From the 965 PRM (volume 4, part 1, section 14.2.41): * @@ -2253,11 +2367,11 @@ void brw_SAMPLE(struct brw_compile *p, * are allowed in SIMD16 mode and they could not work without SecHalf. For * these reasons, we allow BRW_COMPRESSION_2NDHALF here. */ - if (insn->header.compression_control != BRW_COMPRESSION_2NDHALF) - insn->header.compression_control = BRW_COMPRESSION_NONE; + if (brw_inst_qtr_control(devinfo, insn) != BRW_COMPRESSION_2NDHALF) + brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE); - if (brw->gen < 6) - insn->header.destreg__conditionalmod = msg_reg_nr; + if (devinfo->gen < 6) + brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr); brw_set_dest(p, insn, dest); brw_set_src0(p, insn, src0); @@ -2272,30 +2386,76 @@ void brw_SAMPLE(struct brw_compile *p, return_format); } +/* Adjust the message header's sampler state pointer to + * select the correct group of 16 samplers. + */ +void brw_adjust_sampler_state_pointer(struct brw_codegen *p, + struct brw_reg header, + struct brw_reg sampler_index) +{ + /* The "Sampler Index" field can only store values between 0 and 15. + * However, we can add an offset to the "Sampler State Pointer" + * field, effectively selecting a different set of 16 samplers. + * + * The "Sampler State Pointer" needs to be aligned to a 32-byte + * offset, and each sampler state is only 16-bytes, so we can't + * exclusively use the offset - we have to use both. + */ + + const struct brw_device_info *devinfo = p->devinfo; + + if (sampler_index.file == BRW_IMMEDIATE_VALUE) { + const int sampler_state_size = 16; /* 16 bytes */ + uint32_t sampler = sampler_index.dw1.ud; + + if (sampler >= 16) { + assert(devinfo->is_haswell || devinfo->gen >= 8); + brw_ADD(p, + get_element_ud(header, 3), + get_element_ud(brw_vec8_grf(0, 0), 3), + brw_imm_ud(16 * (sampler / 16) * sampler_state_size)); + } + } else { + /* Non-const sampler array indexing case */ + if (devinfo->gen < 8 && !devinfo->is_haswell) { + return; + } + + struct brw_reg temp = get_element_ud(header, 3); + + brw_AND(p, temp, get_element_ud(sampler_index, 0), brw_imm_ud(0x0f0)); + brw_SHL(p, temp, temp, brw_imm_ud(4)); + brw_ADD(p, + get_element_ud(header, 3), + get_element_ud(brw_vec8_grf(0, 0), 3), + temp); + } +} + /* All these variables are pretty confusing - we might be better off * using bitmasks and macros for this, in the old style. Or perhaps * just having the caller instantiate the fields in dword3 itself. */ -void brw_urb_WRITE(struct brw_compile *p, +void brw_urb_WRITE(struct brw_codegen *p, struct brw_reg dest, - GLuint msg_reg_nr, + unsigned msg_reg_nr, struct brw_reg src0, enum brw_urb_write_flags flags, - GLuint msg_length, - GLuint response_length, - GLuint offset, - GLuint swizzle) + unsigned msg_length, + unsigned response_length, + unsigned offset, + unsigned swizzle) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); - if (brw->gen == 7 && !(flags & BRW_URB_WRITE_USE_CHANNEL_MASKS)) { + if (devinfo->gen >= 7 && !(flags & BRW_URB_WRITE_USE_CHANNEL_MASKS)) { /* Enable Channel Masks in the URB_WRITE_HWORD message header */ brw_push_insn_state(p); - brw_set_access_mode(p, BRW_ALIGN_1); - brw_set_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_access_mode(p, BRW_ALIGN_1); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); brw_OR(p, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE, msg_reg_nr, 5), BRW_REGISTER_TYPE_UD), retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD), @@ -2311,8 +2471,8 @@ void brw_urb_WRITE(struct brw_compile *p, brw_set_src0(p, insn, src0); brw_set_src1(p, insn, brw_imm_d(0)); - if (brw->gen < 6) - insn->header.destreg__conditionalmod = msg_reg_nr; + if (devinfo->gen < 6) + brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr); brw_set_urb_message(p, insn, @@ -2323,32 +2483,109 @@ void brw_urb_WRITE(struct brw_compile *p, swizzle); } -static int -next_ip(struct brw_compile *p, int ip) +struct brw_inst * +brw_send_indirect_message(struct brw_codegen *p, + unsigned sfid, + struct brw_reg dst, + struct brw_reg payload, + struct brw_reg desc) { - struct brw_instruction *insn = (void *)p->store + ip; + const struct brw_device_info *devinfo = p->devinfo; + struct brw_inst *send, *setup; - if (insn->header.cmpt_control) - return ip + 8; - else - return ip + 16; + assert(desc.type == BRW_REGISTER_TYPE_UD); + + if (desc.file == BRW_IMMEDIATE_VALUE) { + setup = send = next_insn(p, BRW_OPCODE_SEND); + brw_set_src1(p, send, desc); + + } else { + struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD); + + brw_push_insn_state(p); + brw_set_default_access_mode(p, BRW_ALIGN_1); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_predicate_control(p, BRW_PREDICATE_NONE); + + /* Load the indirect descriptor to an address register using OR so the + * caller can specify additional descriptor bits with the usual + * brw_set_*_message() helper functions. + */ + setup = brw_OR(p, addr, desc, brw_imm_ud(0)); + + brw_pop_insn_state(p); + + send = next_insn(p, BRW_OPCODE_SEND); + brw_set_src1(p, send, addr); + } + + brw_set_dest(p, send, dst); + brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD)); + brw_inst_set_sfid(devinfo, send, sfid); + + return setup; +} + +static struct brw_inst * +brw_send_indirect_surface_message(struct brw_codegen *p, + unsigned sfid, + struct brw_reg dst, + struct brw_reg payload, + struct brw_reg surface, + unsigned message_len, + unsigned response_len, + bool header_present) +{ + const struct brw_device_info *devinfo = p->devinfo; + struct brw_inst *insn; + + if (surface.file != BRW_IMMEDIATE_VALUE) { + struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD); + + brw_push_insn_state(p); + brw_set_default_access_mode(p, BRW_ALIGN_1); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_predicate_control(p, BRW_PREDICATE_NONE); + + /* Mask out invalid bits from the surface index to avoid hangs e.g. when + * some surface array is accessed out of bounds. + */ + insn = brw_AND(p, addr, + suboffset(vec1(retype(surface, BRW_REGISTER_TYPE_UD)), + BRW_GET_SWZ(surface.dw1.bits.swizzle, 0)), + brw_imm_ud(0xff)); + + brw_pop_insn_state(p); + + surface = addr; + } + + insn = brw_send_indirect_message(p, sfid, dst, payload, surface); + brw_inst_set_mlen(devinfo, insn, message_len); + brw_inst_set_rlen(devinfo, insn, response_len); + brw_inst_set_header_present(devinfo, insn, header_present); + + return insn; } static int -brw_find_next_block_end(struct brw_compile *p, int start) +brw_find_next_block_end(struct brw_codegen *p, int start_offset) { - int ip; + int offset; void *store = p->store; + const struct brw_device_info *devinfo = p->devinfo; - for (ip = next_ip(p, start); ip < p->next_insn_offset; ip = next_ip(p, ip)) { - struct brw_instruction *insn = store + ip; + for (offset = next_offset(devinfo, store, start_offset); + offset < p->next_insn_offset; + offset = next_offset(devinfo, store, offset)) { + brw_inst *insn = store + offset; - switch (insn->header.opcode) { + switch (brw_inst_opcode(devinfo, insn)) { case BRW_OPCODE_ENDIF: case BRW_OPCODE_ELSE: case BRW_OPCODE_WHILE: case BRW_OPCODE_HALT: - return ip; + return offset; } } @@ -2360,81 +2597,90 @@ brw_find_next_block_end(struct brw_compile *p, int start) * instruction. */ static int -brw_find_loop_end(struct brw_compile *p, int start) +brw_find_loop_end(struct brw_codegen *p, int start_offset) { - struct brw_context *brw = p->brw; - int ip; - int scale = 8; + const struct brw_device_info *devinfo = p->devinfo; + int offset; + int scale = 16 / brw_jump_scale(devinfo); void *store = p->store; + assert(devinfo->gen >= 6); + /* Always start after the instruction (such as a WHILE) we're trying to fix * up. */ - for (ip = next_ip(p, start); ip < p->next_insn_offset; ip = next_ip(p, ip)) { - struct brw_instruction *insn = store + ip; - - if (insn->header.opcode == BRW_OPCODE_WHILE) { - int jip = brw->gen == 6 ? insn->bits1.branch_gen6.jump_count - : insn->bits3.break_cont.jip; - if (ip + jip * scale <= start) - return ip; + for (offset = next_offset(devinfo, store, start_offset); + offset < p->next_insn_offset; + offset = next_offset(devinfo, store, offset)) { + brw_inst *insn = store + offset; + + if (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_WHILE) { + int jip = devinfo->gen == 6 ? brw_inst_gen6_jump_count(devinfo, insn) + : brw_inst_jip(devinfo, insn); + if (offset + jip * scale <= start_offset) + return offset; } } assert(!"not reached"); - return start; + return start_offset; } /* After program generation, go back and update the UIP and JIP of * BREAK, CONT, and HALT instructions to their correct locations. */ void -brw_set_uip_jip(struct brw_compile *p) +brw_set_uip_jip(struct brw_codegen *p) { - struct brw_context *brw = p->brw; - int ip; - int scale = 8; + const struct brw_device_info *devinfo = p->devinfo; + int offset; + int br = brw_jump_scale(devinfo); + int scale = 16 / br; void *store = p->store; - if (brw->gen < 6) + if (devinfo->gen < 6) return; - for (ip = 0; ip < p->next_insn_offset; ip = next_ip(p, ip)) { - struct brw_instruction *insn = store + ip; + for (offset = 0; offset < p->next_insn_offset; + offset = next_offset(devinfo, store, offset)) { + brw_inst *insn = store + offset; - if (insn->header.cmpt_control) { + if (brw_inst_cmpt_control(devinfo, insn)) { /* Fixups for compacted BREAK/CONTINUE not supported yet. */ - assert(insn->header.opcode != BRW_OPCODE_BREAK && - insn->header.opcode != BRW_OPCODE_CONTINUE && - insn->header.opcode != BRW_OPCODE_HALT); + assert(brw_inst_opcode(devinfo, insn) != BRW_OPCODE_BREAK && + brw_inst_opcode(devinfo, insn) != BRW_OPCODE_CONTINUE && + brw_inst_opcode(devinfo, insn) != BRW_OPCODE_HALT); continue; } - int block_end_ip = brw_find_next_block_end(p, ip); - switch (insn->header.opcode) { + int block_end_offset = brw_find_next_block_end(p, offset); + switch (brw_inst_opcode(devinfo, insn)) { case BRW_OPCODE_BREAK: - assert(block_end_ip != 0); - insn->bits3.break_cont.jip = (block_end_ip - ip) / scale; + assert(block_end_offset != 0); + brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale); /* Gen7 UIP points to WHILE; Gen6 points just after it */ - insn->bits3.break_cont.uip = - (brw_find_loop_end(p, ip) - ip + - (brw->gen == 6 ? 16 : 0)) / scale; + brw_inst_set_uip(devinfo, insn, + (brw_find_loop_end(p, offset) - offset + + (devinfo->gen == 6 ? 16 : 0)) / scale); break; case BRW_OPCODE_CONTINUE: - assert(block_end_ip != 0); - insn->bits3.break_cont.jip = (block_end_ip - ip) / scale; - insn->bits3.break_cont.uip = - (brw_find_loop_end(p, ip) - ip) / scale; + assert(block_end_offset != 0); + brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale); + brw_inst_set_uip(devinfo, insn, + (brw_find_loop_end(p, offset) - offset) / scale); - assert(insn->bits3.break_cont.uip != 0); - assert(insn->bits3.break_cont.jip != 0); + assert(brw_inst_uip(devinfo, insn) != 0); + assert(brw_inst_jip(devinfo, insn) != 0); break; - case BRW_OPCODE_ENDIF: - if (block_end_ip == 0) - insn->bits3.break_cont.jip = 2; + case BRW_OPCODE_ENDIF: { + int32_t jump = (block_end_offset == 0) ? + 1 * br : (block_end_offset - offset) / scale; + if (devinfo->gen >= 7) + brw_inst_set_jip(devinfo, insn, jump); else - insn->bits3.break_cont.jip = (block_end_ip - ip) / scale; + brw_inst_set_gen6_jump_count(devinfo, insn, jump); break; + } case BRW_OPCODE_HALT: /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19): @@ -2448,28 +2694,28 @@ brw_set_uip_jip(struct brw_compile *p) * The uip will have already been set by whoever set up the * instruction. */ - if (block_end_ip == 0) { - insn->bits3.break_cont.jip = insn->bits3.break_cont.uip; + if (block_end_offset == 0) { + brw_inst_set_jip(devinfo, insn, brw_inst_uip(devinfo, insn)); } else { - insn->bits3.break_cont.jip = (block_end_ip - ip) / scale; + brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale); } - assert(insn->bits3.break_cont.uip != 0); - assert(insn->bits3.break_cont.jip != 0); + assert(brw_inst_uip(devinfo, insn) != 0); + assert(brw_inst_jip(devinfo, insn) != 0); break; } } } -void brw_ff_sync(struct brw_compile *p, +void brw_ff_sync(struct brw_codegen *p, struct brw_reg dest, - GLuint msg_reg_nr, + unsigned msg_reg_nr, struct brw_reg src0, bool allocate, - GLuint response_length, + unsigned response_length, bool eot) { - struct brw_context *brw = p->brw; - struct brw_instruction *insn; + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); @@ -2478,8 +2724,8 @@ void brw_ff_sync(struct brw_compile *p, brw_set_src0(p, insn, src0); brw_set_src1(p, insn, brw_imm_d(0)); - if (brw->gen < 6) - insn->header.destreg__conditionalmod = msg_reg_nr; + if (devinfo->gen < 6) + brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr); brw_set_ff_sync_message(p, insn, @@ -2500,14 +2746,14 @@ void brw_ff_sync(struct brw_compile *p, * writes are complete by sending the final write as a committed write." */ void -brw_svb_write(struct brw_compile *p, +brw_svb_write(struct brw_codegen *p, struct brw_reg dest, - GLuint msg_reg_nr, + unsigned msg_reg_nr, struct brw_reg src0, - GLuint binding_table_index, + unsigned binding_table_index, bool send_commit_msg) { - struct brw_instruction *insn; + brw_inst *insn; gen6_resolve_implied_move(p, &src0, msg_reg_nr); @@ -2527,122 +2773,592 @@ brw_svb_write(struct brw_compile *p, send_commit_msg); /* send_commit_msg */ } +static unsigned +brw_surface_payload_size(struct brw_codegen *p, + unsigned num_channels, + bool has_simd4x2, + bool has_simd16) +{ + if (has_simd4x2 && brw_inst_access_mode(p->devinfo, p->current) == BRW_ALIGN_16) + return 1; + else if (has_simd16 && p->compressed) + return 2 * num_channels; + else + return num_channels; +} + static void -brw_set_dp_untyped_atomic_message(struct brw_compile *p, - struct brw_instruction *insn, - GLuint atomic_op, - GLuint bind_table_index, - GLuint msg_length, - GLuint response_length, - bool header_present) +brw_set_dp_untyped_atomic_message(struct brw_codegen *p, + brw_inst *insn, + unsigned atomic_op, + bool response_expected) +{ + const struct brw_device_info *devinfo = p->devinfo; + unsigned msg_control = + atomic_op | /* Atomic Operation Type: BRW_AOP_* */ + (response_expected ? 1 << 5 : 0); /* Return data expected */ + + if (devinfo->gen >= 8 || devinfo->is_haswell) { + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (!p->compressed) + msg_control |= 1 << 4; /* SIMD8 mode */ + + brw_inst_set_dp_msg_type(devinfo, insn, + HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP); + } else { + brw_inst_set_dp_msg_type(devinfo, insn, + HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2); + } + } else { + brw_inst_set_dp_msg_type(devinfo, insn, + GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP); + + if (!p->compressed) + msg_control |= 1 << 4; /* SIMD8 mode */ + } + + brw_inst_set_dp_msg_control(devinfo, insn, msg_control); +} + +void +brw_untyped_atomic(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg payload, + struct brw_reg surface, + unsigned atomic_op, + unsigned msg_length, + bool response_expected) +{ + const struct brw_device_info *devinfo = p->devinfo; + const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN7_SFID_DATAPORT_DATA_CACHE); + const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1; + /* Mask out unused components -- This is especially important in Align16 + * mode on generations that don't have native support for SIMD4x2 atomics, + * because unused but enabled components will cause the dataport to perform + * additional atomic operations on the addresses that happen to be in the + * uninitialized Y, Z and W coordinates of the payload. + */ + const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X; + struct brw_inst *insn = brw_send_indirect_surface_message( + p, sfid, brw_writemask(dst, mask), payload, surface, msg_length, + brw_surface_payload_size(p, response_expected, + devinfo->gen >= 8 || devinfo->is_haswell, true), + align1); + + brw_set_dp_untyped_atomic_message( + p, insn, atomic_op, response_expected); +} + +static void +brw_set_dp_untyped_surface_read_message(struct brw_codegen *p, + struct brw_inst *insn, + unsigned num_channels) { - if (p->brw->is_haswell) { - brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1, - msg_length, response_length, - header_present, false); + const struct brw_device_info *devinfo = p->devinfo; + /* Set mask of 32-bit channels to drop. */ + unsigned msg_control = 0xf & (0xf << num_channels); + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (p->compressed) + msg_control |= 1 << 4; /* SIMD16 mode */ + else + msg_control |= 2 << 4; /* SIMD8 mode */ + } - if (insn->header.access_mode == BRW_ALIGN_1) { - if (insn->header.execution_size != BRW_EXECUTE_16) - insn->bits3.ud |= 1 << 12; /* SIMD8 mode */ + brw_inst_set_dp_msg_type(devinfo, insn, + (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ : + GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ)); + brw_inst_set_dp_msg_control(devinfo, insn, msg_control); +} + +void +brw_untyped_surface_read(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg payload, + struct brw_reg surface, + unsigned msg_length, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN7_SFID_DATAPORT_DATA_CACHE); + const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1); + struct brw_inst *insn = brw_send_indirect_surface_message( + p, sfid, dst, payload, surface, msg_length, + brw_surface_payload_size(p, num_channels, true, true), + align1); + + brw_set_dp_untyped_surface_read_message( + p, insn, num_channels); +} - insn->bits3.gen7_dp.msg_type = - HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP; +static void +brw_set_dp_untyped_surface_write_message(struct brw_codegen *p, + struct brw_inst *insn, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + /* Set mask of 32-bit channels to drop. */ + unsigned msg_control = 0xf & (0xf << num_channels); + + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (p->compressed) + msg_control |= 1 << 4; /* SIMD16 mode */ + else + msg_control |= 2 << 4; /* SIMD8 mode */ + } else { + if (devinfo->gen >= 8 || devinfo->is_haswell) + msg_control |= 0 << 4; /* SIMD4x2 mode */ + else + msg_control |= 2 << 4; /* SIMD8 mode */ + } + + brw_inst_set_dp_msg_type(devinfo, insn, + devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE : + GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE); + brw_inst_set_dp_msg_control(devinfo, insn, msg_control); +} + +void +brw_untyped_surface_write(struct brw_codegen *p, + struct brw_reg payload, + struct brw_reg surface, + unsigned msg_length, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN7_SFID_DATAPORT_DATA_CACHE); + const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1; + /* Mask out unused components -- See comment in brw_untyped_atomic(). */ + const unsigned mask = devinfo->gen == 7 && !devinfo->is_haswell && !align1 ? + WRITEMASK_X : WRITEMASK_XYZW; + struct brw_inst *insn = brw_send_indirect_surface_message( + p, sfid, brw_writemask(brw_null_reg(), mask), + payload, surface, msg_length, 0, align1); + + brw_set_dp_untyped_surface_write_message( + p, insn, num_channels); +} + +static void +brw_set_dp_typed_atomic_message(struct brw_codegen *p, + struct brw_inst *insn, + unsigned atomic_op, + bool response_expected) +{ + const struct brw_device_info *devinfo = p->devinfo; + unsigned msg_control = + atomic_op | /* Atomic Operation Type: BRW_AOP_* */ + (response_expected ? 1 << 5 : 0); /* Return data expected */ + + if (devinfo->gen >= 8 || devinfo->is_haswell) { + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */ + + brw_inst_set_dp_msg_type(devinfo, insn, + HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP); } else { - insn->bits3.gen7_dp.msg_type = - HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2; + brw_inst_set_dp_msg_type(devinfo, insn, + HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2); } } else { - brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE, - msg_length, response_length, - header_present, false); - - insn->bits3.gen7_dp.msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP; + brw_inst_set_dp_msg_type(devinfo, insn, + GEN7_DATAPORT_RC_TYPED_ATOMIC_OP); - if (insn->header.execution_size != BRW_EXECUTE_16) - insn->bits3.ud |= 1 << 12; /* SIMD8 mode */ + if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */ } - if (response_length) - insn->bits3.ud |= 1 << 13; /* Return data expected */ + brw_inst_set_dp_msg_control(devinfo, insn, msg_control); +} - insn->bits3.gen7_dp.binding_table_index = bind_table_index; - insn->bits3.ud |= atomic_op << 8; +void +brw_typed_atomic(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg payload, + struct brw_reg surface, + unsigned atomic_op, + unsigned msg_length, + bool response_expected) { + const struct brw_device_info *devinfo = p->devinfo; + const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN6_SFID_DATAPORT_RENDER_CACHE); + const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1); + /* Mask out unused components -- See comment in brw_untyped_atomic(). */ + const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X; + struct brw_inst *insn = brw_send_indirect_surface_message( + p, sfid, brw_writemask(dst, mask), payload, surface, msg_length, + brw_surface_payload_size(p, response_expected, + devinfo->gen >= 8 || devinfo->is_haswell, false), + true); + + brw_set_dp_typed_atomic_message( + p, insn, atomic_op, response_expected); +} + +static void +brw_set_dp_typed_surface_read_message(struct brw_codegen *p, + struct brw_inst *insn, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + /* Set mask of unused channels. */ + unsigned msg_control = 0xf & (0xf << num_channels); + + if (devinfo->gen >= 8 || devinfo->is_haswell) { + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */ + else + msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */ + } + + brw_inst_set_dp_msg_type(devinfo, insn, + HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ); + } else { + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */ + } + + brw_inst_set_dp_msg_type(devinfo, insn, + GEN7_DATAPORT_RC_TYPED_SURFACE_READ); + } + + brw_inst_set_dp_msg_control(devinfo, insn, msg_control); } void -brw_untyped_atomic(struct brw_compile *p, - struct brw_reg dest, - struct brw_reg mrf, - GLuint atomic_op, - GLuint bind_table_index, - GLuint msg_length, - GLuint response_length) { - struct brw_instruction *insn = brw_next_insn(p, BRW_OPCODE_SEND); - - brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD)); - brw_set_src0(p, insn, retype(mrf, BRW_REGISTER_TYPE_UD)); - brw_set_src1(p, insn, brw_imm_d(0)); - brw_set_dp_untyped_atomic_message( - p, insn, atomic_op, bind_table_index, msg_length, response_length, - insn->header.access_mode == BRW_ALIGN_1); +brw_typed_surface_read(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg payload, + struct brw_reg surface, + unsigned msg_length, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN6_SFID_DATAPORT_RENDER_CACHE); + struct brw_inst *insn = brw_send_indirect_surface_message( + p, sfid, dst, payload, surface, msg_length, + brw_surface_payload_size(p, num_channels, + devinfo->gen >= 8 || devinfo->is_haswell, false), + true); + + brw_set_dp_typed_surface_read_message( + p, insn, num_channels); } static void -brw_set_dp_untyped_surface_read_message(struct brw_compile *p, - struct brw_instruction *insn, - GLuint bind_table_index, - GLuint msg_length, - GLuint response_length, - bool header_present) -{ - const unsigned dispatch_width = - (insn->header.execution_size == BRW_EXECUTE_16 ? 16 : 8); - const unsigned num_channels = response_length / (dispatch_width / 8); - - if (p->brw->is_haswell) { - brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1, - msg_length, response_length, - header_present, false); - - insn->bits3.gen7_dp.msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ; +brw_set_dp_typed_surface_write_message(struct brw_codegen *p, + struct brw_inst *insn, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + /* Set mask of unused channels. */ + unsigned msg_control = 0xf & (0xf << num_channels); + + if (devinfo->gen >= 8 || devinfo->is_haswell) { + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */ + else + msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */ + } + + brw_inst_set_dp_msg_type(devinfo, insn, + HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE); + } else { - brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE, - msg_length, response_length, - header_present, false); + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + if (brw_inst_qtr_control(devinfo, p->current) == GEN6_COMPRESSION_2Q) + msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */ + } - insn->bits3.gen7_dp.msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ; + brw_inst_set_dp_msg_type(devinfo, insn, + GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE); } - if (insn->header.access_mode == BRW_ALIGN_1) { - if (dispatch_width == 16) - insn->bits3.ud |= 1 << 12; /* SIMD16 mode */ - else - insn->bits3.ud |= 2 << 12; /* SIMD8 mode */ + brw_inst_set_dp_msg_control(devinfo, insn, msg_control); +} + +void +brw_typed_surface_write(struct brw_codegen *p, + struct brw_reg payload, + struct brw_reg surface, + unsigned msg_length, + unsigned num_channels) +{ + const struct brw_device_info *devinfo = p->devinfo; + const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN6_SFID_DATAPORT_RENDER_CACHE); + const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1); + /* Mask out unused components -- See comment in brw_untyped_atomic(). */ + const unsigned mask = (devinfo->gen == 7 && !devinfo->is_haswell && !align1 ? + WRITEMASK_X : WRITEMASK_XYZW); + struct brw_inst *insn = brw_send_indirect_surface_message( + p, sfid, brw_writemask(brw_null_reg(), mask), + payload, surface, msg_length, 0, true); + + brw_set_dp_typed_surface_write_message( + p, insn, num_channels); +} + +static void +brw_set_memory_fence_message(struct brw_codegen *p, + struct brw_inst *insn, + enum brw_message_target sfid, + bool commit_enable) +{ + const struct brw_device_info *devinfo = p->devinfo; + + brw_set_message_descriptor(p, insn, sfid, + 1 /* message length */, + (commit_enable ? 1 : 0) /* response length */, + true /* header present */, + false); + + switch (sfid) { + case GEN6_SFID_DATAPORT_RENDER_CACHE: + brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_RC_MEMORY_FENCE); + break; + case GEN7_SFID_DATAPORT_DATA_CACHE: + brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_DC_MEMORY_FENCE); + break; + default: + unreachable("Not reached"); } - insn->bits3.gen7_dp.binding_table_index = bind_table_index; + if (commit_enable) + brw_inst_set_dp_msg_control(devinfo, insn, 1 << 5); +} - /* Set mask of 32-bit channels to drop. */ - insn->bits3.ud |= (0xf & (0xf << num_channels)) << 8; +void +brw_memory_fence(struct brw_codegen *p, + struct brw_reg dst) +{ + const struct brw_device_info *devinfo = p->devinfo; + const bool commit_enable = devinfo->gen == 7 && !devinfo->is_haswell; + struct brw_inst *insn; + + /* Set dst as destination for dependency tracking, the MEMORY_FENCE + * message doesn't write anything back. + */ + insn = next_insn(p, BRW_OPCODE_SEND); + brw_set_dest(p, insn, dst); + brw_set_src0(p, insn, dst); + brw_set_memory_fence_message(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE, + commit_enable); + + if (devinfo->gen == 7 && !devinfo->is_haswell) { + /* IVB does typed surface access through the render cache, so we need to + * flush it too. Use a different register so both flushes can be + * pipelined by the hardware. + */ + insn = next_insn(p, BRW_OPCODE_SEND); + brw_set_dest(p, insn, offset(dst, 1)); + brw_set_src0(p, insn, offset(dst, 1)); + brw_set_memory_fence_message(p, insn, GEN6_SFID_DATAPORT_RENDER_CACHE, + commit_enable); + + /* Now write the response of the second message into the response of the + * first to trigger a pipeline stall -- This way future render and data + * cache messages will be properly ordered with respect to past data and + * render cache messages. + */ + brw_push_insn_state(p); + brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_MOV(p, dst, offset(dst, 1)); + brw_pop_insn_state(p); + } } void -brw_untyped_surface_read(struct brw_compile *p, - struct brw_reg dest, - struct brw_reg mrf, - GLuint bind_table_index, - GLuint msg_length, - GLuint response_length) +brw_pixel_interpolator_query(struct brw_codegen *p, + struct brw_reg dest, + struct brw_reg mrf, + bool noperspective, + unsigned mode, + unsigned data, + unsigned msg_length, + unsigned response_length) { - struct brw_instruction *insn = next_insn(p, BRW_OPCODE_SEND); + const struct brw_device_info *devinfo = p->devinfo; + struct brw_inst *insn = next_insn(p, BRW_OPCODE_SEND); - brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD)); - brw_set_src0(p, insn, retype(mrf, BRW_REGISTER_TYPE_UD)); - brw_set_dp_untyped_surface_read_message( - p, insn, bind_table_index, msg_length, response_length, - insn->header.access_mode == BRW_ALIGN_1); + brw_set_dest(p, insn, dest); + brw_set_src0(p, insn, mrf); + brw_set_message_descriptor(p, insn, GEN7_SFID_PIXEL_INTERPOLATOR, + msg_length, response_length, + false /* header is never present for PI */, + false); + + brw_inst_set_pi_simd_mode( + devinfo, insn, brw_inst_exec_size(devinfo, insn) == BRW_EXECUTE_16); + brw_inst_set_pi_slot_group(devinfo, insn, 0); /* zero unless 32/64px dispatch */ + brw_inst_set_pi_nopersp(devinfo, insn, noperspective); + brw_inst_set_pi_message_type(devinfo, insn, mode); + brw_inst_set_pi_message_data(devinfo, insn, data); +} + +void +brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst) +{ + const struct brw_device_info *devinfo = p->devinfo; + brw_inst *inst; + + assert(devinfo->gen >= 7); + + brw_push_insn_state(p); + + if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) { + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + + if (devinfo->gen >= 8) { + /* Getting the first active channel index is easy on Gen8: Just find + * the first bit set in the mask register. The same register exists + * on HSW already but it reads back as all ones when the current + * instruction has execution masking disabled, so it's kind of + * useless. + */ + inst = brw_FBL(p, vec1(dst), + retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD)); + + /* Quarter control has the effect of magically shifting the value of + * this register. Make sure it's set to zero. + */ + brw_inst_set_qtr_control(devinfo, inst, GEN6_COMPRESSION_1Q); + } else { + const struct brw_reg flag = retype(brw_flag_reg(1, 0), + BRW_REGISTER_TYPE_UD); + + brw_MOV(p, flag, brw_imm_ud(0)); + + /* Run a 16-wide instruction returning zero with execution masking + * and a conditional modifier enabled in order to get the current + * execution mask in f1.0. + */ + inst = brw_MOV(p, brw_null_reg(), brw_imm_ud(0)); + brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_16); + brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); + brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z); + brw_inst_set_flag_reg_nr(devinfo, inst, 1); + + brw_FBL(p, vec1(dst), flag); + } + } else { + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + + if (devinfo->gen >= 8) { + /* In SIMD4x2 mode the first active channel index is just the + * negation of the first bit of the mask register. + */ + inst = brw_AND(p, brw_writemask(dst, WRITEMASK_X), + negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD)), + brw_imm_ud(1)); + + } else { + /* Overwrite the destination without and with execution masking to + * find out which of the channels is active. + */ + brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X), + brw_imm_ud(1)); + + inst = brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X), + brw_imm_ud(0)); + brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE); + } + } + + brw_pop_insn_state(p); +} + +void +brw_broadcast(struct brw_codegen *p, + struct brw_reg dst, + struct brw_reg src, + struct brw_reg idx) +{ + const struct brw_device_info *devinfo = p->devinfo; + const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1; + brw_inst *inst; + + assert(src.file == BRW_GENERAL_REGISTER_FILE && + src.address_mode == BRW_ADDRESS_DIRECT); + + if ((src.vstride == 0 && (src.hstride == 0 || !align1)) || + idx.file == BRW_IMMEDIATE_VALUE) { + /* Trivial, the source is already uniform or the index is a constant. + * We will typically not get here if the optimizer is doing its job, but + * asserting would be mean. + */ + const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.dw1.ud : 0; + brw_MOV(p, dst, + (align1 ? stride(suboffset(src, i), 0, 1, 0) : + stride(suboffset(src, 4 * i), 0, 4, 1))); + } else { + if (align1) { + const struct brw_reg addr = + retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD); + const unsigned offset = src.nr * REG_SIZE + src.subnr; + /* Limit in bytes of the signed indirect addressing immediate. */ + const unsigned limit = 512; + + brw_push_insn_state(p); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_predicate_control(p, BRW_PREDICATE_NONE); + + /* Take into account the component size and horizontal stride. */ + assert(src.vstride == src.hstride + src.width); + brw_SHL(p, addr, vec1(idx), + brw_imm_ud(_mesa_logbase2(type_sz(src.type)) + + src.hstride - 1)); + + /* We can only address up to limit bytes using the indirect + * addressing immediate, account for the difference if the source + * register is above this limit. + */ + if (offset >= limit) + brw_ADD(p, addr, addr, brw_imm_ud(offset - offset % limit)); + + brw_pop_insn_state(p); + + /* Use indirect addressing to fetch the specified component. */ + brw_MOV(p, dst, + retype(brw_vec1_indirect(addr.subnr, offset % limit), + src.type)); + } else { + /* In SIMD4x2 mode the index can be either zero or one, replicate it + * to all bits of a flag register, + */ + inst = brw_MOV(p, + brw_null_reg(), + stride(brw_swizzle1(idx, 0), 0, 4, 1)); + brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NONE); + brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_NZ); + brw_inst_set_flag_reg_nr(devinfo, inst, 1); + + /* and use predicated SEL to pick the right channel. */ + inst = brw_SEL(p, dst, + stride(suboffset(src, 4), 0, 4, 1), + stride(src, 0, 4, 1)); + brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NORMAL); + brw_inst_set_flag_reg_nr(devinfo, inst, 1); + } + } } /** @@ -2661,18 +3377,20 @@ brw_untyped_surface_read(struct brw_compile *p, * format, and is only accessible through the legacy DATA_CACHE dataport * messages. */ -void brw_shader_time_add(struct brw_compile *p, +void brw_shader_time_add(struct brw_codegen *p, struct brw_reg payload, uint32_t surf_index) { - struct brw_context *brw = p->brw; - assert(brw->gen >= 7); + const unsigned sfid = (p->devinfo->gen >= 8 || p->devinfo->is_haswell ? + HSW_SFID_DATAPORT_DATA_CACHE_1 : + GEN7_SFID_DATAPORT_DATA_CACHE); + assert(p->devinfo->gen >= 7); brw_push_insn_state(p); - brw_set_access_mode(p, BRW_ALIGN_1); - brw_set_mask_control(p, BRW_MASK_DISABLE); - struct brw_instruction *send = brw_next_insn(p, BRW_OPCODE_SEND); - brw_pop_insn_state(p); + brw_set_default_access_mode(p, BRW_ALIGN_1); + brw_set_default_mask_control(p, BRW_MASK_DISABLE); + brw_set_default_compression_control(p, BRW_COMPRESSION_NONE); + brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND); /* We use brw_vec1_reg and unmasked because we want to increment the given * offset only once. @@ -2681,8 +3399,61 @@ void brw_shader_time_add(struct brw_compile *p, BRW_ARF_NULL, 0)); brw_set_src0(p, send, brw_vec1_reg(payload.file, payload.nr, 0)); - brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, surf_index, - 2 /* message length */, - 0 /* response length */, - false /* header present */); + brw_set_src1(p, send, brw_imm_ud(0)); + brw_set_message_descriptor(p, send, sfid, 2, 0, false, false); + brw_inst_set_binding_table_index(p->devinfo, send, surf_index); + brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, false); + + brw_pop_insn_state(p); +} + + +/** + * Emit the SEND message for a barrier + */ +void +brw_barrier(struct brw_codegen *p, struct brw_reg src) +{ + const struct brw_device_info *devinfo = p->devinfo; + struct brw_inst *inst; + + assert(devinfo->gen >= 7); + + inst = next_insn(p, BRW_OPCODE_SEND); + brw_set_dest(p, inst, brw_null_reg()); + brw_set_src0(p, inst, src); + brw_set_src1(p, inst, brw_null_reg()); + + brw_set_message_descriptor(p, inst, BRW_SFID_MESSAGE_GATEWAY, + 1 /* msg_length */, + 0 /* response_length */, + false /* header_present */, + false /* end_of_thread */); + + brw_inst_set_gateway_notify(devinfo, inst, 1); + brw_inst_set_gateway_subfuncid(devinfo, inst, + BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG); + + brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE); +} + + +/** + * Emit the wait instruction for a barrier + */ +void +brw_WAIT(struct brw_codegen *p) +{ + const struct brw_device_info *devinfo = p->devinfo; + struct brw_inst *insn; + + struct brw_reg src = brw_notification_reg(); + + insn = next_insn(p, BRW_OPCODE_WAIT); + brw_set_dest(p, insn, src); + brw_set_src0(p, insn, src); + brw_set_src1(p, insn, brw_null_reg()); + + brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1); + brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE); }