2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_context.h"
34 #include "brw_defines.h"
37 #include "util/ralloc.h"
40 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
41 * registers, implicitly moving the operand to a message register.
43 * On Sandybridge, this is no longer the case. This function performs the
44 * explicit move; it should be called before emitting a SEND instruction.
47 gen6_resolve_implied_move(struct brw_codegen
*p
,
51 const struct brw_device_info
*devinfo
= p
->devinfo
;
55 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
58 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct brw_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 * Convert a brw_reg_type enumeration value into the hardware representation.
91 * The hardware encoding may depend on whether the value is an immediate.
94 brw_reg_type_to_hw_type(const struct brw_device_info
*devinfo
,
95 enum brw_reg_type type
, enum brw_reg_file file
)
97 if (file
== BRW_IMMEDIATE_VALUE
) {
98 static const int imm_hw_types
[] = {
99 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
100 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
101 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
102 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
103 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
104 [BRW_REGISTER_TYPE_UB
] = -1,
105 [BRW_REGISTER_TYPE_B
] = -1,
106 [BRW_REGISTER_TYPE_UV
] = BRW_HW_REG_IMM_TYPE_UV
,
107 [BRW_REGISTER_TYPE_VF
] = BRW_HW_REG_IMM_TYPE_VF
,
108 [BRW_REGISTER_TYPE_V
] = BRW_HW_REG_IMM_TYPE_V
,
109 [BRW_REGISTER_TYPE_DF
] = GEN8_HW_REG_IMM_TYPE_DF
,
110 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_IMM_TYPE_HF
,
111 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
112 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
114 assert(type
< ARRAY_SIZE(imm_hw_types
));
115 assert(imm_hw_types
[type
] != -1);
116 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_DF
);
117 return imm_hw_types
[type
];
119 /* Non-immediate registers */
120 static const int hw_types
[] = {
121 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
122 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
123 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
124 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
125 [BRW_REGISTER_TYPE_UB
] = BRW_HW_REG_NON_IMM_TYPE_UB
,
126 [BRW_REGISTER_TYPE_B
] = BRW_HW_REG_NON_IMM_TYPE_B
,
127 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
128 [BRW_REGISTER_TYPE_UV
] = -1,
129 [BRW_REGISTER_TYPE_VF
] = -1,
130 [BRW_REGISTER_TYPE_V
] = -1,
131 [BRW_REGISTER_TYPE_DF
] = GEN7_HW_REG_NON_IMM_TYPE_DF
,
132 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_NON_IMM_TYPE_HF
,
133 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
134 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
136 assert(type
< ARRAY_SIZE(hw_types
));
137 assert(hw_types
[type
] != -1);
138 assert(devinfo
->gen
>= 7 || type
< BRW_REGISTER_TYPE_DF
);
139 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_HF
);
140 return hw_types
[type
];
145 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
147 const struct brw_device_info
*devinfo
= p
->devinfo
;
149 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
150 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
151 else if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
152 assert(dest
.nr
< 128);
154 gen7_convert_mrf_to_grf(p
, &dest
);
156 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
157 brw_inst_set_dst_reg_type(devinfo
, inst
,
158 brw_reg_type_to_hw_type(devinfo
, dest
.type
,
160 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
162 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
163 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
165 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
166 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
167 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
168 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
169 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
171 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
172 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
173 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
174 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
175 assert(dest
.writemask
!= 0);
177 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
178 * Although Dst.HorzStride is a don't care for Align16, HW needs
179 * this to be programmed as "01".
181 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
184 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
186 /* These are different sizes in align1 vs align16:
188 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
189 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
190 dest
.indirect_offset
);
191 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
192 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
193 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
195 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
196 dest
.indirect_offset
);
197 /* even ignored in da16, still need to set as '01' */
198 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
202 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
203 * or 16 (SIMD16), as that's normally correct. However, when dealing with
204 * small registers, we automatically reduce it to match the register size.
206 if (dest
.width
< BRW_EXECUTE_8
)
207 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
210 extern int reg_type_size
[];
213 validate_reg(const struct brw_device_info
*devinfo
,
214 brw_inst
*inst
, struct brw_reg reg
)
216 const int hstride_for_reg
[] = {0, 1, 2, 4};
217 const int vstride_for_reg
[] = {0, 1, 2, 4, 8, 16, 32};
218 const int width_for_reg
[] = {1, 2, 4, 8, 16};
219 const int execsize_for_reg
[] = {1, 2, 4, 8, 16, 32};
220 int width
, hstride
, vstride
, execsize
;
222 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
223 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
224 * mean the destination has to be 128-bit aligned and the
225 * destination horiz stride has to be a word.
227 if (reg
.type
== BRW_REGISTER_TYPE_V
) {
228 assert(hstride_for_reg
[brw_inst_dst_hstride(devinfo
, inst
)] *
229 reg_type_size
[brw_inst_dst_reg_type(devinfo
, inst
)] == 2);
235 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
236 reg
.file
== BRW_ARF_NULL
)
239 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
241 * "Swizzling is not allowed when an accumulator is used as an implicit
242 * source or an explicit source in an instruction."
244 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
245 reg
.nr
== BRW_ARF_ACCUMULATOR
)
246 assert(reg
.swizzle
== BRW_SWIZZLE_XYZW
);
248 assert(reg
.hstride
>= 0 && reg
.hstride
< ARRAY_SIZE(hstride_for_reg
));
249 hstride
= hstride_for_reg
[reg
.hstride
];
251 if (reg
.vstride
== 0xf) {
254 assert(reg
.vstride
>= 0 && reg
.vstride
< ARRAY_SIZE(vstride_for_reg
));
255 vstride
= vstride_for_reg
[reg
.vstride
];
258 assert(reg
.width
>= 0 && reg
.width
< ARRAY_SIZE(width_for_reg
));
259 width
= width_for_reg
[reg
.width
];
261 assert(brw_inst_exec_size(devinfo
, inst
) >= 0 &&
262 brw_inst_exec_size(devinfo
, inst
) < ARRAY_SIZE(execsize_for_reg
));
263 execsize
= execsize_for_reg
[brw_inst_exec_size(devinfo
, inst
)];
265 /* Restrictions from 3.3.10: Register Region Restrictions. */
267 assert(execsize
>= width
);
270 if (execsize
== width
&& hstride
!= 0) {
271 assert(vstride
== -1 || vstride
== width
* hstride
);
275 if (execsize
== width
&& hstride
== 0) {
276 /* no restriction on vstride. */
281 assert(hstride
== 0);
285 if (execsize
== 1 && width
== 1) {
286 assert(hstride
== 0);
287 assert(vstride
== 0);
291 if (vstride
== 0 && hstride
== 0) {
295 /* 10. Check destination issues. */
299 is_compactable_immediate(unsigned imm
)
301 /* We get the low 12 bits as-is. */
304 /* We get one bit replicated through the top 20 bits. */
305 return imm
== 0 || imm
== 0xfffff000;
309 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
311 const struct brw_device_info
*devinfo
= p
->devinfo
;
313 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
314 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
315 else if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
316 assert(reg
.nr
< 128);
318 gen7_convert_mrf_to_grf(p
, ®
);
320 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
321 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
322 /* Any source modifiers or regions will be ignored, since this just
323 * identifies the MRF/GRF to start reading the message contents from.
324 * Check for some likely failures.
328 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
331 validate_reg(devinfo
, inst
, reg
);
333 brw_inst_set_src0_reg_file(devinfo
, inst
, reg
.file
);
334 brw_inst_set_src0_reg_type(devinfo
, inst
,
335 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
336 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
337 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
338 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
340 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
341 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
343 /* The Bspec's section titled "Non-present Operands" claims that if src0
344 * is an immediate that src1's type must be the same as that of src0.
346 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
347 * that do not follow this rule. E.g., from the IVB/HSW table:
349 * DataTypeIndex 18-Bit Mapping Mapped Meaning
350 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
352 * And from the SNB table:
354 * DataTypeIndex 18-Bit Mapping Mapped Meaning
355 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
357 * Neither of these cause warnings from the simulator when used,
358 * compacted or otherwise. In fact, all compaction mappings that have an
359 * immediate in src0 use a:ud for src1.
361 * The GM45 instruction compaction tables do not contain mapped meanings
362 * so it's not clear whether it has the restriction. We'll assume it was
363 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
365 brw_inst_set_src1_reg_file(devinfo
, inst
, BRW_ARCHITECTURE_REGISTER_FILE
);
366 if (devinfo
->gen
< 6) {
367 brw_inst_set_src1_reg_type(devinfo
, inst
,
368 brw_inst_src0_reg_type(devinfo
, inst
));
370 brw_inst_set_src1_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
373 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
374 * for immediate values. Presumably the hardware engineers realized
375 * that the only useful floating-point value that could be represented
376 * in this format is 0.0, which can also be represented as a VF-typed
377 * immediate, so they gave us the previously mentioned mapping on IVB+.
379 * Strangely, we do have a mapping for imm:f in src1, so we don't need
382 * If we see a 0.0:F, change the type to VF so that it can be compacted.
384 if (brw_inst_imm_ud(devinfo
, inst
) == 0x0 &&
385 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_F
) {
386 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_IMM_TYPE_VF
);
389 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
390 * set the types to :UD so the instruction can be compacted.
392 if (is_compactable_immediate(brw_inst_imm_ud(devinfo
, inst
)) &&
393 brw_inst_cond_modifier(devinfo
, inst
) == BRW_CONDITIONAL_NONE
&&
394 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
&&
395 brw_inst_dst_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
) {
396 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
397 brw_inst_set_dst_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
400 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
401 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
402 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
403 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
405 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
408 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
410 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
411 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
413 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
417 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
418 if (reg
.width
== BRW_WIDTH_1
&&
419 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
420 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
421 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
422 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
424 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
425 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
426 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
429 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
430 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
431 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
432 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
433 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
434 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
435 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
436 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
438 /* This is an oddity of the fact we're using the same
439 * descriptions for registers in align_16 as align_1:
441 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
442 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
444 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
451 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
453 const struct brw_device_info
*devinfo
= p
->devinfo
;
455 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
456 assert(reg
.nr
< 128);
458 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
460 * "Accumulator registers may be accessed explicitly as src0
463 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
464 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
466 gen7_convert_mrf_to_grf(p
, ®
);
467 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
469 validate_reg(devinfo
, inst
, reg
);
471 brw_inst_set_src1_reg_file(devinfo
, inst
, reg
.file
);
472 brw_inst_set_src1_reg_type(devinfo
, inst
,
473 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
474 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
475 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
477 /* Only src1 can be immediate in two-argument instructions.
479 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
481 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
482 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
484 /* This is a hardware restriction, which may or may not be lifted
487 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
488 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
490 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
491 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
492 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
494 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
497 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
498 if (reg
.width
== BRW_WIDTH_1
&&
499 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
500 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
501 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
502 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
504 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
505 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
506 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
509 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
510 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
511 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
512 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
513 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
514 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
515 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
516 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
518 /* This is an oddity of the fact we're using the same
519 * descriptions for registers in align_16 as align_1:
521 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
522 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
524 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
530 * Set the Message Descriptor and Extended Message Descriptor fields
533 * \note This zeroes out the Function Control bits, so it must be called
534 * \b before filling out any message-specific data. Callers can
535 * choose not to fill in irrelevant bits; they will be zero.
538 brw_set_message_descriptor(struct brw_codegen
*p
,
540 enum brw_message_target sfid
,
542 unsigned response_length
,
546 const struct brw_device_info
*devinfo
= p
->devinfo
;
548 brw_set_src1(p
, inst
, brw_imm_d(0));
550 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
551 * itself; instead, it will be a MOV/OR into the address register.
553 * In this case, we avoid setting the extended message descriptor bits,
554 * since they go on the later SEND/SENDC instead and if set here would
555 * instead clobber the conditionalmod bits.
557 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
558 if (opcode
== BRW_OPCODE_SEND
|| opcode
== BRW_OPCODE_SENDC
) {
559 brw_inst_set_sfid(devinfo
, inst
, sfid
);
562 brw_inst_set_mlen(devinfo
, inst
, msg_length
);
563 brw_inst_set_rlen(devinfo
, inst
, response_length
);
564 brw_inst_set_eot(devinfo
, inst
, end_of_thread
);
566 if (devinfo
->gen
>= 5) {
567 brw_inst_set_header_present(devinfo
, inst
, header_present
);
571 static void brw_set_math_message( struct brw_codegen
*p
,
574 unsigned integer_type
,
578 const struct brw_device_info
*devinfo
= p
->devinfo
;
580 unsigned response_length
;
582 /* Infer message length from the function */
584 case BRW_MATH_FUNCTION_POW
:
585 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
586 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
587 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
595 /* Infer response length from the function */
597 case BRW_MATH_FUNCTION_SINCOS
:
598 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
607 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
608 msg_length
, response_length
, false, false);
609 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
610 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
611 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
612 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
613 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
614 brw_inst_set_saturate(devinfo
, inst
, 0);
618 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
621 unsigned response_length
,
624 const struct brw_device_info
*devinfo
= p
->devinfo
;
626 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
627 1, response_length
, true, end_of_thread
);
628 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
629 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
630 /* The following fields are not used by FF_SYNC: */
631 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
632 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
633 brw_inst_set_urb_used(devinfo
, insn
, 0);
634 brw_inst_set_urb_complete(devinfo
, insn
, 0);
637 static void brw_set_urb_message( struct brw_codegen
*p
,
639 enum brw_urb_write_flags flags
,
641 unsigned response_length
,
643 unsigned swizzle_control
)
645 const struct brw_device_info
*devinfo
= p
->devinfo
;
647 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
648 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
649 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
651 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
652 msg_length
, response_length
, true,
653 flags
& BRW_URB_WRITE_EOT
);
655 if (flags
& BRW_URB_WRITE_OWORD
) {
656 assert(msg_length
== 2); /* header + one OWORD of data */
657 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
659 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
662 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
663 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
665 if (devinfo
->gen
< 8) {
666 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
669 if (devinfo
->gen
< 7) {
670 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
671 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
673 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
674 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
679 brw_set_dp_write_message(struct brw_codegen
*p
,
681 unsigned binding_table_index
,
682 unsigned msg_control
,
686 unsigned last_render_target
,
687 unsigned response_length
,
688 unsigned end_of_thread
,
689 unsigned send_commit_msg
)
691 const struct brw_device_info
*devinfo
= p
->devinfo
;
694 if (devinfo
->gen
>= 7) {
695 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
696 if (msg_type
== GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
)
697 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
699 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
700 } else if (devinfo
->gen
== 6) {
701 /* Use the render cache for all write messages. */
702 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
704 sfid
= BRW_SFID_DATAPORT_WRITE
;
707 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
708 header_present
, end_of_thread
);
710 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
711 brw_inst_set_dp_write_msg_type(devinfo
, insn
, msg_type
);
712 brw_inst_set_dp_write_msg_control(devinfo
, insn
, msg_control
);
713 brw_inst_set_rt_last(devinfo
, insn
, last_render_target
);
714 if (devinfo
->gen
< 7) {
715 brw_inst_set_dp_write_commit(devinfo
, insn
, send_commit_msg
);
720 brw_set_dp_read_message(struct brw_codegen
*p
,
722 unsigned binding_table_index
,
723 unsigned msg_control
,
725 unsigned target_cache
,
728 unsigned response_length
)
730 const struct brw_device_info
*devinfo
= p
->devinfo
;
733 if (devinfo
->gen
>= 7) {
734 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
735 } else if (devinfo
->gen
== 6) {
736 if (target_cache
== BRW_DATAPORT_READ_TARGET_RENDER_CACHE
)
737 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
739 sfid
= GEN6_SFID_DATAPORT_SAMPLER_CACHE
;
741 sfid
= BRW_SFID_DATAPORT_READ
;
744 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
745 header_present
, false);
747 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
748 brw_inst_set_dp_read_msg_type(devinfo
, insn
, msg_type
);
749 brw_inst_set_dp_read_msg_control(devinfo
, insn
, msg_control
);
750 if (devinfo
->gen
< 6)
751 brw_inst_set_dp_read_target_cache(devinfo
, insn
, target_cache
);
755 brw_set_sampler_message(struct brw_codegen
*p
,
757 unsigned binding_table_index
,
760 unsigned response_length
,
762 unsigned header_present
,
764 unsigned return_format
)
766 const struct brw_device_info
*devinfo
= p
->devinfo
;
768 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
769 response_length
, header_present
, false);
771 brw_inst_set_binding_table_index(devinfo
, inst
, binding_table_index
);
772 brw_inst_set_sampler(devinfo
, inst
, sampler
);
773 brw_inst_set_sampler_msg_type(devinfo
, inst
, msg_type
);
774 if (devinfo
->gen
>= 5) {
775 brw_inst_set_sampler_simd_mode(devinfo
, inst
, simd_mode
);
776 } else if (devinfo
->gen
== 4 && !devinfo
->is_g4x
) {
777 brw_inst_set_sampler_return_format(devinfo
, inst
, return_format
);
782 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
786 bool invalidate_after_read
,
788 unsigned addr_offset
,
793 const struct brw_device_info
*devinfo
= p
->devinfo
;
794 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
795 (devinfo
->gen
>= 8 && num_regs
== 8));
796 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
797 mlen
, rlen
, header_present
, false);
798 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
799 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
800 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
801 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
802 brw_inst_set_scratch_block_size(devinfo
, inst
, ffs(num_regs
) - 1);
803 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
806 #define next_insn brw_next_insn
808 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
810 const struct brw_device_info
*devinfo
= p
->devinfo
;
813 if (p
->nr_insn
+ 1 > p
->store_size
) {
815 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
818 p
->next_insn_offset
+= 16;
819 insn
= &p
->store
[p
->nr_insn
++];
820 memcpy(insn
, p
->current
, sizeof(*insn
));
822 brw_inst_set_opcode(devinfo
, insn
, opcode
);
827 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
828 struct brw_reg dest
, struct brw_reg src
)
830 brw_inst
*insn
= next_insn(p
, opcode
);
831 brw_set_dest(p
, insn
, dest
);
832 brw_set_src0(p
, insn
, src
);
837 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
838 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
840 brw_inst
*insn
= next_insn(p
, opcode
);
841 brw_set_dest(p
, insn
, dest
);
842 brw_set_src0(p
, insn
, src0
);
843 brw_set_src1(p
, insn
, src1
);
848 get_3src_subreg_nr(struct brw_reg reg
)
850 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
851 * use 32-bit units (components 0..7). Since they only support F/D/UD
852 * types, this doesn't lose any flexibility, but uses fewer bits.
854 return reg
.subnr
/ 4;
858 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
859 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
861 const struct brw_device_info
*devinfo
= p
->devinfo
;
862 brw_inst
*inst
= next_insn(p
, opcode
);
864 gen7_convert_mrf_to_grf(p
, &dest
);
866 assert(brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
);
868 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
869 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
870 assert(dest
.nr
< 128);
871 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
872 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
873 dest
.type
== BRW_REGISTER_TYPE_D
||
874 dest
.type
== BRW_REGISTER_TYPE_UD
);
875 if (devinfo
->gen
== 6) {
876 brw_inst_set_3src_dst_reg_file(devinfo
, inst
,
877 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
879 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
880 brw_inst_set_3src_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
881 brw_inst_set_3src_dst_writemask(devinfo
, inst
, dest
.writemask
);
883 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
884 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
885 assert(src0
.nr
< 128);
886 brw_inst_set_3src_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
887 brw_inst_set_3src_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
888 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
889 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
890 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
891 brw_inst_set_3src_src0_rep_ctrl(devinfo
, inst
,
892 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
894 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
895 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
896 assert(src1
.nr
< 128);
897 brw_inst_set_3src_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
898 brw_inst_set_3src_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
899 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
900 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
901 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
902 brw_inst_set_3src_src1_rep_ctrl(devinfo
, inst
,
903 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
905 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
906 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
907 assert(src2
.nr
< 128);
908 brw_inst_set_3src_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
909 brw_inst_set_3src_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
910 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
911 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
912 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
913 brw_inst_set_3src_src2_rep_ctrl(devinfo
, inst
,
914 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
916 if (devinfo
->gen
>= 7) {
917 /* Set both the source and destination types based on dest.type,
918 * ignoring the source register types. The MAD and LRP emitters ensure
919 * that all four types are float. The BFE and BFI2 emitters, however,
920 * may send us mixed D and UD types and want us to ignore that and use
921 * the destination type.
924 case BRW_REGISTER_TYPE_F
:
925 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
926 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
928 case BRW_REGISTER_TYPE_D
:
929 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
930 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
932 case BRW_REGISTER_TYPE_UD
:
933 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
934 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
937 unreachable("not reached");
945 /***********************************************************************
946 * Convenience routines.
949 brw_inst *brw_##OP(struct brw_codegen *p, \
950 struct brw_reg dest, \
951 struct brw_reg src0) \
953 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
957 brw_inst *brw_##OP(struct brw_codegen *p, \
958 struct brw_reg dest, \
959 struct brw_reg src0, \
960 struct brw_reg src1) \
962 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
966 brw_inst *brw_##OP(struct brw_codegen *p, \
967 struct brw_reg dest, \
968 struct brw_reg src0, \
969 struct brw_reg src1, \
970 struct brw_reg src2) \
972 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
976 brw_inst *brw_##OP(struct brw_codegen *p, \
977 struct brw_reg dest, \
978 struct brw_reg src0, \
979 struct brw_reg src1, \
980 struct brw_reg src2) \
982 assert(dest.type == BRW_REGISTER_TYPE_F); \
983 assert(src0.type == BRW_REGISTER_TYPE_F); \
984 assert(src1.type == BRW_REGISTER_TYPE_F); \
985 assert(src2.type == BRW_REGISTER_TYPE_F); \
986 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
989 /* Rounding operations (other than RNDD) require two instructions - the first
990 * stores a rounded value (possibly the wrong way) in the dest register, but
991 * also sets a per-channel "increment bit" in the flag register. A predicated
992 * add of 1.0 fixes dest to contain the desired result.
994 * Sandybridge and later appear to round correctly without an ADD.
997 void brw_##OP(struct brw_codegen *p, \
998 struct brw_reg dest, \
999 struct brw_reg src) \
1001 const struct brw_device_info *devinfo = p->devinfo; \
1002 brw_inst *rnd, *add; \
1003 rnd = next_insn(p, BRW_OPCODE_##OP); \
1004 brw_set_dest(p, rnd, dest); \
1005 brw_set_src0(p, rnd, src); \
1007 if (devinfo->gen < 6) { \
1008 /* turn on round-increments */ \
1009 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
1010 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1011 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1051 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1052 struct brw_reg src0
, struct brw_reg src1
)
1055 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1056 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1057 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1058 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1059 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1062 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1063 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1064 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1065 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1066 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1069 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1073 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1074 struct brw_reg src0
, struct brw_reg src1
)
1076 assert(dest
.type
== src0
.type
);
1077 assert(src0
.type
== src1
.type
);
1078 switch (src0
.type
) {
1079 case BRW_REGISTER_TYPE_B
:
1080 case BRW_REGISTER_TYPE_UB
:
1081 case BRW_REGISTER_TYPE_W
:
1082 case BRW_REGISTER_TYPE_UW
:
1083 case BRW_REGISTER_TYPE_D
:
1084 case BRW_REGISTER_TYPE_UD
:
1087 unreachable("Bad type for brw_AVG");
1090 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1094 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1095 struct brw_reg src0
, struct brw_reg src1
)
1098 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1099 src0
.type
== BRW_REGISTER_TYPE_UD
||
1100 src1
.type
== BRW_REGISTER_TYPE_D
||
1101 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1102 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1105 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1106 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1107 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1108 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1109 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1112 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1113 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1114 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1115 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1116 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1119 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1120 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1121 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1122 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1124 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1128 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1129 struct brw_reg src0
, struct brw_reg src1
)
1131 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1132 src0
.width
= BRW_WIDTH_1
;
1133 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1134 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1138 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1139 struct brw_reg src0
, struct brw_reg src1
)
1141 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1142 src0
.width
= BRW_WIDTH_1
;
1143 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1144 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1145 src1
.width
= BRW_WIDTH_8
;
1146 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1147 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1151 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1153 const struct brw_device_info
*devinfo
= p
->devinfo
;
1154 const bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1155 /* The F32TO16 instruction doesn't support 32-bit destination types in
1156 * Align1 mode, and neither does the Gen8 implementation in terms of a
1157 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1158 * an undocumented feature.
1160 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1161 (!align16
|| devinfo
->gen
>= 8));
1165 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1167 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1168 dst
.type
== BRW_REGISTER_TYPE_W
||
1169 dst
.type
== BRW_REGISTER_TYPE_UW
||
1170 dst
.type
== BRW_REGISTER_TYPE_HF
);
1173 brw_push_insn_state(p
);
1175 if (needs_zero_fill
) {
1176 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1177 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1180 if (devinfo
->gen
>= 8) {
1181 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1183 assert(devinfo
->gen
== 7);
1184 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1187 if (needs_zero_fill
) {
1188 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1189 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_ud(0u));
1190 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1193 brw_pop_insn_state(p
);
1198 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1200 const struct brw_device_info
*devinfo
= p
->devinfo
;
1201 bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1204 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1206 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1208 * Because this instruction does not have a 16-bit floating-point
1209 * type, the source data type must be Word (W). The destination type
1210 * must be F (Float).
1212 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1213 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1215 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1216 src
.type
== BRW_REGISTER_TYPE_UW
||
1217 src
.type
== BRW_REGISTER_TYPE_HF
);
1220 if (devinfo
->gen
>= 8) {
1221 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1223 assert(devinfo
->gen
== 7);
1224 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1229 void brw_NOP(struct brw_codegen
*p
)
1231 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1232 brw_set_dest(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1233 brw_set_src0(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1234 brw_set_src1(p
, insn
, brw_imm_ud(0x0));
1241 /***********************************************************************
1242 * Comparisons, if/else/endif
1246 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1247 unsigned predicate_control
)
1249 const struct brw_device_info
*devinfo
= p
->devinfo
;
1250 struct brw_reg ip
= brw_ip_reg();
1251 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1253 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_2
);
1254 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1255 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1256 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1262 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1264 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1266 p
->if_stack_depth
++;
1267 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1268 p
->if_stack_array_size
*= 2;
1269 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1270 p
->if_stack_array_size
);
1275 pop_if_stack(struct brw_codegen
*p
)
1277 p
->if_stack_depth
--;
1278 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1282 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1284 if (p
->loop_stack_array_size
< p
->loop_stack_depth
) {
1285 p
->loop_stack_array_size
*= 2;
1286 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1287 p
->loop_stack_array_size
);
1288 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1289 p
->loop_stack_array_size
);
1292 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1293 p
->loop_stack_depth
++;
1294 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1298 get_inner_do_insn(struct brw_codegen
*p
)
1300 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1303 /* EU takes the value from the flag register and pushes it onto some
1304 * sort of a stack (presumably merging with any flag value already on
1305 * the stack). Within an if block, the flags at the top of the stack
1306 * control execution on each channel of the unit, eg. on each of the
1307 * 16 pixel values in our wm programs.
1309 * When the matching 'else' instruction is reached (presumably by
1310 * countdown of the instruction count patched in by our ELSE/ENDIF
1311 * functions), the relevant flags are inverted.
1313 * When the matching 'endif' instruction is reached, the flags are
1314 * popped off. If the stack is now empty, normal execution resumes.
1317 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1319 const struct brw_device_info
*devinfo
= p
->devinfo
;
1322 insn
= next_insn(p
, BRW_OPCODE_IF
);
1324 /* Override the defaults for this instruction:
1326 if (devinfo
->gen
< 6) {
1327 brw_set_dest(p
, insn
, brw_ip_reg());
1328 brw_set_src0(p
, insn
, brw_ip_reg());
1329 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1330 } else if (devinfo
->gen
== 6) {
1331 brw_set_dest(p
, insn
, brw_imm_w(0));
1332 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1333 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1334 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1335 } else if (devinfo
->gen
== 7) {
1336 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1337 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1338 brw_set_src1(p
, insn
, brw_imm_w(0));
1339 brw_inst_set_jip(devinfo
, insn
, 0);
1340 brw_inst_set_uip(devinfo
, insn
, 0);
1342 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1343 brw_set_src0(p
, insn
, brw_imm_d(0));
1344 brw_inst_set_jip(devinfo
, insn
, 0);
1345 brw_inst_set_uip(devinfo
, insn
, 0);
1348 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1349 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1350 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1351 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1352 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1353 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1355 push_if_stack(p
, insn
);
1356 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1360 /* This function is only used for gen6-style IF instructions with an
1361 * embedded comparison (conditional modifier). It is not used on gen7.
1364 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1365 struct brw_reg src0
, struct brw_reg src1
)
1367 const struct brw_device_info
*devinfo
= p
->devinfo
;
1370 insn
= next_insn(p
, BRW_OPCODE_IF
);
1372 brw_set_dest(p
, insn
, brw_imm_w(0));
1373 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1375 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1376 brw_set_src0(p
, insn
, src0
);
1377 brw_set_src1(p
, insn
, src1
);
1379 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1380 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1381 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1383 push_if_stack(p
, insn
);
1388 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1391 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1392 brw_inst
*if_inst
, brw_inst
*else_inst
)
1394 const struct brw_device_info
*devinfo
= p
->devinfo
;
1396 /* The next instruction (where the ENDIF would be, if it existed) */
1397 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1399 assert(p
->single_program_flow
);
1400 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1401 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1402 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1404 /* Convert IF to an ADD instruction that moves the instruction pointer
1405 * to the first instruction of the ELSE block. If there is no ELSE
1406 * block, point to where ENDIF would be. Reverse the predicate.
1408 * There's no need to execute an ENDIF since we don't need to do any
1409 * stack operations, and if we're currently executing, we just want to
1410 * continue normally.
1412 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1413 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1415 if (else_inst
!= NULL
) {
1416 /* Convert ELSE to an ADD instruction that points where the ENDIF
1419 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1421 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1422 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1424 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1429 * Patch IF and ELSE instructions with appropriate jump targets.
1432 patch_IF_ELSE(struct brw_codegen
*p
,
1433 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1435 const struct brw_device_info
*devinfo
= p
->devinfo
;
1437 /* We shouldn't be patching IF and ELSE instructions in single program flow
1438 * mode when gen < 6, because in single program flow mode on those
1439 * platforms, we convert flow control instructions to conditional ADDs that
1440 * operate on IP (see brw_ENDIF).
1442 * However, on Gen6, writing to IP doesn't work in single program flow mode
1443 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1444 * not be updated by non-flow control instructions."). And on later
1445 * platforms, there is no significant benefit to converting control flow
1446 * instructions to conditional ADDs. So we do patch IF and ELSE
1447 * instructions in single program flow mode on those platforms.
1449 if (devinfo
->gen
< 6)
1450 assert(!p
->single_program_flow
);
1452 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1453 assert(endif_inst
!= NULL
);
1454 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1456 unsigned br
= brw_jump_scale(devinfo
);
1458 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1459 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1461 if (else_inst
== NULL
) {
1462 /* Patch IF -> ENDIF */
1463 if (devinfo
->gen
< 6) {
1464 /* Turn it into an IFF, which means no mask stack operations for
1465 * all-false and jumping past the ENDIF.
1467 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1468 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1469 br
* (endif_inst
- if_inst
+ 1));
1470 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1471 } else if (devinfo
->gen
== 6) {
1472 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1473 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1475 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1476 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1479 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1481 /* Patch IF -> ELSE */
1482 if (devinfo
->gen
< 6) {
1483 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1484 br
* (else_inst
- if_inst
));
1485 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1486 } else if (devinfo
->gen
== 6) {
1487 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1488 br
* (else_inst
- if_inst
+ 1));
1491 /* Patch ELSE -> ENDIF */
1492 if (devinfo
->gen
< 6) {
1493 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1496 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1497 br
* (endif_inst
- else_inst
+ 1));
1498 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1499 } else if (devinfo
->gen
== 6) {
1500 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1501 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1502 br
* (endif_inst
- else_inst
));
1504 /* The IF instruction's JIP should point just past the ELSE */
1505 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1506 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1507 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1508 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1509 if (devinfo
->gen
>= 8) {
1510 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1511 * should point to ENDIF.
1513 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1520 brw_ELSE(struct brw_codegen
*p
)
1522 const struct brw_device_info
*devinfo
= p
->devinfo
;
1525 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1527 if (devinfo
->gen
< 6) {
1528 brw_set_dest(p
, insn
, brw_ip_reg());
1529 brw_set_src0(p
, insn
, brw_ip_reg());
1530 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1531 } else if (devinfo
->gen
== 6) {
1532 brw_set_dest(p
, insn
, brw_imm_w(0));
1533 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1534 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1535 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1536 } else if (devinfo
->gen
== 7) {
1537 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1538 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1539 brw_set_src1(p
, insn
, brw_imm_w(0));
1540 brw_inst_set_jip(devinfo
, insn
, 0);
1541 brw_inst_set_uip(devinfo
, insn
, 0);
1543 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1544 brw_set_src0(p
, insn
, brw_imm_d(0));
1545 brw_inst_set_jip(devinfo
, insn
, 0);
1546 brw_inst_set_uip(devinfo
, insn
, 0);
1549 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1550 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1551 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1552 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1554 push_if_stack(p
, insn
);
1558 brw_ENDIF(struct brw_codegen
*p
)
1560 const struct brw_device_info
*devinfo
= p
->devinfo
;
1561 brw_inst
*insn
= NULL
;
1562 brw_inst
*else_inst
= NULL
;
1563 brw_inst
*if_inst
= NULL
;
1565 bool emit_endif
= true;
1567 /* In single program flow mode, we can express IF and ELSE instructions
1568 * equivalently as ADD instructions that operate on IP. On platforms prior
1569 * to Gen6, flow control instructions cause an implied thread switch, so
1570 * this is a significant savings.
1572 * However, on Gen6, writing to IP doesn't work in single program flow mode
1573 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1574 * not be updated by non-flow control instructions."). And on later
1575 * platforms, there is no significant benefit to converting control flow
1576 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1579 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1583 * A single next_insn() may change the base address of instruction store
1584 * memory(p->store), so call it first before referencing the instruction
1585 * store pointer from an index
1588 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1590 /* Pop the IF and (optional) ELSE instructions from the stack */
1591 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1592 tmp
= pop_if_stack(p
);
1593 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1595 tmp
= pop_if_stack(p
);
1600 /* ENDIF is useless; don't bother emitting it. */
1601 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1605 if (devinfo
->gen
< 6) {
1606 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1607 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1608 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1609 } else if (devinfo
->gen
== 6) {
1610 brw_set_dest(p
, insn
, brw_imm_w(0));
1611 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1612 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1613 } else if (devinfo
->gen
== 7) {
1614 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1615 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1616 brw_set_src1(p
, insn
, brw_imm_w(0));
1618 brw_set_src0(p
, insn
, brw_imm_d(0));
1621 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1622 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1623 if (devinfo
->gen
< 6)
1624 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1626 /* Also pop item off the stack in the endif instruction: */
1627 if (devinfo
->gen
< 6) {
1628 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1629 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1630 } else if (devinfo
->gen
== 6) {
1631 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1633 brw_inst_set_jip(devinfo
, insn
, 2);
1635 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1639 brw_BREAK(struct brw_codegen
*p
)
1641 const struct brw_device_info
*devinfo
= p
->devinfo
;
1644 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1645 if (devinfo
->gen
>= 8) {
1646 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1647 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1648 } else if (devinfo
->gen
>= 6) {
1649 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1650 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1651 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1653 brw_set_dest(p
, insn
, brw_ip_reg());
1654 brw_set_src0(p
, insn
, brw_ip_reg());
1655 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1656 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1657 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1659 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1660 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1667 brw_CONT(struct brw_codegen
*p
)
1669 const struct brw_device_info
*devinfo
= p
->devinfo
;
1672 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1673 brw_set_dest(p
, insn
, brw_ip_reg());
1674 if (devinfo
->gen
>= 8) {
1675 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1677 brw_set_src0(p
, insn
, brw_ip_reg());
1678 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1681 if (devinfo
->gen
< 6) {
1682 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1683 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1685 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1686 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1692 gen6_HALT(struct brw_codegen
*p
)
1694 const struct brw_device_info
*devinfo
= p
->devinfo
;
1697 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1698 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1699 if (devinfo
->gen
>= 8) {
1700 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1702 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1703 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1706 if (p
->compressed
) {
1707 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_16
);
1709 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1710 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_8
);
1717 * The DO/WHILE is just an unterminated loop -- break or continue are
1718 * used for control within the loop. We have a few ways they can be
1721 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1722 * jip and no DO instruction.
1724 * For non-uniform control flow pre-gen6, there's a DO instruction to
1725 * push the mask, and a WHILE to jump back, and BREAK to get out and
1728 * For gen6, there's no more mask stack, so no need for DO. WHILE
1729 * just points back to the first instruction of the loop.
1732 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1734 const struct brw_device_info
*devinfo
= p
->devinfo
;
1736 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1737 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1738 return &p
->store
[p
->nr_insn
];
1740 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1742 push_loop_stack(p
, insn
);
1744 /* Override the defaults for this instruction:
1746 brw_set_dest(p
, insn
, brw_null_reg());
1747 brw_set_src0(p
, insn
, brw_null_reg());
1748 brw_set_src1(p
, insn
, brw_null_reg());
1750 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1751 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1752 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1759 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1762 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1763 * nesting, since it can always just point to the end of the block/current loop.
1766 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1768 const struct brw_device_info
*devinfo
= p
->devinfo
;
1769 brw_inst
*do_inst
= get_inner_do_insn(p
);
1771 unsigned br
= brw_jump_scale(devinfo
);
1773 assert(devinfo
->gen
< 6);
1775 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1776 /* If the jump count is != 0, that means that this instruction has already
1777 * been patched because it's part of a loop inside of the one we're
1780 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1781 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1782 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1783 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1784 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1785 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1791 brw_WHILE(struct brw_codegen
*p
)
1793 const struct brw_device_info
*devinfo
= p
->devinfo
;
1794 brw_inst
*insn
, *do_insn
;
1795 unsigned br
= brw_jump_scale(devinfo
);
1797 if (devinfo
->gen
>= 6) {
1798 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1799 do_insn
= get_inner_do_insn(p
);
1801 if (devinfo
->gen
>= 8) {
1802 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1803 brw_set_src0(p
, insn
, brw_imm_d(0));
1804 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1805 } else if (devinfo
->gen
== 7) {
1806 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1807 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1808 brw_set_src1(p
, insn
, brw_imm_w(0));
1809 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1811 brw_set_dest(p
, insn
, brw_imm_w(0));
1812 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1813 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1814 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1817 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1820 if (p
->single_program_flow
) {
1821 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1822 do_insn
= get_inner_do_insn(p
);
1824 brw_set_dest(p
, insn
, brw_ip_reg());
1825 brw_set_src0(p
, insn
, brw_ip_reg());
1826 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1827 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1829 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1830 do_insn
= get_inner_do_insn(p
);
1832 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1834 brw_set_dest(p
, insn
, brw_ip_reg());
1835 brw_set_src0(p
, insn
, brw_ip_reg());
1836 brw_set_src1(p
, insn
, brw_imm_d(0));
1838 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1839 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1840 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1842 brw_patch_break_cont(p
, insn
);
1845 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1847 p
->loop_stack_depth
--;
1854 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1856 const struct brw_device_info
*devinfo
= p
->devinfo
;
1857 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1860 if (devinfo
->gen
>= 5)
1863 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1864 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1866 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1867 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1870 /* To integrate with the above, it makes sense that the comparison
1871 * instruction should populate the flag register. It might be simpler
1872 * just to use the flag reg for most WM tasks?
1874 void brw_CMP(struct brw_codegen
*p
,
1875 struct brw_reg dest
,
1876 unsigned conditional
,
1877 struct brw_reg src0
,
1878 struct brw_reg src1
)
1880 const struct brw_device_info
*devinfo
= p
->devinfo
;
1881 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1883 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1884 brw_set_dest(p
, insn
, dest
);
1885 brw_set_src0(p
, insn
, src0
);
1886 brw_set_src1(p
, insn
, src1
);
1888 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1890 * "Any CMP instruction with a null destination must use a {switch}."
1892 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1893 * mentioned on their work-arounds pages.
1895 if (devinfo
->gen
== 7) {
1896 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1897 dest
.nr
== BRW_ARF_NULL
) {
1898 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1903 /***********************************************************************
1904 * Helpers for the various SEND message types:
1907 /** Extended math function, float[8].
1909 void gen4_math(struct brw_codegen
*p
,
1910 struct brw_reg dest
,
1912 unsigned msg_reg_nr
,
1914 unsigned precision
)
1916 const struct brw_device_info
*devinfo
= p
->devinfo
;
1917 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1919 if (has_scalar_region(src
)) {
1920 data_type
= BRW_MATH_DATA_SCALAR
;
1922 data_type
= BRW_MATH_DATA_VECTOR
;
1925 assert(devinfo
->gen
< 6);
1927 /* Example code doesn't set predicate_control for send
1930 brw_inst_set_pred_control(devinfo
, insn
, 0);
1931 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1933 brw_set_dest(p
, insn
, dest
);
1934 brw_set_src0(p
, insn
, src
);
1935 brw_set_math_message(p
,
1938 src
.type
== BRW_REGISTER_TYPE_D
,
1943 void gen6_math(struct brw_codegen
*p
,
1944 struct brw_reg dest
,
1946 struct brw_reg src0
,
1947 struct brw_reg src1
)
1949 const struct brw_device_info
*devinfo
= p
->devinfo
;
1950 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1952 assert(devinfo
->gen
>= 6);
1954 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1955 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1956 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
1957 (devinfo
->gen
>= 8 && src0
.file
== BRW_IMMEDIATE_VALUE
));
1959 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1960 if (devinfo
->gen
== 6) {
1961 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1962 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1965 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1966 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1967 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1968 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1969 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1970 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1971 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1973 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1974 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1975 if (function
== BRW_MATH_FUNCTION_POW
) {
1976 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1977 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1979 assert(src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1980 src1
.nr
== BRW_ARF_NULL
);
1984 /* Source modifiers are ignored for extended math instructions on Gen6. */
1985 if (devinfo
->gen
== 6) {
1986 assert(!src0
.negate
);
1988 assert(!src1
.negate
);
1992 brw_inst_set_math_function(devinfo
, insn
, function
);
1994 brw_set_dest(p
, insn
, dest
);
1995 brw_set_src0(p
, insn
, src0
);
1996 brw_set_src1(p
, insn
, src1
);
2000 * Return the right surface index to access the thread scratch space using
2001 * stateless dataport messages.
2004 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2006 /* The scratch space is thread-local so IA coherency is unnecessary. */
2007 if (p
->devinfo
->gen
>= 8)
2008 return GEN8_BTI_STATELESS_NON_COHERENT
;
2010 return BRW_BTI_STATELESS
;
2014 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2015 * using a constant offset per channel.
2017 * The offset must be aligned to oword size (16 bytes). Used for
2018 * register spilling.
2020 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2025 const struct brw_device_info
*devinfo
= p
->devinfo
;
2026 uint32_t msg_control
, msg_type
;
2029 if (devinfo
->gen
>= 6)
2032 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2034 if (num_regs
== 1) {
2035 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2038 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2042 /* Set up the message header. This is g0, with g0.2 filled with
2043 * the offset. We don't want to leave our offset around in g0 or
2044 * it'll screw up texture samples, so set it up inside the message
2048 brw_push_insn_state(p
);
2049 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2050 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2051 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2053 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2055 /* set message header global offset field (reg 0, element 2) */
2057 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2059 2), BRW_REGISTER_TYPE_UD
),
2060 brw_imm_ud(offset
));
2062 brw_pop_insn_state(p
);
2066 struct brw_reg dest
;
2067 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2068 int send_commit_msg
;
2069 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2070 BRW_REGISTER_TYPE_UW
);
2072 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_NONE
) {
2073 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2074 src_header
= vec16(src_header
);
2076 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2077 if (devinfo
->gen
< 6)
2078 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2080 /* Until gen6, writes followed by reads from the same location
2081 * are not guaranteed to be ordered unless write_commit is set.
2082 * If set, then a no-op write is issued to the destination
2083 * register to set a dependency, and a read from the destination
2084 * can be used to ensure the ordering.
2086 * For gen6, only writes between different threads need ordering
2087 * protection. Our use of DP writes is all about register
2088 * spilling within a thread.
2090 if (devinfo
->gen
>= 6) {
2091 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2092 send_commit_msg
= 0;
2095 send_commit_msg
= 1;
2098 brw_set_dest(p
, insn
, dest
);
2099 if (devinfo
->gen
>= 6) {
2100 brw_set_src0(p
, insn
, mrf
);
2102 brw_set_src0(p
, insn
, brw_null_reg());
2105 if (devinfo
->gen
>= 6)
2106 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2108 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2110 brw_set_dp_write_message(p
,
2112 brw_scratch_surface_idx(p
),
2116 true, /* header_present */
2117 0, /* not a render target */
2118 send_commit_msg
, /* response_length */
2126 * Read a block of owords (half a GRF each) from the scratch buffer
2127 * using a constant index per channel.
2129 * Offset must be aligned to oword size (16 bytes). Used for register
2133 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2134 struct brw_reg dest
,
2139 const struct brw_device_info
*devinfo
= p
->devinfo
;
2140 uint32_t msg_control
;
2143 if (devinfo
->gen
>= 6)
2146 if (p
->devinfo
->gen
>= 7) {
2147 /* On gen 7 and above, we no longer have message registers and we can
2148 * send from any register we want. By using the destination register
2149 * for the message, we guarantee that the implied message write won't
2150 * accidentally overwrite anything. This has been a problem because
2151 * the MRF registers and source for the final FB write are both fixed
2154 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2156 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2158 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2160 if (num_regs
== 1) {
2161 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2164 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2169 brw_push_insn_state(p
);
2170 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2171 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2172 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2174 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2176 /* set message header global offset field (reg 0, element 2) */
2177 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2179 brw_pop_insn_state(p
);
2183 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2185 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2186 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2188 brw_set_dest(p
, insn
, dest
); /* UW? */
2189 if (devinfo
->gen
>= 6) {
2190 brw_set_src0(p
, insn
, mrf
);
2192 brw_set_src0(p
, insn
, brw_null_reg());
2193 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2196 brw_set_dp_read_message(p
,
2198 brw_scratch_surface_idx(p
),
2200 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2201 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
,
2203 true, /* header_present */
2209 gen7_block_read_scratch(struct brw_codegen
*p
,
2210 struct brw_reg dest
,
2214 const struct brw_device_info
*devinfo
= p
->devinfo
;
2215 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2216 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2218 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2219 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2221 /* The HW requires that the header is present; this is to get the g0.5
2224 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2226 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2227 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2228 * is 32 bytes, which happens to be the size of a register.
2231 assert(offset
< (1 << 12));
2233 gen7_set_dp_scratch_message(p
, insn
,
2234 false, /* scratch read */
2236 false, /* invalidate after read */
2239 1, /* mlen: just g0 */
2240 num_regs
, /* rlen */
2241 true); /* header present */
2245 * Read a float[4] vector from the data port Data Cache (const buffer).
2246 * Location (in buffer) should be a multiple of 16.
2247 * Used for fetching shader constants.
2249 void brw_oword_block_read(struct brw_codegen
*p
,
2250 struct brw_reg dest
,
2253 uint32_t bind_table_index
)
2255 const struct brw_device_info
*devinfo
= p
->devinfo
;
2257 /* On newer hardware, offset is in units of owords. */
2258 if (devinfo
->gen
>= 6)
2261 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2263 brw_push_insn_state(p
);
2264 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2265 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2266 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2267 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2269 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2271 /* set message header global offset field (reg 0, element 2) */
2273 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2275 2), BRW_REGISTER_TYPE_UD
),
2276 brw_imm_ud(offset
));
2278 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2280 /* cast dest to a uword[8] vector */
2281 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2283 brw_set_dest(p
, insn
, dest
);
2284 if (devinfo
->gen
>= 6) {
2285 brw_set_src0(p
, insn
, mrf
);
2287 brw_set_src0(p
, insn
, brw_null_reg());
2288 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2291 brw_set_dp_read_message(p
,
2294 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW
,
2295 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2296 BRW_DATAPORT_READ_TARGET_DATA_CACHE
,
2298 true, /* header_present */
2299 1); /* response_length (1 reg, 2 owords!) */
2301 brw_pop_insn_state(p
);
2305 void brw_fb_WRITE(struct brw_codegen
*p
,
2307 struct brw_reg payload
,
2308 struct brw_reg implied_header
,
2309 unsigned msg_control
,
2310 unsigned binding_table_index
,
2311 unsigned msg_length
,
2312 unsigned response_length
,
2314 bool last_render_target
,
2315 bool header_present
)
2317 const struct brw_device_info
*devinfo
= p
->devinfo
;
2320 struct brw_reg dest
, src0
;
2322 if (dispatch_width
== 16)
2323 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2325 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2327 if (devinfo
->gen
>= 6) {
2328 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2330 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2332 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2334 if (devinfo
->gen
>= 6) {
2335 /* headerless version, just submit color payload */
2338 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2340 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2341 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2342 src0
= implied_header
;
2344 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2347 brw_set_dest(p
, insn
, dest
);
2348 brw_set_src0(p
, insn
, src0
);
2349 brw_set_dp_write_message(p
,
2351 binding_table_index
,
2359 0 /* send_commit_msg */);
2364 * Texture sample instruction.
2365 * Note: the msg_type plus msg_length values determine exactly what kind
2366 * of sampling operation is performed. See volume 4, page 161 of docs.
2368 void brw_SAMPLE(struct brw_codegen
*p
,
2369 struct brw_reg dest
,
2370 unsigned msg_reg_nr
,
2371 struct brw_reg src0
,
2372 unsigned binding_table_index
,
2375 unsigned response_length
,
2376 unsigned msg_length
,
2377 unsigned header_present
,
2379 unsigned return_format
)
2381 const struct brw_device_info
*devinfo
= p
->devinfo
;
2384 if (msg_reg_nr
!= -1)
2385 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2387 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2388 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2390 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2392 * "Instruction compression is not allowed for this instruction (that
2393 * is, send). The hardware behavior is undefined if this instruction is
2394 * set as compressed. However, compress control can be set to "SecHalf"
2395 * to affect the EMask generation."
2397 * No similar wording is found in later PRMs, but there are examples
2398 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2399 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2400 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2402 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_2NDHALF
)
2403 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2405 if (devinfo
->gen
< 6)
2406 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2408 brw_set_dest(p
, insn
, dest
);
2409 brw_set_src0(p
, insn
, src0
);
2410 brw_set_sampler_message(p
, insn
,
2411 binding_table_index
,
2421 /* Adjust the message header's sampler state pointer to
2422 * select the correct group of 16 samplers.
2424 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2425 struct brw_reg header
,
2426 struct brw_reg sampler_index
)
2428 /* The "Sampler Index" field can only store values between 0 and 15.
2429 * However, we can add an offset to the "Sampler State Pointer"
2430 * field, effectively selecting a different set of 16 samplers.
2432 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2433 * offset, and each sampler state is only 16-bytes, so we can't
2434 * exclusively use the offset - we have to use both.
2437 const struct brw_device_info
*devinfo
= p
->devinfo
;
2439 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2440 const int sampler_state_size
= 16; /* 16 bytes */
2441 uint32_t sampler
= sampler_index
.ud
;
2443 if (sampler
>= 16) {
2444 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2446 get_element_ud(header
, 3),
2447 get_element_ud(brw_vec8_grf(0, 0), 3),
2448 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2451 /* Non-const sampler array indexing case */
2452 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2456 struct brw_reg temp
= get_element_ud(header
, 3);
2458 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2459 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2461 get_element_ud(header
, 3),
2462 get_element_ud(brw_vec8_grf(0, 0), 3),
2467 /* All these variables are pretty confusing - we might be better off
2468 * using bitmasks and macros for this, in the old style. Or perhaps
2469 * just having the caller instantiate the fields in dword3 itself.
2471 void brw_urb_WRITE(struct brw_codegen
*p
,
2472 struct brw_reg dest
,
2473 unsigned msg_reg_nr
,
2474 struct brw_reg src0
,
2475 enum brw_urb_write_flags flags
,
2476 unsigned msg_length
,
2477 unsigned response_length
,
2481 const struct brw_device_info
*devinfo
= p
->devinfo
;
2484 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2486 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2487 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2488 brw_push_insn_state(p
);
2489 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2490 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2491 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2492 BRW_REGISTER_TYPE_UD
),
2493 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2494 brw_imm_ud(0xff00));
2495 brw_pop_insn_state(p
);
2498 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2500 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2502 brw_set_dest(p
, insn
, dest
);
2503 brw_set_src0(p
, insn
, src0
);
2504 brw_set_src1(p
, insn
, brw_imm_d(0));
2506 if (devinfo
->gen
< 6)
2507 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2509 brw_set_urb_message(p
,
2519 brw_send_indirect_message(struct brw_codegen
*p
,
2522 struct brw_reg payload
,
2523 struct brw_reg desc
)
2525 const struct brw_device_info
*devinfo
= p
->devinfo
;
2526 struct brw_inst
*send
;
2529 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2531 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2533 /* We hold on to the setup instruction (the SEND in the direct case, the OR
2534 * in the indirect case) by its index in the instruction store. The
2535 * pointer returned by next_insn() may become invalid if emitting the SEND
2536 * in the indirect case reallocs the store.
2539 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2541 send
= next_insn(p
, BRW_OPCODE_SEND
);
2542 brw_set_src1(p
, send
, desc
);
2545 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2547 brw_push_insn_state(p
);
2548 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2549 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2550 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2552 /* Load the indirect descriptor to an address register using OR so the
2553 * caller can specify additional descriptor bits with the usual
2554 * brw_set_*_message() helper functions.
2557 brw_OR(p
, addr
, desc
, brw_imm_ud(0));
2559 brw_pop_insn_state(p
);
2561 send
= next_insn(p
, BRW_OPCODE_SEND
);
2562 brw_set_src1(p
, send
, addr
);
2565 brw_set_dest(p
, send
, dst
);
2566 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2567 brw_inst_set_sfid(devinfo
, send
, sfid
);
2569 return &p
->store
[setup
];
2572 static struct brw_inst
*
2573 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2576 struct brw_reg payload
,
2577 struct brw_reg surface
,
2578 unsigned message_len
,
2579 unsigned response_len
,
2580 bool header_present
)
2582 const struct brw_device_info
*devinfo
= p
->devinfo
;
2583 struct brw_inst
*insn
;
2585 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2586 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2588 brw_push_insn_state(p
);
2589 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2590 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2591 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2593 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2594 * some surface array is accessed out of bounds.
2596 insn
= brw_AND(p
, addr
,
2597 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2598 BRW_GET_SWZ(surface
.swizzle
, 0)),
2601 brw_pop_insn_state(p
);
2606 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
);
2607 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2608 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2609 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2615 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2618 void *store
= p
->store
;
2619 const struct brw_device_info
*devinfo
= p
->devinfo
;
2623 for (offset
= next_offset(devinfo
, store
, start_offset
);
2624 offset
< p
->next_insn_offset
;
2625 offset
= next_offset(devinfo
, store
, offset
)) {
2626 brw_inst
*insn
= store
+ offset
;
2628 switch (brw_inst_opcode(devinfo
, insn
)) {
2632 case BRW_OPCODE_ENDIF
:
2637 case BRW_OPCODE_ELSE
:
2638 case BRW_OPCODE_WHILE
:
2639 case BRW_OPCODE_HALT
:
2648 /* There is no DO instruction on gen6, so to find the end of the loop
2649 * we have to see if the loop is jumping back before our start
2653 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2655 const struct brw_device_info
*devinfo
= p
->devinfo
;
2657 int scale
= 16 / brw_jump_scale(devinfo
);
2658 void *store
= p
->store
;
2660 assert(devinfo
->gen
>= 6);
2662 /* Always start after the instruction (such as a WHILE) we're trying to fix
2665 for (offset
= next_offset(devinfo
, store
, start_offset
);
2666 offset
< p
->next_insn_offset
;
2667 offset
= next_offset(devinfo
, store
, offset
)) {
2668 brw_inst
*insn
= store
+ offset
;
2670 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2671 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2672 : brw_inst_jip(devinfo
, insn
);
2673 if (offset
+ jip
* scale
<= start_offset
)
2677 assert(!"not reached");
2678 return start_offset
;
2681 /* After program generation, go back and update the UIP and JIP of
2682 * BREAK, CONT, and HALT instructions to their correct locations.
2685 brw_set_uip_jip(struct brw_codegen
*p
)
2687 const struct brw_device_info
*devinfo
= p
->devinfo
;
2689 int br
= brw_jump_scale(devinfo
);
2690 int scale
= 16 / br
;
2691 void *store
= p
->store
;
2693 if (devinfo
->gen
< 6)
2696 for (offset
= 0; offset
< p
->next_insn_offset
;
2697 offset
= next_offset(devinfo
, store
, offset
)) {
2698 brw_inst
*insn
= store
+ offset
;
2700 if (brw_inst_cmpt_control(devinfo
, insn
)) {
2701 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2702 assert(brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_BREAK
&&
2703 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_CONTINUE
&&
2704 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_HALT
);
2708 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2709 switch (brw_inst_opcode(devinfo
, insn
)) {
2710 case BRW_OPCODE_BREAK
:
2711 assert(block_end_offset
!= 0);
2712 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2713 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2714 brw_inst_set_uip(devinfo
, insn
,
2715 (brw_find_loop_end(p
, offset
) - offset
+
2716 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2718 case BRW_OPCODE_CONTINUE
:
2719 assert(block_end_offset
!= 0);
2720 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2721 brw_inst_set_uip(devinfo
, insn
,
2722 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2724 assert(brw_inst_uip(devinfo
, insn
) != 0);
2725 assert(brw_inst_jip(devinfo
, insn
) != 0);
2728 case BRW_OPCODE_ENDIF
: {
2729 int32_t jump
= (block_end_offset
== 0) ?
2730 1 * br
: (block_end_offset
- offset
) / scale
;
2731 if (devinfo
->gen
>= 7)
2732 brw_inst_set_jip(devinfo
, insn
, jump
);
2734 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2738 case BRW_OPCODE_HALT
:
2739 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2741 * "In case of the halt instruction not inside any conditional
2742 * code block, the value of <JIP> and <UIP> should be the
2743 * same. In case of the halt instruction inside conditional code
2744 * block, the <UIP> should be the end of the program, and the
2745 * <JIP> should be end of the most inner conditional code block."
2747 * The uip will have already been set by whoever set up the
2750 if (block_end_offset
== 0) {
2751 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2753 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2755 assert(brw_inst_uip(devinfo
, insn
) != 0);
2756 assert(brw_inst_jip(devinfo
, insn
) != 0);
2762 void brw_ff_sync(struct brw_codegen
*p
,
2763 struct brw_reg dest
,
2764 unsigned msg_reg_nr
,
2765 struct brw_reg src0
,
2767 unsigned response_length
,
2770 const struct brw_device_info
*devinfo
= p
->devinfo
;
2773 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2775 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2776 brw_set_dest(p
, insn
, dest
);
2777 brw_set_src0(p
, insn
, src0
);
2778 brw_set_src1(p
, insn
, brw_imm_d(0));
2780 if (devinfo
->gen
< 6)
2781 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2783 brw_set_ff_sync_message(p
,
2791 * Emit the SEND instruction necessary to generate stream output data on Gen6
2792 * (for transform feedback).
2794 * If send_commit_msg is true, this is the last piece of stream output data
2795 * from this thread, so send the data as a committed write. According to the
2796 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2798 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2799 * writes are complete by sending the final write as a committed write."
2802 brw_svb_write(struct brw_codegen
*p
,
2803 struct brw_reg dest
,
2804 unsigned msg_reg_nr
,
2805 struct brw_reg src0
,
2806 unsigned binding_table_index
,
2807 bool send_commit_msg
)
2811 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2813 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2814 brw_set_dest(p
, insn
, dest
);
2815 brw_set_src0(p
, insn
, src0
);
2816 brw_set_src1(p
, insn
, brw_imm_d(0));
2817 brw_set_dp_write_message(p
, insn
,
2818 binding_table_index
,
2819 0, /* msg_control: ignored */
2820 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2822 true, /* header_present */
2823 0, /* last_render_target: ignored */
2824 send_commit_msg
, /* response_length */
2825 0, /* end_of_thread */
2826 send_commit_msg
); /* send_commit_msg */
2830 brw_surface_payload_size(struct brw_codegen
*p
,
2831 unsigned num_channels
,
2835 if (has_simd4x2
&& brw_inst_access_mode(p
->devinfo
, p
->current
) == BRW_ALIGN_16
)
2837 else if (has_simd16
&& p
->compressed
)
2838 return 2 * num_channels
;
2840 return num_channels
;
2844 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2847 bool response_expected
)
2849 const struct brw_device_info
*devinfo
= p
->devinfo
;
2850 unsigned msg_control
=
2851 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2852 (response_expected
? 1 << 5 : 0); /* Return data expected */
2854 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2855 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2857 msg_control
|= 1 << 4; /* SIMD8 mode */
2859 brw_inst_set_dp_msg_type(devinfo
, insn
,
2860 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2862 brw_inst_set_dp_msg_type(devinfo
, insn
,
2863 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2866 brw_inst_set_dp_msg_type(devinfo
, insn
,
2867 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2870 msg_control
|= 1 << 4; /* SIMD8 mode */
2873 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2877 brw_untyped_atomic(struct brw_codegen
*p
,
2879 struct brw_reg payload
,
2880 struct brw_reg surface
,
2882 unsigned msg_length
,
2883 bool response_expected
)
2885 const struct brw_device_info
*devinfo
= p
->devinfo
;
2886 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2887 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2888 GEN7_SFID_DATAPORT_DATA_CACHE
);
2889 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2890 /* Mask out unused components -- This is especially important in Align16
2891 * mode on generations that don't have native support for SIMD4x2 atomics,
2892 * because unused but enabled components will cause the dataport to perform
2893 * additional atomic operations on the addresses that happen to be in the
2894 * uninitialized Y, Z and W coordinates of the payload.
2896 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2897 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2898 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2899 brw_surface_payload_size(p
, response_expected
,
2900 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2903 brw_set_dp_untyped_atomic_message(
2904 p
, insn
, atomic_op
, response_expected
);
2908 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
2909 struct brw_inst
*insn
,
2910 unsigned num_channels
)
2912 const struct brw_device_info
*devinfo
= p
->devinfo
;
2913 /* Set mask of 32-bit channels to drop. */
2914 unsigned msg_control
= 0xf & (0xf << num_channels
);
2916 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2918 msg_control
|= 1 << 4; /* SIMD16 mode */
2920 msg_control
|= 2 << 4; /* SIMD8 mode */
2923 brw_inst_set_dp_msg_type(devinfo
, insn
,
2924 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2925 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2926 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
2927 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2931 brw_untyped_surface_read(struct brw_codegen
*p
,
2933 struct brw_reg payload
,
2934 struct brw_reg surface
,
2935 unsigned msg_length
,
2936 unsigned num_channels
)
2938 const struct brw_device_info
*devinfo
= p
->devinfo
;
2939 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2940 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2941 GEN7_SFID_DATAPORT_DATA_CACHE
);
2942 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2943 p
, sfid
, dst
, payload
, surface
, msg_length
,
2944 brw_surface_payload_size(p
, num_channels
, true, true),
2947 brw_set_dp_untyped_surface_read_message(
2948 p
, insn
, num_channels
);
2952 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
2953 struct brw_inst
*insn
,
2954 unsigned num_channels
)
2956 const struct brw_device_info
*devinfo
= p
->devinfo
;
2957 /* Set mask of 32-bit channels to drop. */
2958 unsigned msg_control
= 0xf & (0xf << num_channels
);
2960 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2962 msg_control
|= 1 << 4; /* SIMD16 mode */
2964 msg_control
|= 2 << 4; /* SIMD8 mode */
2966 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
2967 msg_control
|= 0 << 4; /* SIMD4x2 mode */
2969 msg_control
|= 2 << 4; /* SIMD8 mode */
2972 brw_inst_set_dp_msg_type(devinfo
, insn
,
2973 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2974 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
2975 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
2976 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2980 brw_untyped_surface_write(struct brw_codegen
*p
,
2981 struct brw_reg payload
,
2982 struct brw_reg surface
,
2983 unsigned msg_length
,
2984 unsigned num_channels
)
2986 const struct brw_device_info
*devinfo
= p
->devinfo
;
2987 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2988 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2989 GEN7_SFID_DATAPORT_DATA_CACHE
);
2990 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2991 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2992 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
2993 WRITEMASK_X
: WRITEMASK_XYZW
;
2994 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2995 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2996 payload
, surface
, msg_length
, 0, align1
);
2998 brw_set_dp_untyped_surface_write_message(
2999 p
, insn
, num_channels
);
3003 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
3004 struct brw_inst
*insn
,
3006 bool response_expected
)
3008 const struct brw_device_info
*devinfo
= p
->devinfo
;
3009 unsigned msg_control
=
3010 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3011 (response_expected
? 1 << 5 : 0); /* Return data expected */
3013 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3014 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3015 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3016 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3018 brw_inst_set_dp_msg_type(devinfo
, insn
,
3019 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
3021 brw_inst_set_dp_msg_type(devinfo
, insn
,
3022 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
3026 brw_inst_set_dp_msg_type(devinfo
, insn
,
3027 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
3029 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3030 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3033 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3037 brw_typed_atomic(struct brw_codegen
*p
,
3039 struct brw_reg payload
,
3040 struct brw_reg surface
,
3042 unsigned msg_length
,
3043 bool response_expected
) {
3044 const struct brw_device_info
*devinfo
= p
->devinfo
;
3045 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3046 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3047 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3048 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3049 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3050 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3051 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3052 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
3053 brw_surface_payload_size(p
, response_expected
,
3054 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3057 brw_set_dp_typed_atomic_message(
3058 p
, insn
, atomic_op
, response_expected
);
3062 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3063 struct brw_inst
*insn
,
3064 unsigned num_channels
)
3066 const struct brw_device_info
*devinfo
= p
->devinfo
;
3067 /* Set mask of unused channels. */
3068 unsigned msg_control
= 0xf & (0xf << num_channels
);
3070 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3071 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3072 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3073 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3075 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3078 brw_inst_set_dp_msg_type(devinfo
, insn
,
3079 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3081 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3082 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3083 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3086 brw_inst_set_dp_msg_type(devinfo
, insn
,
3087 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3090 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3094 brw_typed_surface_read(struct brw_codegen
*p
,
3096 struct brw_reg payload
,
3097 struct brw_reg surface
,
3098 unsigned msg_length
,
3099 unsigned num_channels
)
3101 const struct brw_device_info
*devinfo
= p
->devinfo
;
3102 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3103 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3104 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3105 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3106 p
, sfid
, dst
, payload
, surface
, msg_length
,
3107 brw_surface_payload_size(p
, num_channels
,
3108 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3111 brw_set_dp_typed_surface_read_message(
3112 p
, insn
, num_channels
);
3116 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3117 struct brw_inst
*insn
,
3118 unsigned num_channels
)
3120 const struct brw_device_info
*devinfo
= p
->devinfo
;
3121 /* Set mask of unused channels. */
3122 unsigned msg_control
= 0xf & (0xf << num_channels
);
3124 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3125 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3126 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3127 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3129 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3132 brw_inst_set_dp_msg_type(devinfo
, insn
,
3133 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3136 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3137 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3138 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3141 brw_inst_set_dp_msg_type(devinfo
, insn
,
3142 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3145 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3149 brw_typed_surface_write(struct brw_codegen
*p
,
3150 struct brw_reg payload
,
3151 struct brw_reg surface
,
3152 unsigned msg_length
,
3153 unsigned num_channels
)
3155 const struct brw_device_info
*devinfo
= p
->devinfo
;
3156 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3157 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3158 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3159 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3160 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3161 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3162 WRITEMASK_X
: WRITEMASK_XYZW
);
3163 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3164 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3165 payload
, surface
, msg_length
, 0, true);
3167 brw_set_dp_typed_surface_write_message(
3168 p
, insn
, num_channels
);
3172 brw_set_memory_fence_message(struct brw_codegen
*p
,
3173 struct brw_inst
*insn
,
3174 enum brw_message_target sfid
,
3177 const struct brw_device_info
*devinfo
= p
->devinfo
;
3179 brw_set_message_descriptor(p
, insn
, sfid
,
3180 1 /* message length */,
3181 (commit_enable
? 1 : 0) /* response length */,
3182 true /* header present */,
3186 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3187 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3189 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3190 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3193 unreachable("Not reached");
3197 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3201 brw_memory_fence(struct brw_codegen
*p
,
3204 const struct brw_device_info
*devinfo
= p
->devinfo
;
3205 const bool commit_enable
= devinfo
->gen
== 7 && !devinfo
->is_haswell
;
3206 struct brw_inst
*insn
;
3208 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3209 * message doesn't write anything back.
3211 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3212 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3213 brw_set_dest(p
, insn
, dst
);
3214 brw_set_src0(p
, insn
, dst
);
3215 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3218 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3219 /* IVB does typed surface access through the render cache, so we need to
3220 * flush it too. Use a different register so both flushes can be
3221 * pipelined by the hardware.
3223 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3224 brw_set_dest(p
, insn
, offset(dst
, 1));
3225 brw_set_src0(p
, insn
, offset(dst
, 1));
3226 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3229 /* Now write the response of the second message into the response of the
3230 * first to trigger a pipeline stall -- This way future render and data
3231 * cache messages will be properly ordered with respect to past data and
3232 * render cache messages.
3234 brw_push_insn_state(p
);
3235 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3236 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3237 brw_MOV(p
, dst
, offset(dst
, 1));
3238 brw_pop_insn_state(p
);
3243 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3244 struct brw_reg dest
,
3248 struct brw_reg data
,
3249 unsigned msg_length
,
3250 unsigned response_length
)
3252 const struct brw_device_info
*devinfo
= p
->devinfo
;
3253 struct brw_inst
*insn
;
3254 const uint16_t exec_size
= brw_inst_exec_size(devinfo
, p
->current
);
3256 /* brw_send_indirect_message will automatically use a direct send message
3257 * if data is actually immediate.
3259 insn
= brw_send_indirect_message(p
,
3260 GEN7_SFID_PIXEL_INTERPOLATOR
,
3264 brw_inst_set_mlen(devinfo
, insn
, msg_length
);
3265 brw_inst_set_rlen(devinfo
, insn
, response_length
);
3267 brw_inst_set_pi_simd_mode(devinfo
, insn
, exec_size
== BRW_EXECUTE_16
);
3268 brw_inst_set_pi_slot_group(devinfo
, insn
, 0); /* zero unless 32/64px dispatch */
3269 brw_inst_set_pi_nopersp(devinfo
, insn
, noperspective
);
3270 brw_inst_set_pi_message_type(devinfo
, insn
, mode
);
3274 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
)
3276 const struct brw_device_info
*devinfo
= p
->devinfo
;
3279 assert(devinfo
->gen
>= 7);
3281 brw_push_insn_state(p
);
3283 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3284 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3286 if (devinfo
->gen
>= 8) {
3287 /* Getting the first active channel index is easy on Gen8: Just find
3288 * the first bit set in the mask register. The same register exists
3289 * on HSW already but it reads back as all ones when the current
3290 * instruction has execution masking disabled, so it's kind of
3293 inst
= brw_FBL(p
, vec1(dst
),
3294 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
));
3296 /* Quarter control has the effect of magically shifting the value of
3297 * this register. Make sure it's set to zero.
3299 brw_inst_set_qtr_control(devinfo
, inst
, GEN6_COMPRESSION_1Q
);
3301 const struct brw_reg flag
= retype(brw_flag_reg(1, 0),
3302 BRW_REGISTER_TYPE_UD
);
3304 brw_MOV(p
, flag
, brw_imm_ud(0));
3306 /* Run a 16-wide instruction returning zero with execution masking
3307 * and a conditional modifier enabled in order to get the current
3308 * execution mask in f1.0.
3310 inst
= brw_MOV(p
, brw_null_reg(), brw_imm_ud(0));
3311 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_16
);
3312 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3313 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3314 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3316 brw_FBL(p
, vec1(dst
), flag
);
3319 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3321 if (devinfo
->gen
>= 8) {
3322 /* In SIMD4x2 mode the first active channel index is just the
3323 * negation of the first bit of the mask register.
3325 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3326 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3330 /* Overwrite the destination without and with execution masking to
3331 * find out which of the channels is active.
3333 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3336 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3338 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3342 brw_pop_insn_state(p
);
3346 brw_broadcast(struct brw_codegen
*p
,
3351 const struct brw_device_info
*devinfo
= p
->devinfo
;
3352 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3355 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3356 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3358 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3359 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3360 /* Trivial, the source is already uniform or the index is a constant.
3361 * We will typically not get here if the optimizer is doing its job, but
3362 * asserting would be mean.
3364 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3366 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3367 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3370 const struct brw_reg addr
=
3371 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3372 const unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3373 /* Limit in bytes of the signed indirect addressing immediate. */
3374 const unsigned limit
= 512;
3376 brw_push_insn_state(p
);
3377 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3378 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3380 /* Take into account the component size and horizontal stride. */
3381 assert(src
.vstride
== src
.hstride
+ src
.width
);
3382 brw_SHL(p
, addr
, vec1(idx
),
3383 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3386 /* We can only address up to limit bytes using the indirect
3387 * addressing immediate, account for the difference if the source
3388 * register is above this limit.
3390 if (offset
>= limit
)
3391 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3393 brw_pop_insn_state(p
);
3395 /* Use indirect addressing to fetch the specified component. */
3397 retype(brw_vec1_indirect(addr
.subnr
, offset
% limit
),
3400 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3401 * to all bits of a flag register,
3405 stride(brw_swizzle1(idx
, 0), 0, 4, 1));
3406 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3407 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3408 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3410 /* and use predicated SEL to pick the right channel. */
3411 inst
= brw_SEL(p
, dst
,
3412 stride(suboffset(src
, 4), 0, 4, 1),
3413 stride(src
, 0, 4, 1));
3414 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3415 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3421 * This instruction is generated as a single-channel align1 instruction by
3422 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3424 * We can't use the typed atomic op in the FS because that has the execution
3425 * mask ANDed with the pixel mask, but we just want to write the one dword for
3428 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3429 * one u32. So we use the same untyped atomic write message as the pixel
3432 * The untyped atomic operation requires a BUFFER surface type with RAW
3433 * format, and is only accessible through the legacy DATA_CACHE dataport
3436 void brw_shader_time_add(struct brw_codegen
*p
,
3437 struct brw_reg payload
,
3438 uint32_t surf_index
)
3440 const unsigned sfid
= (p
->devinfo
->gen
>= 8 || p
->devinfo
->is_haswell
?
3441 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3442 GEN7_SFID_DATAPORT_DATA_CACHE
);
3443 assert(p
->devinfo
->gen
>= 7);
3445 brw_push_insn_state(p
);
3446 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3447 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3448 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3449 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3451 /* We use brw_vec1_reg and unmasked because we want to increment the given
3454 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3456 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3458 brw_set_src1(p
, send
, brw_imm_ud(0));
3459 brw_set_message_descriptor(p
, send
, sfid
, 2, 0, false, false);
3460 brw_inst_set_binding_table_index(p
->devinfo
, send
, surf_index
);
3461 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3463 brw_pop_insn_state(p
);
3468 * Emit the SEND message for a barrier
3471 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3473 const struct brw_device_info
*devinfo
= p
->devinfo
;
3474 struct brw_inst
*inst
;
3476 assert(devinfo
->gen
>= 7);
3478 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3479 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3480 brw_set_src0(p
, inst
, src
);
3481 brw_set_src1(p
, inst
, brw_null_reg());
3483 brw_set_message_descriptor(p
, inst
, BRW_SFID_MESSAGE_GATEWAY
,
3485 0 /* response_length */,
3486 false /* header_present */,
3487 false /* end_of_thread */);
3489 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3490 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3491 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3493 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3498 * Emit the wait instruction for a barrier
3501 brw_WAIT(struct brw_codegen
*p
)
3503 const struct brw_device_info
*devinfo
= p
->devinfo
;
3504 struct brw_inst
*insn
;
3506 struct brw_reg src
= brw_notification_reg();
3508 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3509 brw_set_dest(p
, insn
, src
);
3510 brw_set_src0(p
, insn
, src
);
3511 brw_set_src1(p
, insn
, brw_null_reg());
3513 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3514 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);