2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_context.h"
34 #include "brw_defines.h"
37 #include "util/ralloc.h"
40 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
41 * registers, implicitly moving the operand to a message register.
43 * On Sandybridge, this is no longer the case. This function performs the
44 * explicit move; it should be called before emitting a SEND instruction.
47 gen6_resolve_implied_move(struct brw_codegen
*p
,
51 const struct brw_device_info
*devinfo
= p
->devinfo
;
55 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
58 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct brw_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 * Convert a brw_reg_type enumeration value into the hardware representation.
91 * The hardware encoding may depend on whether the value is an immediate.
94 brw_reg_type_to_hw_type(const struct brw_device_info
*devinfo
,
95 enum brw_reg_type type
, enum brw_reg_file file
)
97 if (file
== BRW_IMMEDIATE_VALUE
) {
98 static const int imm_hw_types
[] = {
99 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
100 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
101 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
102 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
103 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
104 [BRW_REGISTER_TYPE_UB
] = -1,
105 [BRW_REGISTER_TYPE_B
] = -1,
106 [BRW_REGISTER_TYPE_UV
] = BRW_HW_REG_IMM_TYPE_UV
,
107 [BRW_REGISTER_TYPE_VF
] = BRW_HW_REG_IMM_TYPE_VF
,
108 [BRW_REGISTER_TYPE_V
] = BRW_HW_REG_IMM_TYPE_V
,
109 [BRW_REGISTER_TYPE_DF
] = GEN8_HW_REG_IMM_TYPE_DF
,
110 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_IMM_TYPE_HF
,
111 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
112 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
114 assert(type
< ARRAY_SIZE(imm_hw_types
));
115 assert(imm_hw_types
[type
] != -1);
116 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_DF
);
117 return imm_hw_types
[type
];
119 /* Non-immediate registers */
120 static const int hw_types
[] = {
121 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
122 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
123 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
124 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
125 [BRW_REGISTER_TYPE_UB
] = BRW_HW_REG_NON_IMM_TYPE_UB
,
126 [BRW_REGISTER_TYPE_B
] = BRW_HW_REG_NON_IMM_TYPE_B
,
127 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
128 [BRW_REGISTER_TYPE_UV
] = -1,
129 [BRW_REGISTER_TYPE_VF
] = -1,
130 [BRW_REGISTER_TYPE_V
] = -1,
131 [BRW_REGISTER_TYPE_DF
] = GEN7_HW_REG_NON_IMM_TYPE_DF
,
132 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_NON_IMM_TYPE_HF
,
133 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
134 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
136 assert(type
< ARRAY_SIZE(hw_types
));
137 assert(hw_types
[type
] != -1);
138 assert(devinfo
->gen
>= 7 || type
< BRW_REGISTER_TYPE_DF
);
139 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_HF
);
140 return hw_types
[type
];
145 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
147 const struct brw_device_info
*devinfo
= p
->devinfo
;
149 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
150 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
151 else if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
152 assert(dest
.nr
< 128);
154 gen7_convert_mrf_to_grf(p
, &dest
);
156 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
157 brw_inst_set_dst_reg_type(devinfo
, inst
,
158 brw_reg_type_to_hw_type(devinfo
, dest
.type
,
160 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
162 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
163 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
165 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
166 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
167 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
168 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
169 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
171 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
172 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
173 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
174 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
175 assert(dest
.writemask
!= 0);
177 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
178 * Although Dst.HorzStride is a don't care for Align16, HW needs
179 * this to be programmed as "01".
181 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
184 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
186 /* These are different sizes in align1 vs align16:
188 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
189 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
190 dest
.indirect_offset
);
191 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
192 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
193 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
195 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
196 dest
.indirect_offset
);
197 /* even ignored in da16, still need to set as '01' */
198 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
202 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
203 * or 16 (SIMD16), as that's normally correct. However, when dealing with
204 * small registers, we automatically reduce it to match the register size.
206 * In platforms that support fp64 we can emit instructions with a width of
207 * 4 that need two SIMD8 registers and an exec_size of 8 or 16. In these
208 * cases we need to make sure that these instructions have their exec sizes
209 * set properly when they are emitted and we can't rely on this code to fix
213 if (devinfo
->gen
>= 6)
214 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
216 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
219 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
222 extern int reg_type_size
[];
225 validate_reg(const struct brw_device_info
*devinfo
,
226 brw_inst
*inst
, struct brw_reg reg
)
228 const int hstride_for_reg
[] = {0, 1, 2, 4};
229 const int vstride_for_reg
[] = {0, 1, 2, 4, 8, 16, 32};
230 const int width_for_reg
[] = {1, 2, 4, 8, 16};
231 const int execsize_for_reg
[] = {1, 2, 4, 8, 16, 32};
232 int width
, hstride
, vstride
, execsize
;
234 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
235 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
236 * mean the destination has to be 128-bit aligned and the
237 * destination horiz stride has to be a word.
239 if (reg
.type
== BRW_REGISTER_TYPE_V
) {
240 assert(hstride_for_reg
[brw_inst_dst_hstride(devinfo
, inst
)] *
241 reg_type_size
[brw_inst_dst_reg_type(devinfo
, inst
)] == 2);
247 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
248 reg
.file
== BRW_ARF_NULL
)
251 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
253 * "Swizzling is not allowed when an accumulator is used as an implicit
254 * source or an explicit source in an instruction."
256 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
257 reg
.nr
== BRW_ARF_ACCUMULATOR
)
258 assert(reg
.swizzle
== BRW_SWIZZLE_XYZW
);
260 assert(reg
.hstride
>= 0 && reg
.hstride
< ARRAY_SIZE(hstride_for_reg
));
261 hstride
= hstride_for_reg
[reg
.hstride
];
263 if (reg
.vstride
== 0xf) {
266 assert(reg
.vstride
>= 0 && reg
.vstride
< ARRAY_SIZE(vstride_for_reg
));
267 vstride
= vstride_for_reg
[reg
.vstride
];
270 assert(reg
.width
>= 0 && reg
.width
< ARRAY_SIZE(width_for_reg
));
271 width
= width_for_reg
[reg
.width
];
273 assert(brw_inst_exec_size(devinfo
, inst
) >= 0 &&
274 brw_inst_exec_size(devinfo
, inst
) < ARRAY_SIZE(execsize_for_reg
));
275 execsize
= execsize_for_reg
[brw_inst_exec_size(devinfo
, inst
)];
277 /* Restrictions from 3.3.10: Register Region Restrictions. */
279 assert(execsize
>= width
);
282 if (execsize
== width
&& hstride
!= 0) {
283 assert(vstride
== -1 || vstride
== width
* hstride
);
287 if (execsize
== width
&& hstride
== 0) {
288 /* no restriction on vstride. */
293 assert(hstride
== 0);
297 if (execsize
== 1 && width
== 1) {
298 assert(hstride
== 0);
299 assert(vstride
== 0);
303 if (vstride
== 0 && hstride
== 0) {
307 /* 10. Check destination issues. */
311 is_compactable_immediate(unsigned imm
)
313 /* We get the low 12 bits as-is. */
316 /* We get one bit replicated through the top 20 bits. */
317 return imm
== 0 || imm
== 0xfffff000;
321 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
323 const struct brw_device_info
*devinfo
= p
->devinfo
;
325 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
326 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
327 else if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
328 assert(reg
.nr
< 128);
330 gen7_convert_mrf_to_grf(p
, ®
);
332 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
333 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
334 /* Any source modifiers or regions will be ignored, since this just
335 * identifies the MRF/GRF to start reading the message contents from.
336 * Check for some likely failures.
340 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
343 validate_reg(devinfo
, inst
, reg
);
345 brw_inst_set_src0_reg_file(devinfo
, inst
, reg
.file
);
346 brw_inst_set_src0_reg_type(devinfo
, inst
,
347 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
348 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
349 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
350 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
352 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
353 if (reg
.type
== BRW_REGISTER_TYPE_DF
)
354 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
356 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
358 /* The Bspec's section titled "Non-present Operands" claims that if src0
359 * is an immediate that src1's type must be the same as that of src0.
361 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
362 * that do not follow this rule. E.g., from the IVB/HSW table:
364 * DataTypeIndex 18-Bit Mapping Mapped Meaning
365 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
367 * And from the SNB table:
369 * DataTypeIndex 18-Bit Mapping Mapped Meaning
370 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
372 * Neither of these cause warnings from the simulator when used,
373 * compacted or otherwise. In fact, all compaction mappings that have an
374 * immediate in src0 use a:ud for src1.
376 * The GM45 instruction compaction tables do not contain mapped meanings
377 * so it's not clear whether it has the restriction. We'll assume it was
378 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
380 * Don't do any of this for 64-bit immediates, since the src1 fields
381 * overlap with the immediate and setting them would overwrite the
384 if (type_sz(reg
.type
) < 8) {
385 brw_inst_set_src1_reg_file(devinfo
, inst
,
386 BRW_ARCHITECTURE_REGISTER_FILE
);
387 if (devinfo
->gen
< 6) {
388 brw_inst_set_src1_reg_type(devinfo
, inst
,
389 brw_inst_src0_reg_type(devinfo
, inst
));
391 brw_inst_set_src1_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
395 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
396 * for immediate values. Presumably the hardware engineers realized
397 * that the only useful floating-point value that could be represented
398 * in this format is 0.0, which can also be represented as a VF-typed
399 * immediate, so they gave us the previously mentioned mapping on IVB+.
401 * Strangely, we do have a mapping for imm:f in src1, so we don't need
404 * If we see a 0.0:F, change the type to VF so that it can be compacted.
406 if (brw_inst_imm_ud(devinfo
, inst
) == 0x0 &&
407 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_F
) {
408 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_IMM_TYPE_VF
);
411 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
412 * set the types to :UD so the instruction can be compacted.
414 if (is_compactable_immediate(brw_inst_imm_ud(devinfo
, inst
)) &&
415 brw_inst_cond_modifier(devinfo
, inst
) == BRW_CONDITIONAL_NONE
&&
416 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
&&
417 brw_inst_dst_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
) {
418 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
419 brw_inst_set_dst_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
422 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
423 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
424 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
425 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
427 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
430 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
432 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
433 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
435 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
439 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
440 if (reg
.width
== BRW_WIDTH_1
&&
441 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
442 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
443 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
444 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
446 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
447 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
448 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
451 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
452 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
453 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
454 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
455 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
456 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
457 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
458 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
460 /* This is an oddity of the fact we're using the same
461 * descriptions for registers in align_16 as align_1:
463 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
464 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
466 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
473 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
475 const struct brw_device_info
*devinfo
= p
->devinfo
;
477 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
478 assert(reg
.nr
< 128);
480 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
482 * "Accumulator registers may be accessed explicitly as src0
485 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
486 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
488 gen7_convert_mrf_to_grf(p
, ®
);
489 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
491 validate_reg(devinfo
, inst
, reg
);
493 brw_inst_set_src1_reg_file(devinfo
, inst
, reg
.file
);
494 brw_inst_set_src1_reg_type(devinfo
, inst
,
495 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
496 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
497 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
499 /* Only src1 can be immediate in two-argument instructions.
501 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
503 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
504 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
506 /* This is a hardware restriction, which may or may not be lifted
509 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
510 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
512 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
513 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
514 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
516 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
519 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
520 if (reg
.width
== BRW_WIDTH_1
&&
521 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
522 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
523 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
524 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
526 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
527 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
528 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
531 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
532 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
533 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
534 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
535 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
536 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
537 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
538 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
540 /* This is an oddity of the fact we're using the same
541 * descriptions for registers in align_16 as align_1:
543 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
544 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
546 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
552 * Set the Message Descriptor and Extended Message Descriptor fields
555 * \note This zeroes out the Function Control bits, so it must be called
556 * \b before filling out any message-specific data. Callers can
557 * choose not to fill in irrelevant bits; they will be zero.
560 brw_set_message_descriptor(struct brw_codegen
*p
,
562 enum brw_message_target sfid
,
564 unsigned response_length
,
568 const struct brw_device_info
*devinfo
= p
->devinfo
;
570 brw_set_src1(p
, inst
, brw_imm_d(0));
572 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
573 * itself; instead, it will be a MOV/OR into the address register.
575 * In this case, we avoid setting the extended message descriptor bits,
576 * since they go on the later SEND/SENDC instead and if set here would
577 * instead clobber the conditionalmod bits.
579 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
580 if (opcode
== BRW_OPCODE_SEND
|| opcode
== BRW_OPCODE_SENDC
) {
581 brw_inst_set_sfid(devinfo
, inst
, sfid
);
584 brw_inst_set_mlen(devinfo
, inst
, msg_length
);
585 brw_inst_set_rlen(devinfo
, inst
, response_length
);
586 brw_inst_set_eot(devinfo
, inst
, end_of_thread
);
588 if (devinfo
->gen
>= 5) {
589 brw_inst_set_header_present(devinfo
, inst
, header_present
);
593 static void brw_set_math_message( struct brw_codegen
*p
,
596 unsigned integer_type
,
600 const struct brw_device_info
*devinfo
= p
->devinfo
;
602 unsigned response_length
;
604 /* Infer message length from the function */
606 case BRW_MATH_FUNCTION_POW
:
607 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
608 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
609 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
617 /* Infer response length from the function */
619 case BRW_MATH_FUNCTION_SINCOS
:
620 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
629 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
630 msg_length
, response_length
, false, false);
631 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
632 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
633 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
634 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
635 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
636 brw_inst_set_saturate(devinfo
, inst
, 0);
640 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
643 unsigned response_length
,
646 const struct brw_device_info
*devinfo
= p
->devinfo
;
648 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
649 1, response_length
, true, end_of_thread
);
650 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
651 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
652 /* The following fields are not used by FF_SYNC: */
653 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
654 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
655 brw_inst_set_urb_used(devinfo
, insn
, 0);
656 brw_inst_set_urb_complete(devinfo
, insn
, 0);
659 static void brw_set_urb_message( struct brw_codegen
*p
,
661 enum brw_urb_write_flags flags
,
663 unsigned response_length
,
665 unsigned swizzle_control
)
667 const struct brw_device_info
*devinfo
= p
->devinfo
;
669 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
670 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
671 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
673 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
674 msg_length
, response_length
, true,
675 flags
& BRW_URB_WRITE_EOT
);
677 if (flags
& BRW_URB_WRITE_OWORD
) {
678 assert(msg_length
== 2); /* header + one OWORD of data */
679 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
681 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
684 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
685 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
687 if (devinfo
->gen
< 8) {
688 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
691 if (devinfo
->gen
< 7) {
692 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
693 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
695 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
696 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
701 brw_set_dp_write_message(struct brw_codegen
*p
,
703 unsigned binding_table_index
,
704 unsigned msg_control
,
708 unsigned last_render_target
,
709 unsigned response_length
,
710 unsigned end_of_thread
,
711 unsigned send_commit_msg
)
713 const struct brw_device_info
*devinfo
= p
->devinfo
;
716 if (devinfo
->gen
>= 7) {
717 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
718 if (msg_type
== GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
)
719 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
721 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
722 } else if (devinfo
->gen
== 6) {
723 /* Use the render cache for all write messages. */
724 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
726 sfid
= BRW_SFID_DATAPORT_WRITE
;
729 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
730 header_present
, end_of_thread
);
732 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
733 brw_inst_set_dp_write_msg_type(devinfo
, insn
, msg_type
);
734 brw_inst_set_dp_write_msg_control(devinfo
, insn
, msg_control
);
735 brw_inst_set_rt_last(devinfo
, insn
, last_render_target
);
736 if (devinfo
->gen
< 7) {
737 brw_inst_set_dp_write_commit(devinfo
, insn
, send_commit_msg
);
742 brw_set_dp_read_message(struct brw_codegen
*p
,
744 unsigned binding_table_index
,
745 unsigned msg_control
,
747 unsigned target_cache
,
750 unsigned response_length
)
752 const struct brw_device_info
*devinfo
= p
->devinfo
;
755 if (devinfo
->gen
>= 7) {
756 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
757 } else if (devinfo
->gen
== 6) {
758 if (target_cache
== BRW_DATAPORT_READ_TARGET_RENDER_CACHE
)
759 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
761 sfid
= GEN6_SFID_DATAPORT_SAMPLER_CACHE
;
763 sfid
= BRW_SFID_DATAPORT_READ
;
766 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
767 header_present
, false);
769 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
770 brw_inst_set_dp_read_msg_type(devinfo
, insn
, msg_type
);
771 brw_inst_set_dp_read_msg_control(devinfo
, insn
, msg_control
);
772 if (devinfo
->gen
< 6)
773 brw_inst_set_dp_read_target_cache(devinfo
, insn
, target_cache
);
777 brw_set_sampler_message(struct brw_codegen
*p
,
779 unsigned binding_table_index
,
782 unsigned response_length
,
784 unsigned header_present
,
786 unsigned return_format
)
788 const struct brw_device_info
*devinfo
= p
->devinfo
;
790 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
791 response_length
, header_present
, false);
793 brw_inst_set_binding_table_index(devinfo
, inst
, binding_table_index
);
794 brw_inst_set_sampler(devinfo
, inst
, sampler
);
795 brw_inst_set_sampler_msg_type(devinfo
, inst
, msg_type
);
796 if (devinfo
->gen
>= 5) {
797 brw_inst_set_sampler_simd_mode(devinfo
, inst
, simd_mode
);
798 } else if (devinfo
->gen
== 4 && !devinfo
->is_g4x
) {
799 brw_inst_set_sampler_return_format(devinfo
, inst
, return_format
);
804 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
808 bool invalidate_after_read
,
810 unsigned addr_offset
,
815 const struct brw_device_info
*devinfo
= p
->devinfo
;
816 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
817 (devinfo
->gen
>= 8 && num_regs
== 8));
818 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
819 mlen
, rlen
, header_present
, false);
820 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
821 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
822 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
823 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
824 brw_inst_set_scratch_block_size(devinfo
, inst
, ffs(num_regs
) - 1);
825 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
828 #define next_insn brw_next_insn
830 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
832 const struct brw_device_info
*devinfo
= p
->devinfo
;
835 if (p
->nr_insn
+ 1 > p
->store_size
) {
837 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
840 p
->next_insn_offset
+= 16;
841 insn
= &p
->store
[p
->nr_insn
++];
842 memcpy(insn
, p
->current
, sizeof(*insn
));
844 brw_inst_set_opcode(devinfo
, insn
, opcode
);
849 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
850 struct brw_reg dest
, struct brw_reg src
)
852 brw_inst
*insn
= next_insn(p
, opcode
);
853 brw_set_dest(p
, insn
, dest
);
854 brw_set_src0(p
, insn
, src
);
859 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
860 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
862 /* 64-bit immediates are only supported on 1-src instructions */
863 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
864 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
866 brw_inst
*insn
= next_insn(p
, opcode
);
867 brw_set_dest(p
, insn
, dest
);
868 brw_set_src0(p
, insn
, src0
);
869 brw_set_src1(p
, insn
, src1
);
874 get_3src_subreg_nr(struct brw_reg reg
)
876 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
877 * use 32-bit units (components 0..7). Since they only support F/D/UD
878 * types, this doesn't lose any flexibility, but uses fewer bits.
880 return reg
.subnr
/ 4;
884 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
885 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
887 const struct brw_device_info
*devinfo
= p
->devinfo
;
888 brw_inst
*inst
= next_insn(p
, opcode
);
890 gen7_convert_mrf_to_grf(p
, &dest
);
892 assert(brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
);
894 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
895 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
896 assert(dest
.nr
< 128);
897 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
898 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
899 dest
.type
== BRW_REGISTER_TYPE_DF
||
900 dest
.type
== BRW_REGISTER_TYPE_D
||
901 dest
.type
== BRW_REGISTER_TYPE_UD
);
902 if (devinfo
->gen
== 6) {
903 brw_inst_set_3src_dst_reg_file(devinfo
, inst
,
904 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
906 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
907 brw_inst_set_3src_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
908 brw_inst_set_3src_dst_writemask(devinfo
, inst
, dest
.writemask
);
910 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
911 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
912 assert(src0
.nr
< 128);
913 brw_inst_set_3src_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
914 brw_inst_set_3src_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
915 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
916 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
917 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
918 brw_inst_set_3src_src0_rep_ctrl(devinfo
, inst
,
919 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
921 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
922 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
923 assert(src1
.nr
< 128);
924 brw_inst_set_3src_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
925 brw_inst_set_3src_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
926 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
927 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
928 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
929 brw_inst_set_3src_src1_rep_ctrl(devinfo
, inst
,
930 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
932 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
933 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
934 assert(src2
.nr
< 128);
935 brw_inst_set_3src_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
936 brw_inst_set_3src_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
937 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
938 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
939 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
940 brw_inst_set_3src_src2_rep_ctrl(devinfo
, inst
,
941 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
943 if (devinfo
->gen
>= 7) {
944 /* Set both the source and destination types based on dest.type,
945 * ignoring the source register types. The MAD and LRP emitters ensure
946 * that all four types are float. The BFE and BFI2 emitters, however,
947 * may send us mixed D and UD types and want us to ignore that and use
948 * the destination type.
951 case BRW_REGISTER_TYPE_F
:
952 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
953 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
955 case BRW_REGISTER_TYPE_DF
:
956 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_DF
);
957 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_DF
);
959 case BRW_REGISTER_TYPE_D
:
960 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
961 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
963 case BRW_REGISTER_TYPE_UD
:
964 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
965 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
968 unreachable("not reached");
976 /***********************************************************************
977 * Convenience routines.
980 brw_inst *brw_##OP(struct brw_codegen *p, \
981 struct brw_reg dest, \
982 struct brw_reg src0) \
984 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
988 brw_inst *brw_##OP(struct brw_codegen *p, \
989 struct brw_reg dest, \
990 struct brw_reg src0, \
991 struct brw_reg src1) \
993 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
997 brw_inst *brw_##OP(struct brw_codegen *p, \
998 struct brw_reg dest, \
999 struct brw_reg src0, \
1000 struct brw_reg src1, \
1001 struct brw_reg src2) \
1003 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1007 brw_inst *brw_##OP(struct brw_codegen *p, \
1008 struct brw_reg dest, \
1009 struct brw_reg src0, \
1010 struct brw_reg src1, \
1011 struct brw_reg src2) \
1013 assert(dest.type == BRW_REGISTER_TYPE_F || \
1014 dest.type == BRW_REGISTER_TYPE_DF); \
1015 if (dest.type == BRW_REGISTER_TYPE_F) { \
1016 assert(src0.type == BRW_REGISTER_TYPE_F); \
1017 assert(src1.type == BRW_REGISTER_TYPE_F); \
1018 assert(src2.type == BRW_REGISTER_TYPE_F); \
1019 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
1020 assert(src0.type == BRW_REGISTER_TYPE_DF); \
1021 assert(src1.type == BRW_REGISTER_TYPE_DF); \
1022 assert(src2.type == BRW_REGISTER_TYPE_DF); \
1024 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1027 /* Rounding operations (other than RNDD) require two instructions - the first
1028 * stores a rounded value (possibly the wrong way) in the dest register, but
1029 * also sets a per-channel "increment bit" in the flag register. A predicated
1030 * add of 1.0 fixes dest to contain the desired result.
1032 * Sandybridge and later appear to round correctly without an ADD.
1035 void brw_##OP(struct brw_codegen *p, \
1036 struct brw_reg dest, \
1037 struct brw_reg src) \
1039 const struct brw_device_info *devinfo = p->devinfo; \
1040 brw_inst *rnd, *add; \
1041 rnd = next_insn(p, BRW_OPCODE_##OP); \
1042 brw_set_dest(p, rnd, dest); \
1043 brw_set_src0(p, rnd, src); \
1045 if (devinfo->gen < 6) { \
1046 /* turn on round-increments */ \
1047 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
1048 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1049 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1089 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1090 struct brw_reg src0
, struct brw_reg src1
)
1093 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1094 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1095 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1096 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1097 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1100 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1101 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1102 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1103 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1104 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1107 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1111 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1112 struct brw_reg src0
, struct brw_reg src1
)
1114 assert(dest
.type
== src0
.type
);
1115 assert(src0
.type
== src1
.type
);
1116 switch (src0
.type
) {
1117 case BRW_REGISTER_TYPE_B
:
1118 case BRW_REGISTER_TYPE_UB
:
1119 case BRW_REGISTER_TYPE_W
:
1120 case BRW_REGISTER_TYPE_UW
:
1121 case BRW_REGISTER_TYPE_D
:
1122 case BRW_REGISTER_TYPE_UD
:
1125 unreachable("Bad type for brw_AVG");
1128 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1132 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1133 struct brw_reg src0
, struct brw_reg src1
)
1136 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1137 src0
.type
== BRW_REGISTER_TYPE_UD
||
1138 src1
.type
== BRW_REGISTER_TYPE_D
||
1139 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1140 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1143 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1144 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1145 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1146 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1147 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1150 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1151 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1152 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1153 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1154 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1157 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1158 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1159 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1160 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1162 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1166 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1167 struct brw_reg src0
, struct brw_reg src1
)
1169 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1170 src0
.width
= BRW_WIDTH_1
;
1171 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1172 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1176 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1177 struct brw_reg src0
, struct brw_reg src1
)
1179 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1180 src0
.width
= BRW_WIDTH_1
;
1181 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1182 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1183 src1
.width
= BRW_WIDTH_8
;
1184 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1185 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1189 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1191 const struct brw_device_info
*devinfo
= p
->devinfo
;
1192 const bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1193 /* The F32TO16 instruction doesn't support 32-bit destination types in
1194 * Align1 mode, and neither does the Gen8 implementation in terms of a
1195 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1196 * an undocumented feature.
1198 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1199 (!align16
|| devinfo
->gen
>= 8));
1203 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1205 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1206 dst
.type
== BRW_REGISTER_TYPE_W
||
1207 dst
.type
== BRW_REGISTER_TYPE_UW
||
1208 dst
.type
== BRW_REGISTER_TYPE_HF
);
1211 brw_push_insn_state(p
);
1213 if (needs_zero_fill
) {
1214 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1215 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1218 if (devinfo
->gen
>= 8) {
1219 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1221 assert(devinfo
->gen
== 7);
1222 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1225 if (needs_zero_fill
) {
1226 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1227 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_ud(0u));
1228 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1231 brw_pop_insn_state(p
);
1236 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1238 const struct brw_device_info
*devinfo
= p
->devinfo
;
1239 bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1242 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1244 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1246 * Because this instruction does not have a 16-bit floating-point
1247 * type, the source data type must be Word (W). The destination type
1248 * must be F (Float).
1250 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1251 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1253 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1254 src
.type
== BRW_REGISTER_TYPE_UW
||
1255 src
.type
== BRW_REGISTER_TYPE_HF
);
1258 if (devinfo
->gen
>= 8) {
1259 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1261 assert(devinfo
->gen
== 7);
1262 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1267 void brw_NOP(struct brw_codegen
*p
)
1269 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1270 brw_inst_set_exec_size(p
->devinfo
, insn
, BRW_EXECUTE_1
);
1271 brw_set_dest(p
, insn
, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD
));
1272 brw_set_src0(p
, insn
, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD
));
1273 brw_set_src1(p
, insn
, brw_imm_ud(0x0));
1280 /***********************************************************************
1281 * Comparisons, if/else/endif
1285 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1286 unsigned predicate_control
)
1288 const struct brw_device_info
*devinfo
= p
->devinfo
;
1289 struct brw_reg ip
= brw_ip_reg();
1290 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1292 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_2
);
1293 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1294 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1295 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1301 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1303 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1305 p
->if_stack_depth
++;
1306 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1307 p
->if_stack_array_size
*= 2;
1308 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1309 p
->if_stack_array_size
);
1314 pop_if_stack(struct brw_codegen
*p
)
1316 p
->if_stack_depth
--;
1317 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1321 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1323 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1324 p
->loop_stack_array_size
*= 2;
1325 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1326 p
->loop_stack_array_size
);
1327 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1328 p
->loop_stack_array_size
);
1331 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1332 p
->loop_stack_depth
++;
1333 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1337 get_inner_do_insn(struct brw_codegen
*p
)
1339 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1342 /* EU takes the value from the flag register and pushes it onto some
1343 * sort of a stack (presumably merging with any flag value already on
1344 * the stack). Within an if block, the flags at the top of the stack
1345 * control execution on each channel of the unit, eg. on each of the
1346 * 16 pixel values in our wm programs.
1348 * When the matching 'else' instruction is reached (presumably by
1349 * countdown of the instruction count patched in by our ELSE/ENDIF
1350 * functions), the relevant flags are inverted.
1352 * When the matching 'endif' instruction is reached, the flags are
1353 * popped off. If the stack is now empty, normal execution resumes.
1356 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1358 const struct brw_device_info
*devinfo
= p
->devinfo
;
1361 insn
= next_insn(p
, BRW_OPCODE_IF
);
1363 /* Override the defaults for this instruction:
1365 if (devinfo
->gen
< 6) {
1366 brw_set_dest(p
, insn
, brw_ip_reg());
1367 brw_set_src0(p
, insn
, brw_ip_reg());
1368 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1369 } else if (devinfo
->gen
== 6) {
1370 brw_set_dest(p
, insn
, brw_imm_w(0));
1371 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1372 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1373 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1374 } else if (devinfo
->gen
== 7) {
1375 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1376 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1377 brw_set_src1(p
, insn
, brw_imm_w(0));
1378 brw_inst_set_jip(devinfo
, insn
, 0);
1379 brw_inst_set_uip(devinfo
, insn
, 0);
1381 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1382 brw_set_src0(p
, insn
, brw_imm_d(0));
1383 brw_inst_set_jip(devinfo
, insn
, 0);
1384 brw_inst_set_uip(devinfo
, insn
, 0);
1387 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1388 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1389 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1390 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1391 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1392 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1394 push_if_stack(p
, insn
);
1395 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1399 /* This function is only used for gen6-style IF instructions with an
1400 * embedded comparison (conditional modifier). It is not used on gen7.
1403 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1404 struct brw_reg src0
, struct brw_reg src1
)
1406 const struct brw_device_info
*devinfo
= p
->devinfo
;
1409 insn
= next_insn(p
, BRW_OPCODE_IF
);
1411 brw_set_dest(p
, insn
, brw_imm_w(0));
1412 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1414 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1415 brw_set_src0(p
, insn
, src0
);
1416 brw_set_src1(p
, insn
, src1
);
1418 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1419 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1420 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1422 push_if_stack(p
, insn
);
1427 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1430 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1431 brw_inst
*if_inst
, brw_inst
*else_inst
)
1433 const struct brw_device_info
*devinfo
= p
->devinfo
;
1435 /* The next instruction (where the ENDIF would be, if it existed) */
1436 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1438 assert(p
->single_program_flow
);
1439 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1440 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1441 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1443 /* Convert IF to an ADD instruction that moves the instruction pointer
1444 * to the first instruction of the ELSE block. If there is no ELSE
1445 * block, point to where ENDIF would be. Reverse the predicate.
1447 * There's no need to execute an ENDIF since we don't need to do any
1448 * stack operations, and if we're currently executing, we just want to
1449 * continue normally.
1451 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1452 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1454 if (else_inst
!= NULL
) {
1455 /* Convert ELSE to an ADD instruction that points where the ENDIF
1458 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1460 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1461 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1463 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1468 * Patch IF and ELSE instructions with appropriate jump targets.
1471 patch_IF_ELSE(struct brw_codegen
*p
,
1472 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1474 const struct brw_device_info
*devinfo
= p
->devinfo
;
1476 /* We shouldn't be patching IF and ELSE instructions in single program flow
1477 * mode when gen < 6, because in single program flow mode on those
1478 * platforms, we convert flow control instructions to conditional ADDs that
1479 * operate on IP (see brw_ENDIF).
1481 * However, on Gen6, writing to IP doesn't work in single program flow mode
1482 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1483 * not be updated by non-flow control instructions."). And on later
1484 * platforms, there is no significant benefit to converting control flow
1485 * instructions to conditional ADDs. So we do patch IF and ELSE
1486 * instructions in single program flow mode on those platforms.
1488 if (devinfo
->gen
< 6)
1489 assert(!p
->single_program_flow
);
1491 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1492 assert(endif_inst
!= NULL
);
1493 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1495 unsigned br
= brw_jump_scale(devinfo
);
1497 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1498 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1500 if (else_inst
== NULL
) {
1501 /* Patch IF -> ENDIF */
1502 if (devinfo
->gen
< 6) {
1503 /* Turn it into an IFF, which means no mask stack operations for
1504 * all-false and jumping past the ENDIF.
1506 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1507 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1508 br
* (endif_inst
- if_inst
+ 1));
1509 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1510 } else if (devinfo
->gen
== 6) {
1511 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1512 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1514 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1515 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1518 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1520 /* Patch IF -> ELSE */
1521 if (devinfo
->gen
< 6) {
1522 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1523 br
* (else_inst
- if_inst
));
1524 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1525 } else if (devinfo
->gen
== 6) {
1526 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1527 br
* (else_inst
- if_inst
+ 1));
1530 /* Patch ELSE -> ENDIF */
1531 if (devinfo
->gen
< 6) {
1532 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1535 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1536 br
* (endif_inst
- else_inst
+ 1));
1537 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1538 } else if (devinfo
->gen
== 6) {
1539 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1540 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1541 br
* (endif_inst
- else_inst
));
1543 /* The IF instruction's JIP should point just past the ELSE */
1544 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1545 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1546 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1547 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1548 if (devinfo
->gen
>= 8) {
1549 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1550 * should point to ENDIF.
1552 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1559 brw_ELSE(struct brw_codegen
*p
)
1561 const struct brw_device_info
*devinfo
= p
->devinfo
;
1564 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1566 if (devinfo
->gen
< 6) {
1567 brw_set_dest(p
, insn
, brw_ip_reg());
1568 brw_set_src0(p
, insn
, brw_ip_reg());
1569 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1570 } else if (devinfo
->gen
== 6) {
1571 brw_set_dest(p
, insn
, brw_imm_w(0));
1572 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1573 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1574 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1575 } else if (devinfo
->gen
== 7) {
1576 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1577 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1578 brw_set_src1(p
, insn
, brw_imm_w(0));
1579 brw_inst_set_jip(devinfo
, insn
, 0);
1580 brw_inst_set_uip(devinfo
, insn
, 0);
1582 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1583 brw_set_src0(p
, insn
, brw_imm_d(0));
1584 brw_inst_set_jip(devinfo
, insn
, 0);
1585 brw_inst_set_uip(devinfo
, insn
, 0);
1588 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1589 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1590 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1591 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1593 push_if_stack(p
, insn
);
1597 brw_ENDIF(struct brw_codegen
*p
)
1599 const struct brw_device_info
*devinfo
= p
->devinfo
;
1600 brw_inst
*insn
= NULL
;
1601 brw_inst
*else_inst
= NULL
;
1602 brw_inst
*if_inst
= NULL
;
1604 bool emit_endif
= true;
1606 /* In single program flow mode, we can express IF and ELSE instructions
1607 * equivalently as ADD instructions that operate on IP. On platforms prior
1608 * to Gen6, flow control instructions cause an implied thread switch, so
1609 * this is a significant savings.
1611 * However, on Gen6, writing to IP doesn't work in single program flow mode
1612 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1613 * not be updated by non-flow control instructions."). And on later
1614 * platforms, there is no significant benefit to converting control flow
1615 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1618 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1622 * A single next_insn() may change the base address of instruction store
1623 * memory(p->store), so call it first before referencing the instruction
1624 * store pointer from an index
1627 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1629 /* Pop the IF and (optional) ELSE instructions from the stack */
1630 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1631 tmp
= pop_if_stack(p
);
1632 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1634 tmp
= pop_if_stack(p
);
1639 /* ENDIF is useless; don't bother emitting it. */
1640 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1644 if (devinfo
->gen
< 6) {
1645 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1646 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1647 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1648 } else if (devinfo
->gen
== 6) {
1649 brw_set_dest(p
, insn
, brw_imm_w(0));
1650 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1651 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1652 } else if (devinfo
->gen
== 7) {
1653 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1654 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1655 brw_set_src1(p
, insn
, brw_imm_w(0));
1657 brw_set_src0(p
, insn
, brw_imm_d(0));
1660 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1661 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1662 if (devinfo
->gen
< 6)
1663 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1665 /* Also pop item off the stack in the endif instruction: */
1666 if (devinfo
->gen
< 6) {
1667 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1668 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1669 } else if (devinfo
->gen
== 6) {
1670 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1672 brw_inst_set_jip(devinfo
, insn
, 2);
1674 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1678 brw_BREAK(struct brw_codegen
*p
)
1680 const struct brw_device_info
*devinfo
= p
->devinfo
;
1683 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1684 if (devinfo
->gen
>= 8) {
1685 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1686 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1687 } else if (devinfo
->gen
>= 6) {
1688 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1689 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1690 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1692 brw_set_dest(p
, insn
, brw_ip_reg());
1693 brw_set_src0(p
, insn
, brw_ip_reg());
1694 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1695 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1696 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1698 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1699 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1706 brw_CONT(struct brw_codegen
*p
)
1708 const struct brw_device_info
*devinfo
= p
->devinfo
;
1711 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1712 brw_set_dest(p
, insn
, brw_ip_reg());
1713 if (devinfo
->gen
>= 8) {
1714 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1716 brw_set_src0(p
, insn
, brw_ip_reg());
1717 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1720 if (devinfo
->gen
< 6) {
1721 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1722 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1724 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1725 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1731 gen6_HALT(struct brw_codegen
*p
)
1733 const struct brw_device_info
*devinfo
= p
->devinfo
;
1736 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1737 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1738 if (devinfo
->gen
>= 8) {
1739 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1741 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1742 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1745 if (p
->compressed
) {
1746 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_16
);
1748 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1749 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_8
);
1756 * The DO/WHILE is just an unterminated loop -- break or continue are
1757 * used for control within the loop. We have a few ways they can be
1760 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1761 * jip and no DO instruction.
1763 * For non-uniform control flow pre-gen6, there's a DO instruction to
1764 * push the mask, and a WHILE to jump back, and BREAK to get out and
1767 * For gen6, there's no more mask stack, so no need for DO. WHILE
1768 * just points back to the first instruction of the loop.
1771 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1773 const struct brw_device_info
*devinfo
= p
->devinfo
;
1775 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1776 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1777 return &p
->store
[p
->nr_insn
];
1779 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1781 push_loop_stack(p
, insn
);
1783 /* Override the defaults for this instruction:
1785 brw_set_dest(p
, insn
, brw_null_reg());
1786 brw_set_src0(p
, insn
, brw_null_reg());
1787 brw_set_src1(p
, insn
, brw_null_reg());
1789 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1790 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1791 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1798 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1801 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1802 * nesting, since it can always just point to the end of the block/current loop.
1805 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1807 const struct brw_device_info
*devinfo
= p
->devinfo
;
1808 brw_inst
*do_inst
= get_inner_do_insn(p
);
1810 unsigned br
= brw_jump_scale(devinfo
);
1812 assert(devinfo
->gen
< 6);
1814 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1815 /* If the jump count is != 0, that means that this instruction has already
1816 * been patched because it's part of a loop inside of the one we're
1819 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1820 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1821 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1822 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1823 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1824 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1830 brw_WHILE(struct brw_codegen
*p
)
1832 const struct brw_device_info
*devinfo
= p
->devinfo
;
1833 brw_inst
*insn
, *do_insn
;
1834 unsigned br
= brw_jump_scale(devinfo
);
1836 if (devinfo
->gen
>= 6) {
1837 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1838 do_insn
= get_inner_do_insn(p
);
1840 if (devinfo
->gen
>= 8) {
1841 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1842 brw_set_src0(p
, insn
, brw_imm_d(0));
1843 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1844 } else if (devinfo
->gen
== 7) {
1845 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1846 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1847 brw_set_src1(p
, insn
, brw_imm_w(0));
1848 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1850 brw_set_dest(p
, insn
, brw_imm_w(0));
1851 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1852 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1853 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1856 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1859 if (p
->single_program_flow
) {
1860 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1861 do_insn
= get_inner_do_insn(p
);
1863 brw_set_dest(p
, insn
, brw_ip_reg());
1864 brw_set_src0(p
, insn
, brw_ip_reg());
1865 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1866 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1868 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1869 do_insn
= get_inner_do_insn(p
);
1871 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1873 brw_set_dest(p
, insn
, brw_ip_reg());
1874 brw_set_src0(p
, insn
, brw_ip_reg());
1875 brw_set_src1(p
, insn
, brw_imm_d(0));
1877 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1878 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1879 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1881 brw_patch_break_cont(p
, insn
);
1884 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1886 p
->loop_stack_depth
--;
1893 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1895 const struct brw_device_info
*devinfo
= p
->devinfo
;
1896 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1899 if (devinfo
->gen
>= 5)
1902 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1903 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1905 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1906 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1909 /* To integrate with the above, it makes sense that the comparison
1910 * instruction should populate the flag register. It might be simpler
1911 * just to use the flag reg for most WM tasks?
1913 void brw_CMP(struct brw_codegen
*p
,
1914 struct brw_reg dest
,
1915 unsigned conditional
,
1916 struct brw_reg src0
,
1917 struct brw_reg src1
)
1919 const struct brw_device_info
*devinfo
= p
->devinfo
;
1920 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1922 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1923 brw_set_dest(p
, insn
, dest
);
1924 brw_set_src0(p
, insn
, src0
);
1925 brw_set_src1(p
, insn
, src1
);
1927 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1929 * "Any CMP instruction with a null destination must use a {switch}."
1931 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1932 * mentioned on their work-arounds pages.
1934 if (devinfo
->gen
== 7) {
1935 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1936 dest
.nr
== BRW_ARF_NULL
) {
1937 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1942 /***********************************************************************
1943 * Helpers for the various SEND message types:
1946 /** Extended math function, float[8].
1948 void gen4_math(struct brw_codegen
*p
,
1949 struct brw_reg dest
,
1951 unsigned msg_reg_nr
,
1953 unsigned precision
)
1955 const struct brw_device_info
*devinfo
= p
->devinfo
;
1956 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1958 if (has_scalar_region(src
)) {
1959 data_type
= BRW_MATH_DATA_SCALAR
;
1961 data_type
= BRW_MATH_DATA_VECTOR
;
1964 assert(devinfo
->gen
< 6);
1966 /* Example code doesn't set predicate_control for send
1969 brw_inst_set_pred_control(devinfo
, insn
, 0);
1970 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1972 brw_set_dest(p
, insn
, dest
);
1973 brw_set_src0(p
, insn
, src
);
1974 brw_set_math_message(p
,
1977 src
.type
== BRW_REGISTER_TYPE_D
,
1982 void gen6_math(struct brw_codegen
*p
,
1983 struct brw_reg dest
,
1985 struct brw_reg src0
,
1986 struct brw_reg src1
)
1988 const struct brw_device_info
*devinfo
= p
->devinfo
;
1989 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1991 assert(devinfo
->gen
>= 6);
1993 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1994 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1995 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
1996 (devinfo
->gen
>= 8 && src0
.file
== BRW_IMMEDIATE_VALUE
));
1998 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1999 if (devinfo
->gen
== 6) {
2000 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2001 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2004 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2005 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2006 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2007 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2008 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2009 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2010 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2012 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
2013 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
2014 if (function
== BRW_MATH_FUNCTION_POW
) {
2015 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2016 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2018 assert(src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
2019 src1
.nr
== BRW_ARF_NULL
);
2023 /* Source modifiers are ignored for extended math instructions on Gen6. */
2024 if (devinfo
->gen
== 6) {
2025 assert(!src0
.negate
);
2027 assert(!src1
.negate
);
2031 brw_inst_set_math_function(devinfo
, insn
, function
);
2033 brw_set_dest(p
, insn
, dest
);
2034 brw_set_src0(p
, insn
, src0
);
2035 brw_set_src1(p
, insn
, src1
);
2039 * Return the right surface index to access the thread scratch space using
2040 * stateless dataport messages.
2043 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2045 /* The scratch space is thread-local so IA coherency is unnecessary. */
2046 if (p
->devinfo
->gen
>= 8)
2047 return GEN8_BTI_STATELESS_NON_COHERENT
;
2049 return BRW_BTI_STATELESS
;
2053 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2054 * using a constant offset per channel.
2056 * The offset must be aligned to oword size (16 bytes). Used for
2057 * register spilling.
2059 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2064 const struct brw_device_info
*devinfo
= p
->devinfo
;
2065 uint32_t msg_control
, msg_type
;
2068 if (devinfo
->gen
>= 6)
2071 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2073 if (num_regs
== 1) {
2074 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2077 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2081 /* Set up the message header. This is g0, with g0.2 filled with
2082 * the offset. We don't want to leave our offset around in g0 or
2083 * it'll screw up texture samples, so set it up inside the message
2087 brw_push_insn_state(p
);
2088 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2089 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2090 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2092 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2094 /* set message header global offset field (reg 0, element 2) */
2096 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2098 2), BRW_REGISTER_TYPE_UD
),
2099 brw_imm_ud(offset
));
2101 brw_pop_insn_state(p
);
2105 struct brw_reg dest
;
2106 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2107 int send_commit_msg
;
2108 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2109 BRW_REGISTER_TYPE_UW
);
2111 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_NONE
) {
2112 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2113 src_header
= vec16(src_header
);
2115 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2116 if (devinfo
->gen
< 6)
2117 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2119 /* Until gen6, writes followed by reads from the same location
2120 * are not guaranteed to be ordered unless write_commit is set.
2121 * If set, then a no-op write is issued to the destination
2122 * register to set a dependency, and a read from the destination
2123 * can be used to ensure the ordering.
2125 * For gen6, only writes between different threads need ordering
2126 * protection. Our use of DP writes is all about register
2127 * spilling within a thread.
2129 if (devinfo
->gen
>= 6) {
2130 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2131 send_commit_msg
= 0;
2134 send_commit_msg
= 1;
2137 brw_set_dest(p
, insn
, dest
);
2138 if (devinfo
->gen
>= 6) {
2139 brw_set_src0(p
, insn
, mrf
);
2141 brw_set_src0(p
, insn
, brw_null_reg());
2144 if (devinfo
->gen
>= 6)
2145 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2147 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2149 brw_set_dp_write_message(p
,
2151 brw_scratch_surface_idx(p
),
2155 true, /* header_present */
2156 0, /* not a render target */
2157 send_commit_msg
, /* response_length */
2165 * Read a block of owords (half a GRF each) from the scratch buffer
2166 * using a constant index per channel.
2168 * Offset must be aligned to oword size (16 bytes). Used for register
2172 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2173 struct brw_reg dest
,
2178 const struct brw_device_info
*devinfo
= p
->devinfo
;
2179 uint32_t msg_control
;
2182 if (devinfo
->gen
>= 6)
2185 if (p
->devinfo
->gen
>= 7) {
2186 /* On gen 7 and above, we no longer have message registers and we can
2187 * send from any register we want. By using the destination register
2188 * for the message, we guarantee that the implied message write won't
2189 * accidentally overwrite anything. This has been a problem because
2190 * the MRF registers and source for the final FB write are both fixed
2193 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2195 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2197 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2199 if (num_regs
== 1) {
2200 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2203 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2208 brw_push_insn_state(p
);
2209 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2210 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2211 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2213 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2215 /* set message header global offset field (reg 0, element 2) */
2216 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2218 brw_pop_insn_state(p
);
2222 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2224 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2225 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2227 brw_set_dest(p
, insn
, dest
); /* UW? */
2228 if (devinfo
->gen
>= 6) {
2229 brw_set_src0(p
, insn
, mrf
);
2231 brw_set_src0(p
, insn
, brw_null_reg());
2232 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2235 brw_set_dp_read_message(p
,
2237 brw_scratch_surface_idx(p
),
2239 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2240 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
,
2242 true, /* header_present */
2248 gen7_block_read_scratch(struct brw_codegen
*p
,
2249 struct brw_reg dest
,
2253 const struct brw_device_info
*devinfo
= p
->devinfo
;
2254 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2255 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2257 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2258 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2260 /* The HW requires that the header is present; this is to get the g0.5
2263 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2265 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2266 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2267 * is 32 bytes, which happens to be the size of a register.
2270 assert(offset
< (1 << 12));
2272 gen7_set_dp_scratch_message(p
, insn
,
2273 false, /* scratch read */
2275 false, /* invalidate after read */
2278 1, /* mlen: just g0 */
2279 num_regs
, /* rlen */
2280 true); /* header present */
2284 * Read a float[4] vector from the data port Data Cache (const buffer).
2285 * Location (in buffer) should be a multiple of 16.
2286 * Used for fetching shader constants.
2288 void brw_oword_block_read(struct brw_codegen
*p
,
2289 struct brw_reg dest
,
2292 uint32_t bind_table_index
)
2294 const struct brw_device_info
*devinfo
= p
->devinfo
;
2296 /* On newer hardware, offset is in units of owords. */
2297 if (devinfo
->gen
>= 6)
2300 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2302 brw_push_insn_state(p
);
2303 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2304 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2305 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2306 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2308 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2310 /* set message header global offset field (reg 0, element 2) */
2312 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2314 2), BRW_REGISTER_TYPE_UD
),
2315 brw_imm_ud(offset
));
2317 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2319 /* cast dest to a uword[8] vector */
2320 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2322 brw_set_dest(p
, insn
, dest
);
2323 if (devinfo
->gen
>= 6) {
2324 brw_set_src0(p
, insn
, mrf
);
2326 brw_set_src0(p
, insn
, brw_null_reg());
2327 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2330 brw_set_dp_read_message(p
,
2333 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW
,
2334 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2335 BRW_DATAPORT_READ_TARGET_DATA_CACHE
,
2337 true, /* header_present */
2338 1); /* response_length (1 reg, 2 owords!) */
2340 brw_pop_insn_state(p
);
2344 void brw_fb_WRITE(struct brw_codegen
*p
,
2346 struct brw_reg payload
,
2347 struct brw_reg implied_header
,
2348 unsigned msg_control
,
2349 unsigned binding_table_index
,
2350 unsigned msg_length
,
2351 unsigned response_length
,
2353 bool last_render_target
,
2354 bool header_present
)
2356 const struct brw_device_info
*devinfo
= p
->devinfo
;
2359 struct brw_reg dest
, src0
;
2361 if (dispatch_width
== 16)
2362 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2364 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2366 if (devinfo
->gen
>= 6) {
2367 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2369 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2371 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2373 if (devinfo
->gen
>= 6) {
2374 /* headerless version, just submit color payload */
2377 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2379 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2380 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2381 src0
= implied_header
;
2383 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2386 brw_set_dest(p
, insn
, dest
);
2387 brw_set_src0(p
, insn
, src0
);
2388 brw_set_dp_write_message(p
,
2390 binding_table_index
,
2398 0 /* send_commit_msg */);
2403 * Texture sample instruction.
2404 * Note: the msg_type plus msg_length values determine exactly what kind
2405 * of sampling operation is performed. See volume 4, page 161 of docs.
2407 void brw_SAMPLE(struct brw_codegen
*p
,
2408 struct brw_reg dest
,
2409 unsigned msg_reg_nr
,
2410 struct brw_reg src0
,
2411 unsigned binding_table_index
,
2414 unsigned response_length
,
2415 unsigned msg_length
,
2416 unsigned header_present
,
2418 unsigned return_format
)
2420 const struct brw_device_info
*devinfo
= p
->devinfo
;
2423 if (msg_reg_nr
!= -1)
2424 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2426 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2427 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2429 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2431 * "Instruction compression is not allowed for this instruction (that
2432 * is, send). The hardware behavior is undefined if this instruction is
2433 * set as compressed. However, compress control can be set to "SecHalf"
2434 * to affect the EMask generation."
2436 * No similar wording is found in later PRMs, but there are examples
2437 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2438 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2439 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2441 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_2NDHALF
)
2442 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2444 if (devinfo
->gen
< 6)
2445 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2447 brw_set_dest(p
, insn
, dest
);
2448 brw_set_src0(p
, insn
, src0
);
2449 brw_set_sampler_message(p
, insn
,
2450 binding_table_index
,
2460 /* Adjust the message header's sampler state pointer to
2461 * select the correct group of 16 samplers.
2463 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2464 struct brw_reg header
,
2465 struct brw_reg sampler_index
)
2467 /* The "Sampler Index" field can only store values between 0 and 15.
2468 * However, we can add an offset to the "Sampler State Pointer"
2469 * field, effectively selecting a different set of 16 samplers.
2471 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2472 * offset, and each sampler state is only 16-bytes, so we can't
2473 * exclusively use the offset - we have to use both.
2476 const struct brw_device_info
*devinfo
= p
->devinfo
;
2478 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2479 const int sampler_state_size
= 16; /* 16 bytes */
2480 uint32_t sampler
= sampler_index
.ud
;
2482 if (sampler
>= 16) {
2483 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2485 get_element_ud(header
, 3),
2486 get_element_ud(brw_vec8_grf(0, 0), 3),
2487 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2490 /* Non-const sampler array indexing case */
2491 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2495 struct brw_reg temp
= get_element_ud(header
, 3);
2497 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2498 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2500 get_element_ud(header
, 3),
2501 get_element_ud(brw_vec8_grf(0, 0), 3),
2506 /* All these variables are pretty confusing - we might be better off
2507 * using bitmasks and macros for this, in the old style. Or perhaps
2508 * just having the caller instantiate the fields in dword3 itself.
2510 void brw_urb_WRITE(struct brw_codegen
*p
,
2511 struct brw_reg dest
,
2512 unsigned msg_reg_nr
,
2513 struct brw_reg src0
,
2514 enum brw_urb_write_flags flags
,
2515 unsigned msg_length
,
2516 unsigned response_length
,
2520 const struct brw_device_info
*devinfo
= p
->devinfo
;
2523 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2525 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2526 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2527 brw_push_insn_state(p
);
2528 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2529 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2530 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2531 BRW_REGISTER_TYPE_UD
),
2532 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2533 brw_imm_ud(0xff00));
2534 brw_pop_insn_state(p
);
2537 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2539 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2541 brw_set_dest(p
, insn
, dest
);
2542 brw_set_src0(p
, insn
, src0
);
2543 brw_set_src1(p
, insn
, brw_imm_d(0));
2545 if (devinfo
->gen
< 6)
2546 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2548 brw_set_urb_message(p
,
2558 brw_send_indirect_message(struct brw_codegen
*p
,
2561 struct brw_reg payload
,
2562 struct brw_reg desc
)
2564 const struct brw_device_info
*devinfo
= p
->devinfo
;
2565 struct brw_inst
*send
;
2568 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2570 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2572 /* We hold on to the setup instruction (the SEND in the direct case, the OR
2573 * in the indirect case) by its index in the instruction store. The
2574 * pointer returned by next_insn() may become invalid if emitting the SEND
2575 * in the indirect case reallocs the store.
2578 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2580 send
= next_insn(p
, BRW_OPCODE_SEND
);
2581 brw_set_src1(p
, send
, desc
);
2584 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2586 brw_push_insn_state(p
);
2587 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2588 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2589 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2591 /* Load the indirect descriptor to an address register using OR so the
2592 * caller can specify additional descriptor bits with the usual
2593 * brw_set_*_message() helper functions.
2596 brw_OR(p
, addr
, desc
, brw_imm_ud(0));
2598 brw_pop_insn_state(p
);
2600 send
= next_insn(p
, BRW_OPCODE_SEND
);
2601 brw_set_src1(p
, send
, addr
);
2604 if (dst
.width
< BRW_EXECUTE_8
)
2605 brw_inst_set_exec_size(devinfo
, send
, dst
.width
);
2607 brw_set_dest(p
, send
, dst
);
2608 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2609 brw_inst_set_sfid(devinfo
, send
, sfid
);
2611 return &p
->store
[setup
];
2614 static struct brw_inst
*
2615 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2618 struct brw_reg payload
,
2619 struct brw_reg surface
,
2620 unsigned message_len
,
2621 unsigned response_len
,
2622 bool header_present
)
2624 const struct brw_device_info
*devinfo
= p
->devinfo
;
2625 struct brw_inst
*insn
;
2627 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2628 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2630 brw_push_insn_state(p
);
2631 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2632 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2633 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2635 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2636 * some surface array is accessed out of bounds.
2638 insn
= brw_AND(p
, addr
,
2639 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2640 BRW_GET_SWZ(surface
.swizzle
, 0)),
2643 brw_pop_insn_state(p
);
2648 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
);
2649 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2650 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2651 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2657 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2660 void *store
= p
->store
;
2661 const struct brw_device_info
*devinfo
= p
->devinfo
;
2665 for (offset
= next_offset(devinfo
, store
, start_offset
);
2666 offset
< p
->next_insn_offset
;
2667 offset
= next_offset(devinfo
, store
, offset
)) {
2668 brw_inst
*insn
= store
+ offset
;
2670 switch (brw_inst_opcode(devinfo
, insn
)) {
2674 case BRW_OPCODE_ENDIF
:
2679 case BRW_OPCODE_ELSE
:
2680 case BRW_OPCODE_WHILE
:
2681 case BRW_OPCODE_HALT
:
2690 /* There is no DO instruction on gen6, so to find the end of the loop
2691 * we have to see if the loop is jumping back before our start
2695 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2697 const struct brw_device_info
*devinfo
= p
->devinfo
;
2699 int scale
= 16 / brw_jump_scale(devinfo
);
2700 void *store
= p
->store
;
2702 assert(devinfo
->gen
>= 6);
2704 /* Always start after the instruction (such as a WHILE) we're trying to fix
2707 for (offset
= next_offset(devinfo
, store
, start_offset
);
2708 offset
< p
->next_insn_offset
;
2709 offset
= next_offset(devinfo
, store
, offset
)) {
2710 brw_inst
*insn
= store
+ offset
;
2712 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2713 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2714 : brw_inst_jip(devinfo
, insn
);
2715 if (offset
+ jip
* scale
<= start_offset
)
2719 assert(!"not reached");
2720 return start_offset
;
2723 /* After program generation, go back and update the UIP and JIP of
2724 * BREAK, CONT, and HALT instructions to their correct locations.
2727 brw_set_uip_jip(struct brw_codegen
*p
)
2729 const struct brw_device_info
*devinfo
= p
->devinfo
;
2731 int br
= brw_jump_scale(devinfo
);
2732 int scale
= 16 / br
;
2733 void *store
= p
->store
;
2735 if (devinfo
->gen
< 6)
2738 for (offset
= 0; offset
< p
->next_insn_offset
;
2739 offset
= next_offset(devinfo
, store
, offset
)) {
2740 brw_inst
*insn
= store
+ offset
;
2742 if (brw_inst_cmpt_control(devinfo
, insn
)) {
2743 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2744 assert(brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_BREAK
&&
2745 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_CONTINUE
&&
2746 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_HALT
);
2750 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2751 switch (brw_inst_opcode(devinfo
, insn
)) {
2752 case BRW_OPCODE_BREAK
:
2753 assert(block_end_offset
!= 0);
2754 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2755 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2756 brw_inst_set_uip(devinfo
, insn
,
2757 (brw_find_loop_end(p
, offset
) - offset
+
2758 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2760 case BRW_OPCODE_CONTINUE
:
2761 assert(block_end_offset
!= 0);
2762 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2763 brw_inst_set_uip(devinfo
, insn
,
2764 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2766 assert(brw_inst_uip(devinfo
, insn
) != 0);
2767 assert(brw_inst_jip(devinfo
, insn
) != 0);
2770 case BRW_OPCODE_ENDIF
: {
2771 int32_t jump
= (block_end_offset
== 0) ?
2772 1 * br
: (block_end_offset
- offset
) / scale
;
2773 if (devinfo
->gen
>= 7)
2774 brw_inst_set_jip(devinfo
, insn
, jump
);
2776 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2780 case BRW_OPCODE_HALT
:
2781 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2783 * "In case of the halt instruction not inside any conditional
2784 * code block, the value of <JIP> and <UIP> should be the
2785 * same. In case of the halt instruction inside conditional code
2786 * block, the <UIP> should be the end of the program, and the
2787 * <JIP> should be end of the most inner conditional code block."
2789 * The uip will have already been set by whoever set up the
2792 if (block_end_offset
== 0) {
2793 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2795 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2797 assert(brw_inst_uip(devinfo
, insn
) != 0);
2798 assert(brw_inst_jip(devinfo
, insn
) != 0);
2804 void brw_ff_sync(struct brw_codegen
*p
,
2805 struct brw_reg dest
,
2806 unsigned msg_reg_nr
,
2807 struct brw_reg src0
,
2809 unsigned response_length
,
2812 const struct brw_device_info
*devinfo
= p
->devinfo
;
2815 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2817 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2818 brw_set_dest(p
, insn
, dest
);
2819 brw_set_src0(p
, insn
, src0
);
2820 brw_set_src1(p
, insn
, brw_imm_d(0));
2822 if (devinfo
->gen
< 6)
2823 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2825 brw_set_ff_sync_message(p
,
2833 * Emit the SEND instruction necessary to generate stream output data on Gen6
2834 * (for transform feedback).
2836 * If send_commit_msg is true, this is the last piece of stream output data
2837 * from this thread, so send the data as a committed write. According to the
2838 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2840 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2841 * writes are complete by sending the final write as a committed write."
2844 brw_svb_write(struct brw_codegen
*p
,
2845 struct brw_reg dest
,
2846 unsigned msg_reg_nr
,
2847 struct brw_reg src0
,
2848 unsigned binding_table_index
,
2849 bool send_commit_msg
)
2853 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2855 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2856 brw_set_dest(p
, insn
, dest
);
2857 brw_set_src0(p
, insn
, src0
);
2858 brw_set_src1(p
, insn
, brw_imm_d(0));
2859 brw_set_dp_write_message(p
, insn
,
2860 binding_table_index
,
2861 0, /* msg_control: ignored */
2862 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2864 true, /* header_present */
2865 0, /* last_render_target: ignored */
2866 send_commit_msg
, /* response_length */
2867 0, /* end_of_thread */
2868 send_commit_msg
); /* send_commit_msg */
2872 brw_surface_payload_size(struct brw_codegen
*p
,
2873 unsigned num_channels
,
2877 if (has_simd4x2
&& brw_inst_access_mode(p
->devinfo
, p
->current
) == BRW_ALIGN_16
)
2879 else if (has_simd16
&& p
->compressed
)
2880 return 2 * num_channels
;
2882 return num_channels
;
2886 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2889 bool response_expected
)
2891 const struct brw_device_info
*devinfo
= p
->devinfo
;
2892 unsigned msg_control
=
2893 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2894 (response_expected
? 1 << 5 : 0); /* Return data expected */
2896 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2897 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2899 msg_control
|= 1 << 4; /* SIMD8 mode */
2901 brw_inst_set_dp_msg_type(devinfo
, insn
,
2902 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2904 brw_inst_set_dp_msg_type(devinfo
, insn
,
2905 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2908 brw_inst_set_dp_msg_type(devinfo
, insn
,
2909 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2912 msg_control
|= 1 << 4; /* SIMD8 mode */
2915 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2919 brw_untyped_atomic(struct brw_codegen
*p
,
2921 struct brw_reg payload
,
2922 struct brw_reg surface
,
2924 unsigned msg_length
,
2925 bool response_expected
)
2927 const struct brw_device_info
*devinfo
= p
->devinfo
;
2928 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2929 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2930 GEN7_SFID_DATAPORT_DATA_CACHE
);
2931 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2932 /* Mask out unused components -- This is especially important in Align16
2933 * mode on generations that don't have native support for SIMD4x2 atomics,
2934 * because unused but enabled components will cause the dataport to perform
2935 * additional atomic operations on the addresses that happen to be in the
2936 * uninitialized Y, Z and W coordinates of the payload.
2938 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2939 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2940 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2941 brw_surface_payload_size(p
, response_expected
,
2942 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2945 brw_set_dp_untyped_atomic_message(
2946 p
, insn
, atomic_op
, response_expected
);
2950 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
2951 struct brw_inst
*insn
,
2952 unsigned num_channels
)
2954 const struct brw_device_info
*devinfo
= p
->devinfo
;
2955 /* Set mask of 32-bit channels to drop. */
2956 unsigned msg_control
= 0xf & (0xf << num_channels
);
2958 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2960 msg_control
|= 1 << 4; /* SIMD16 mode */
2962 msg_control
|= 2 << 4; /* SIMD8 mode */
2965 brw_inst_set_dp_msg_type(devinfo
, insn
,
2966 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2967 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2968 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
2969 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2973 brw_untyped_surface_read(struct brw_codegen
*p
,
2975 struct brw_reg payload
,
2976 struct brw_reg surface
,
2977 unsigned msg_length
,
2978 unsigned num_channels
)
2980 const struct brw_device_info
*devinfo
= p
->devinfo
;
2981 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2982 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2983 GEN7_SFID_DATAPORT_DATA_CACHE
);
2984 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2985 p
, sfid
, dst
, payload
, surface
, msg_length
,
2986 brw_surface_payload_size(p
, num_channels
, true, true),
2989 brw_set_dp_untyped_surface_read_message(
2990 p
, insn
, num_channels
);
2994 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
2995 struct brw_inst
*insn
,
2996 unsigned num_channels
)
2998 const struct brw_device_info
*devinfo
= p
->devinfo
;
2999 /* Set mask of 32-bit channels to drop. */
3000 unsigned msg_control
= 0xf & (0xf << num_channels
);
3002 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3004 msg_control
|= 1 << 4; /* SIMD16 mode */
3006 msg_control
|= 2 << 4; /* SIMD8 mode */
3008 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
3009 msg_control
|= 0 << 4; /* SIMD4x2 mode */
3011 msg_control
|= 2 << 4; /* SIMD8 mode */
3014 brw_inst_set_dp_msg_type(devinfo
, insn
,
3015 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3016 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
3017 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
3018 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3022 brw_untyped_surface_write(struct brw_codegen
*p
,
3023 struct brw_reg payload
,
3024 struct brw_reg surface
,
3025 unsigned msg_length
,
3026 unsigned num_channels
)
3028 const struct brw_device_info
*devinfo
= p
->devinfo
;
3029 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3030 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3031 GEN7_SFID_DATAPORT_DATA_CACHE
);
3032 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3033 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3034 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3035 WRITEMASK_X
: WRITEMASK_XYZW
;
3036 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3037 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3038 payload
, surface
, msg_length
, 0, align1
);
3040 brw_set_dp_untyped_surface_write_message(
3041 p
, insn
, num_channels
);
3045 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
3046 struct brw_inst
*insn
,
3048 bool response_expected
)
3050 const struct brw_device_info
*devinfo
= p
->devinfo
;
3051 unsigned msg_control
=
3052 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3053 (response_expected
? 1 << 5 : 0); /* Return data expected */
3055 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3056 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3057 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3058 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3060 brw_inst_set_dp_msg_type(devinfo
, insn
,
3061 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
3063 brw_inst_set_dp_msg_type(devinfo
, insn
,
3064 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
3068 brw_inst_set_dp_msg_type(devinfo
, insn
,
3069 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
3071 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3072 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3075 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3079 brw_typed_atomic(struct brw_codegen
*p
,
3081 struct brw_reg payload
,
3082 struct brw_reg surface
,
3084 unsigned msg_length
,
3085 bool response_expected
) {
3086 const struct brw_device_info
*devinfo
= p
->devinfo
;
3087 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3088 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3089 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3090 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3091 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3092 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3093 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3094 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
3095 brw_surface_payload_size(p
, response_expected
,
3096 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3099 brw_set_dp_typed_atomic_message(
3100 p
, insn
, atomic_op
, response_expected
);
3104 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3105 struct brw_inst
*insn
,
3106 unsigned num_channels
)
3108 const struct brw_device_info
*devinfo
= p
->devinfo
;
3109 /* Set mask of unused channels. */
3110 unsigned msg_control
= 0xf & (0xf << num_channels
);
3112 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3113 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3114 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3115 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3117 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3120 brw_inst_set_dp_msg_type(devinfo
, insn
,
3121 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3123 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3124 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3125 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3128 brw_inst_set_dp_msg_type(devinfo
, insn
,
3129 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3132 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3136 brw_typed_surface_read(struct brw_codegen
*p
,
3138 struct brw_reg payload
,
3139 struct brw_reg surface
,
3140 unsigned msg_length
,
3141 unsigned num_channels
)
3143 const struct brw_device_info
*devinfo
= p
->devinfo
;
3144 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3145 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3146 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3147 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3148 p
, sfid
, dst
, payload
, surface
, msg_length
,
3149 brw_surface_payload_size(p
, num_channels
,
3150 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3153 brw_set_dp_typed_surface_read_message(
3154 p
, insn
, num_channels
);
3158 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3159 struct brw_inst
*insn
,
3160 unsigned num_channels
)
3162 const struct brw_device_info
*devinfo
= p
->devinfo
;
3163 /* Set mask of unused channels. */
3164 unsigned msg_control
= 0xf & (0xf << num_channels
);
3166 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3167 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3168 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3169 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3171 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3174 brw_inst_set_dp_msg_type(devinfo
, insn
,
3175 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3178 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3179 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3180 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3183 brw_inst_set_dp_msg_type(devinfo
, insn
,
3184 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3187 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3191 brw_typed_surface_write(struct brw_codegen
*p
,
3192 struct brw_reg payload
,
3193 struct brw_reg surface
,
3194 unsigned msg_length
,
3195 unsigned num_channels
)
3197 const struct brw_device_info
*devinfo
= p
->devinfo
;
3198 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3199 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3200 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3201 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3202 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3203 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3204 WRITEMASK_X
: WRITEMASK_XYZW
);
3205 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3206 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3207 payload
, surface
, msg_length
, 0, true);
3209 brw_set_dp_typed_surface_write_message(
3210 p
, insn
, num_channels
);
3214 brw_set_memory_fence_message(struct brw_codegen
*p
,
3215 struct brw_inst
*insn
,
3216 enum brw_message_target sfid
,
3219 const struct brw_device_info
*devinfo
= p
->devinfo
;
3221 brw_set_message_descriptor(p
, insn
, sfid
,
3222 1 /* message length */,
3223 (commit_enable
? 1 : 0) /* response length */,
3224 true /* header present */,
3228 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3229 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3231 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3232 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3235 unreachable("Not reached");
3239 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3243 brw_memory_fence(struct brw_codegen
*p
,
3246 const struct brw_device_info
*devinfo
= p
->devinfo
;
3247 const bool commit_enable
= devinfo
->gen
== 7 && !devinfo
->is_haswell
;
3248 struct brw_inst
*insn
;
3250 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3251 * message doesn't write anything back.
3253 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3254 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3255 brw_set_dest(p
, insn
, dst
);
3256 brw_set_src0(p
, insn
, dst
);
3257 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3260 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3261 /* IVB does typed surface access through the render cache, so we need to
3262 * flush it too. Use a different register so both flushes can be
3263 * pipelined by the hardware.
3265 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3266 brw_set_dest(p
, insn
, offset(dst
, 1));
3267 brw_set_src0(p
, insn
, offset(dst
, 1));
3268 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3271 /* Now write the response of the second message into the response of the
3272 * first to trigger a pipeline stall -- This way future render and data
3273 * cache messages will be properly ordered with respect to past data and
3274 * render cache messages.
3276 brw_push_insn_state(p
);
3277 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3278 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3279 brw_MOV(p
, dst
, offset(dst
, 1));
3280 brw_pop_insn_state(p
);
3285 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3286 struct brw_reg dest
,
3290 struct brw_reg data
,
3291 unsigned msg_length
,
3292 unsigned response_length
)
3294 const struct brw_device_info
*devinfo
= p
->devinfo
;
3295 struct brw_inst
*insn
;
3296 const uint16_t exec_size
= brw_inst_exec_size(devinfo
, p
->current
);
3298 /* brw_send_indirect_message will automatically use a direct send message
3299 * if data is actually immediate.
3301 insn
= brw_send_indirect_message(p
,
3302 GEN7_SFID_PIXEL_INTERPOLATOR
,
3306 brw_inst_set_mlen(devinfo
, insn
, msg_length
);
3307 brw_inst_set_rlen(devinfo
, insn
, response_length
);
3309 brw_inst_set_pi_simd_mode(devinfo
, insn
, exec_size
== BRW_EXECUTE_16
);
3310 brw_inst_set_pi_slot_group(devinfo
, insn
, 0); /* zero unless 32/64px dispatch */
3311 brw_inst_set_pi_nopersp(devinfo
, insn
, noperspective
);
3312 brw_inst_set_pi_message_type(devinfo
, insn
, mode
);
3316 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
)
3318 const struct brw_device_info
*devinfo
= p
->devinfo
;
3321 assert(devinfo
->gen
>= 7);
3323 brw_push_insn_state(p
);
3325 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3326 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3328 if (devinfo
->gen
>= 8) {
3329 /* Getting the first active channel index is easy on Gen8: Just find
3330 * the first bit set in the mask register. The same register exists
3331 * on HSW already but it reads back as all ones when the current
3332 * instruction has execution masking disabled, so it's kind of
3335 inst
= brw_FBL(p
, vec1(dst
),
3336 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
));
3338 /* Quarter control has the effect of magically shifting the value of
3339 * this register. Make sure it's set to zero.
3341 brw_inst_set_qtr_control(devinfo
, inst
, GEN6_COMPRESSION_1Q
);
3343 const struct brw_reg flag
= retype(brw_flag_reg(1, 0),
3344 BRW_REGISTER_TYPE_UD
);
3346 brw_MOV(p
, flag
, brw_imm_ud(0));
3348 /* Run a 16-wide instruction returning zero with execution masking
3349 * and a conditional modifier enabled in order to get the current
3350 * execution mask in f1.0.
3352 inst
= brw_MOV(p
, brw_null_reg(), brw_imm_ud(0));
3353 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_16
);
3354 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3355 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3356 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3358 brw_FBL(p
, vec1(dst
), flag
);
3361 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3363 if (devinfo
->gen
>= 8) {
3364 /* In SIMD4x2 mode the first active channel index is just the
3365 * negation of the first bit of the mask register.
3367 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3368 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3372 /* Overwrite the destination without and with execution masking to
3373 * find out which of the channels is active.
3375 brw_push_insn_state(p
);
3376 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3377 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3380 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3382 brw_pop_insn_state(p
);
3383 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3387 brw_pop_insn_state(p
);
3391 brw_broadcast(struct brw_codegen
*p
,
3396 const struct brw_device_info
*devinfo
= p
->devinfo
;
3397 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3400 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3401 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3403 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3404 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3405 /* Trivial, the source is already uniform or the index is a constant.
3406 * We will typically not get here if the optimizer is doing its job, but
3407 * asserting would be mean.
3409 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3411 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3412 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3415 const struct brw_reg addr
=
3416 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3417 const unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3418 /* Limit in bytes of the signed indirect addressing immediate. */
3419 const unsigned limit
= 512;
3421 brw_push_insn_state(p
);
3422 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3423 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3425 /* Take into account the component size and horizontal stride. */
3426 assert(src
.vstride
== src
.hstride
+ src
.width
);
3427 brw_SHL(p
, addr
, vec1(idx
),
3428 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3431 /* We can only address up to limit bytes using the indirect
3432 * addressing immediate, account for the difference if the source
3433 * register is above this limit.
3435 if (offset
>= limit
)
3436 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3438 brw_pop_insn_state(p
);
3440 /* Use indirect addressing to fetch the specified component. */
3442 retype(brw_vec1_indirect(addr
.subnr
, offset
% limit
),
3445 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3446 * to all bits of a flag register,
3450 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 0, 4, 1));
3451 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3452 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3453 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3455 /* and use predicated SEL to pick the right channel. */
3456 inst
= brw_SEL(p
, dst
,
3457 stride(suboffset(src
, 4), 0, 4, 1),
3458 stride(src
, 0, 4, 1));
3459 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3460 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3466 * This instruction is generated as a single-channel align1 instruction by
3467 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3469 * We can't use the typed atomic op in the FS because that has the execution
3470 * mask ANDed with the pixel mask, but we just want to write the one dword for
3473 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3474 * one u32. So we use the same untyped atomic write message as the pixel
3477 * The untyped atomic operation requires a BUFFER surface type with RAW
3478 * format, and is only accessible through the legacy DATA_CACHE dataport
3481 void brw_shader_time_add(struct brw_codegen
*p
,
3482 struct brw_reg payload
,
3483 uint32_t surf_index
)
3485 const unsigned sfid
= (p
->devinfo
->gen
>= 8 || p
->devinfo
->is_haswell
?
3486 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3487 GEN7_SFID_DATAPORT_DATA_CACHE
);
3488 assert(p
->devinfo
->gen
>= 7);
3490 brw_push_insn_state(p
);
3491 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3492 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3493 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3494 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3496 /* We use brw_vec1_reg and unmasked because we want to increment the given
3499 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3501 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3503 brw_set_src1(p
, send
, brw_imm_ud(0));
3504 brw_set_message_descriptor(p
, send
, sfid
, 2, 0, false, false);
3505 brw_inst_set_binding_table_index(p
->devinfo
, send
, surf_index
);
3506 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3508 brw_pop_insn_state(p
);
3513 * Emit the SEND message for a barrier
3516 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3518 const struct brw_device_info
*devinfo
= p
->devinfo
;
3519 struct brw_inst
*inst
;
3521 assert(devinfo
->gen
>= 7);
3523 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3524 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3525 brw_set_src0(p
, inst
, src
);
3526 brw_set_src1(p
, inst
, brw_null_reg());
3528 brw_set_message_descriptor(p
, inst
, BRW_SFID_MESSAGE_GATEWAY
,
3530 0 /* response_length */,
3531 false /* header_present */,
3532 false /* end_of_thread */);
3534 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3535 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3536 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3538 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3543 * Emit the wait instruction for a barrier
3546 brw_WAIT(struct brw_codegen
*p
)
3548 const struct brw_device_info
*devinfo
= p
->devinfo
;
3549 struct brw_inst
*insn
;
3551 struct brw_reg src
= brw_notification_reg();
3553 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3554 brw_set_dest(p
, insn
, src
);
3555 brw_set_src0(p
, insn
, src
);
3556 brw_set_src1(p
, insn
, brw_null_reg());
3558 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3559 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);