2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_context.h"
34 #include "brw_defines.h"
37 #include "util/ralloc.h"
40 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
41 * registers, implicitly moving the operand to a message register.
43 * On Sandybridge, this is no longer the case. This function performs the
44 * explicit move; it should be called before emitting a SEND instruction.
47 gen6_resolve_implied_move(struct brw_codegen
*p
,
51 const struct brw_device_info
*devinfo
= p
->devinfo
;
55 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
58 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct brw_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 * Convert a brw_reg_type enumeration value into the hardware representation.
91 * The hardware encoding may depend on whether the value is an immediate.
94 brw_reg_type_to_hw_type(const struct brw_device_info
*devinfo
,
95 enum brw_reg_type type
, enum brw_reg_file file
)
97 if (file
== BRW_IMMEDIATE_VALUE
) {
98 static const int imm_hw_types
[] = {
99 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
100 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
101 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
102 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
103 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
104 [BRW_REGISTER_TYPE_UB
] = -1,
105 [BRW_REGISTER_TYPE_B
] = -1,
106 [BRW_REGISTER_TYPE_UV
] = BRW_HW_REG_IMM_TYPE_UV
,
107 [BRW_REGISTER_TYPE_VF
] = BRW_HW_REG_IMM_TYPE_VF
,
108 [BRW_REGISTER_TYPE_V
] = BRW_HW_REG_IMM_TYPE_V
,
109 [BRW_REGISTER_TYPE_DF
] = GEN8_HW_REG_IMM_TYPE_DF
,
110 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_IMM_TYPE_HF
,
111 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
112 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
114 assert(type
< ARRAY_SIZE(imm_hw_types
));
115 assert(imm_hw_types
[type
] != -1);
116 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_DF
);
117 return imm_hw_types
[type
];
119 /* Non-immediate registers */
120 static const int hw_types
[] = {
121 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
122 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
123 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
124 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
125 [BRW_REGISTER_TYPE_UB
] = BRW_HW_REG_NON_IMM_TYPE_UB
,
126 [BRW_REGISTER_TYPE_B
] = BRW_HW_REG_NON_IMM_TYPE_B
,
127 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
128 [BRW_REGISTER_TYPE_UV
] = -1,
129 [BRW_REGISTER_TYPE_VF
] = -1,
130 [BRW_REGISTER_TYPE_V
] = -1,
131 [BRW_REGISTER_TYPE_DF
] = GEN7_HW_REG_NON_IMM_TYPE_DF
,
132 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_NON_IMM_TYPE_HF
,
133 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
134 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
136 assert(type
< ARRAY_SIZE(hw_types
));
137 assert(hw_types
[type
] != -1);
138 assert(devinfo
->gen
>= 7 || type
< BRW_REGISTER_TYPE_DF
);
139 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_HF
);
140 return hw_types
[type
];
145 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
147 const struct brw_device_info
*devinfo
= p
->devinfo
;
149 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
150 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
151 else if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
152 assert(dest
.nr
< 128);
154 gen7_convert_mrf_to_grf(p
, &dest
);
156 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
157 brw_inst_set_dst_reg_type(devinfo
, inst
,
158 brw_reg_type_to_hw_type(devinfo
, dest
.type
,
160 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
162 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
163 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
165 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
166 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
167 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
168 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
169 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
171 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
172 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
173 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
174 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
175 assert(dest
.writemask
!= 0);
177 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
178 * Although Dst.HorzStride is a don't care for Align16, HW needs
179 * this to be programmed as "01".
181 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
184 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
186 /* These are different sizes in align1 vs align16:
188 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
189 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
190 dest
.indirect_offset
);
191 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
192 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
193 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
195 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
196 dest
.indirect_offset
);
197 /* even ignored in da16, still need to set as '01' */
198 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
202 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
203 * or 16 (SIMD16), as that's normally correct. However, when dealing with
204 * small registers, we automatically reduce it to match the register size.
206 * In platforms that support fp64 we can emit instructions with a width of
207 * 4 that need two SIMD8 registers and an exec_size of 8 or 16. In these
208 * cases we need to make sure that these instructions have their exec sizes
209 * set properly when they are emitted and we can't rely on this code to fix
213 if (devinfo
->gen
>= 6)
214 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
216 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
219 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
222 extern int reg_type_size
[];
225 validate_reg(const struct brw_device_info
*devinfo
,
226 brw_inst
*inst
, struct brw_reg reg
)
228 const int hstride_for_reg
[] = {0, 1, 2, 4};
229 const int vstride_for_reg
[] = {0, 1, 2, 4, 8, 16, 32};
230 const int width_for_reg
[] = {1, 2, 4, 8, 16};
231 const int execsize_for_reg
[] = {1, 2, 4, 8, 16, 32};
232 int width
, hstride
, vstride
, execsize
;
234 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
235 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
236 * mean the destination has to be 128-bit aligned and the
237 * destination horiz stride has to be a word.
239 if (reg
.type
== BRW_REGISTER_TYPE_V
) {
240 assert(hstride_for_reg
[brw_inst_dst_hstride(devinfo
, inst
)] *
241 reg_type_size
[brw_inst_dst_reg_type(devinfo
, inst
)] == 2);
247 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
248 reg
.file
== BRW_ARF_NULL
)
251 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
253 * "Swizzling is not allowed when an accumulator is used as an implicit
254 * source or an explicit source in an instruction."
256 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
257 reg
.nr
== BRW_ARF_ACCUMULATOR
)
258 assert(reg
.swizzle
== BRW_SWIZZLE_XYZW
);
260 assert(reg
.hstride
>= 0 && reg
.hstride
< ARRAY_SIZE(hstride_for_reg
));
261 hstride
= hstride_for_reg
[reg
.hstride
];
263 if (reg
.vstride
== 0xf) {
266 assert(reg
.vstride
>= 0 && reg
.vstride
< ARRAY_SIZE(vstride_for_reg
));
267 vstride
= vstride_for_reg
[reg
.vstride
];
270 assert(reg
.width
>= 0 && reg
.width
< ARRAY_SIZE(width_for_reg
));
271 width
= width_for_reg
[reg
.width
];
273 assert(brw_inst_exec_size(devinfo
, inst
) >= 0 &&
274 brw_inst_exec_size(devinfo
, inst
) < ARRAY_SIZE(execsize_for_reg
));
275 execsize
= execsize_for_reg
[brw_inst_exec_size(devinfo
, inst
)];
277 /* Restrictions from 3.3.10: Register Region Restrictions. */
279 assert(execsize
>= width
);
282 if (execsize
== width
&& hstride
!= 0) {
283 assert(vstride
== -1 || vstride
== width
* hstride
);
287 if (execsize
== width
&& hstride
== 0) {
288 /* no restriction on vstride. */
293 assert(hstride
== 0);
297 if (execsize
== 1 && width
== 1) {
298 assert(hstride
== 0);
299 assert(vstride
== 0);
303 if (vstride
== 0 && hstride
== 0) {
307 /* 10. Check destination issues. */
311 is_compactable_immediate(unsigned imm
)
313 /* We get the low 12 bits as-is. */
316 /* We get one bit replicated through the top 20 bits. */
317 return imm
== 0 || imm
== 0xfffff000;
321 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
323 const struct brw_device_info
*devinfo
= p
->devinfo
;
325 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
326 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
327 else if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
328 assert(reg
.nr
< 128);
330 gen7_convert_mrf_to_grf(p
, ®
);
332 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
333 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
334 /* Any source modifiers or regions will be ignored, since this just
335 * identifies the MRF/GRF to start reading the message contents from.
336 * Check for some likely failures.
340 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
343 validate_reg(devinfo
, inst
, reg
);
345 brw_inst_set_src0_reg_file(devinfo
, inst
, reg
.file
);
346 brw_inst_set_src0_reg_type(devinfo
, inst
,
347 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
348 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
349 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
350 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
352 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
353 if (reg
.type
== BRW_REGISTER_TYPE_DF
)
354 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
356 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
358 /* The Bspec's section titled "Non-present Operands" claims that if src0
359 * is an immediate that src1's type must be the same as that of src0.
361 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
362 * that do not follow this rule. E.g., from the IVB/HSW table:
364 * DataTypeIndex 18-Bit Mapping Mapped Meaning
365 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
367 * And from the SNB table:
369 * DataTypeIndex 18-Bit Mapping Mapped Meaning
370 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
372 * Neither of these cause warnings from the simulator when used,
373 * compacted or otherwise. In fact, all compaction mappings that have an
374 * immediate in src0 use a:ud for src1.
376 * The GM45 instruction compaction tables do not contain mapped meanings
377 * so it's not clear whether it has the restriction. We'll assume it was
378 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
380 * Don't do any of this for 64-bit immediates, since the src1 fields
381 * overlap with the immediate and setting them would overwrite the
384 if (type_sz(reg
.type
) < 8) {
385 brw_inst_set_src1_reg_file(devinfo
, inst
,
386 BRW_ARCHITECTURE_REGISTER_FILE
);
387 if (devinfo
->gen
< 6) {
388 brw_inst_set_src1_reg_type(devinfo
, inst
,
389 brw_inst_src0_reg_type(devinfo
, inst
));
391 brw_inst_set_src1_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
395 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
396 * for immediate values. Presumably the hardware engineers realized
397 * that the only useful floating-point value that could be represented
398 * in this format is 0.0, which can also be represented as a VF-typed
399 * immediate, so they gave us the previously mentioned mapping on IVB+.
401 * Strangely, we do have a mapping for imm:f in src1, so we don't need
404 * If we see a 0.0:F, change the type to VF so that it can be compacted.
406 if (brw_inst_imm_ud(devinfo
, inst
) == 0x0 &&
407 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_F
) {
408 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_IMM_TYPE_VF
);
411 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
412 * set the types to :UD so the instruction can be compacted.
414 if (is_compactable_immediate(brw_inst_imm_ud(devinfo
, inst
)) &&
415 brw_inst_cond_modifier(devinfo
, inst
) == BRW_CONDITIONAL_NONE
&&
416 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
&&
417 brw_inst_dst_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
) {
418 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
419 brw_inst_set_dst_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
422 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
423 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
424 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
425 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
427 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
430 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
432 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
433 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
435 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
439 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
440 if (reg
.width
== BRW_WIDTH_1
&&
441 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
442 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
443 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
444 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
446 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
447 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
448 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
451 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
452 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
453 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
454 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
455 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
456 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
457 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
458 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
460 /* This is an oddity of the fact we're using the same
461 * descriptions for registers in align_16 as align_1:
463 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
464 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
466 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
473 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
475 const struct brw_device_info
*devinfo
= p
->devinfo
;
477 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
478 assert(reg
.nr
< 128);
480 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
482 * "Accumulator registers may be accessed explicitly as src0
485 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
486 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
488 gen7_convert_mrf_to_grf(p
, ®
);
489 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
491 validate_reg(devinfo
, inst
, reg
);
493 brw_inst_set_src1_reg_file(devinfo
, inst
, reg
.file
);
494 brw_inst_set_src1_reg_type(devinfo
, inst
,
495 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
496 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
497 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
499 /* Only src1 can be immediate in two-argument instructions.
501 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
503 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
504 /* two-argument instructions can only use 32-bit immediates */
505 assert(type_sz(reg
.type
) < 8);
506 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
508 /* This is a hardware restriction, which may or may not be lifted
511 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
512 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
514 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
515 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
516 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
518 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
521 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
522 if (reg
.width
== BRW_WIDTH_1
&&
523 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
524 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
525 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
526 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
528 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
529 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
530 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
533 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
534 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
535 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
536 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
537 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
538 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
539 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
540 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
542 /* This is an oddity of the fact we're using the same
543 * descriptions for registers in align_16 as align_1:
545 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
546 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
548 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
554 * Set the Message Descriptor and Extended Message Descriptor fields
557 * \note This zeroes out the Function Control bits, so it must be called
558 * \b before filling out any message-specific data. Callers can
559 * choose not to fill in irrelevant bits; they will be zero.
562 brw_set_message_descriptor(struct brw_codegen
*p
,
564 enum brw_message_target sfid
,
566 unsigned response_length
,
570 const struct brw_device_info
*devinfo
= p
->devinfo
;
572 brw_set_src1(p
, inst
, brw_imm_d(0));
574 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
575 * itself; instead, it will be a MOV/OR into the address register.
577 * In this case, we avoid setting the extended message descriptor bits,
578 * since they go on the later SEND/SENDC instead and if set here would
579 * instead clobber the conditionalmod bits.
581 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
582 if (opcode
== BRW_OPCODE_SEND
|| opcode
== BRW_OPCODE_SENDC
) {
583 brw_inst_set_sfid(devinfo
, inst
, sfid
);
586 brw_inst_set_mlen(devinfo
, inst
, msg_length
);
587 brw_inst_set_rlen(devinfo
, inst
, response_length
);
588 brw_inst_set_eot(devinfo
, inst
, end_of_thread
);
590 if (devinfo
->gen
>= 5) {
591 brw_inst_set_header_present(devinfo
, inst
, header_present
);
595 static void brw_set_math_message( struct brw_codegen
*p
,
598 unsigned integer_type
,
602 const struct brw_device_info
*devinfo
= p
->devinfo
;
604 unsigned response_length
;
606 /* Infer message length from the function */
608 case BRW_MATH_FUNCTION_POW
:
609 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
610 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
611 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
619 /* Infer response length from the function */
621 case BRW_MATH_FUNCTION_SINCOS
:
622 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
631 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
632 msg_length
, response_length
, false, false);
633 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
634 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
635 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
636 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
637 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
638 brw_inst_set_saturate(devinfo
, inst
, 0);
642 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
645 unsigned response_length
,
648 const struct brw_device_info
*devinfo
= p
->devinfo
;
650 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
651 1, response_length
, true, end_of_thread
);
652 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
653 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
654 /* The following fields are not used by FF_SYNC: */
655 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
656 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
657 brw_inst_set_urb_used(devinfo
, insn
, 0);
658 brw_inst_set_urb_complete(devinfo
, insn
, 0);
661 static void brw_set_urb_message( struct brw_codegen
*p
,
663 enum brw_urb_write_flags flags
,
665 unsigned response_length
,
667 unsigned swizzle_control
)
669 const struct brw_device_info
*devinfo
= p
->devinfo
;
671 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
672 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
673 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
675 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
676 msg_length
, response_length
, true,
677 flags
& BRW_URB_WRITE_EOT
);
679 if (flags
& BRW_URB_WRITE_OWORD
) {
680 assert(msg_length
== 2); /* header + one OWORD of data */
681 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
683 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
686 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
687 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
689 if (devinfo
->gen
< 8) {
690 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
693 if (devinfo
->gen
< 7) {
694 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
695 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
697 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
698 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
703 brw_set_dp_write_message(struct brw_codegen
*p
,
705 unsigned binding_table_index
,
706 unsigned msg_control
,
710 unsigned last_render_target
,
711 unsigned response_length
,
712 unsigned end_of_thread
,
713 unsigned send_commit_msg
)
715 const struct brw_device_info
*devinfo
= p
->devinfo
;
718 if (devinfo
->gen
>= 7) {
719 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
720 if (msg_type
== GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
)
721 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
723 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
724 } else if (devinfo
->gen
== 6) {
725 /* Use the render cache for all write messages. */
726 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
728 sfid
= BRW_SFID_DATAPORT_WRITE
;
731 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
732 header_present
, end_of_thread
);
734 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
735 brw_inst_set_dp_write_msg_type(devinfo
, insn
, msg_type
);
736 brw_inst_set_dp_write_msg_control(devinfo
, insn
, msg_control
);
737 brw_inst_set_rt_last(devinfo
, insn
, last_render_target
);
738 if (devinfo
->gen
< 7) {
739 brw_inst_set_dp_write_commit(devinfo
, insn
, send_commit_msg
);
744 brw_set_dp_read_message(struct brw_codegen
*p
,
746 unsigned binding_table_index
,
747 unsigned msg_control
,
749 unsigned target_cache
,
752 unsigned response_length
)
754 const struct brw_device_info
*devinfo
= p
->devinfo
;
757 if (devinfo
->gen
>= 7) {
758 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
759 } else if (devinfo
->gen
== 6) {
760 if (target_cache
== BRW_DATAPORT_READ_TARGET_RENDER_CACHE
)
761 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
763 sfid
= GEN6_SFID_DATAPORT_SAMPLER_CACHE
;
765 sfid
= BRW_SFID_DATAPORT_READ
;
768 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
769 header_present
, false);
771 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
772 brw_inst_set_dp_read_msg_type(devinfo
, insn
, msg_type
);
773 brw_inst_set_dp_read_msg_control(devinfo
, insn
, msg_control
);
774 if (devinfo
->gen
< 6)
775 brw_inst_set_dp_read_target_cache(devinfo
, insn
, target_cache
);
779 brw_set_sampler_message(struct brw_codegen
*p
,
781 unsigned binding_table_index
,
784 unsigned response_length
,
786 unsigned header_present
,
788 unsigned return_format
)
790 const struct brw_device_info
*devinfo
= p
->devinfo
;
792 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
793 response_length
, header_present
, false);
795 brw_inst_set_binding_table_index(devinfo
, inst
, binding_table_index
);
796 brw_inst_set_sampler(devinfo
, inst
, sampler
);
797 brw_inst_set_sampler_msg_type(devinfo
, inst
, msg_type
);
798 if (devinfo
->gen
>= 5) {
799 brw_inst_set_sampler_simd_mode(devinfo
, inst
, simd_mode
);
800 } else if (devinfo
->gen
== 4 && !devinfo
->is_g4x
) {
801 brw_inst_set_sampler_return_format(devinfo
, inst
, return_format
);
806 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
810 bool invalidate_after_read
,
812 unsigned addr_offset
,
817 const struct brw_device_info
*devinfo
= p
->devinfo
;
818 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
819 (devinfo
->gen
>= 8 && num_regs
== 8));
820 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
821 mlen
, rlen
, header_present
, false);
822 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
823 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
824 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
825 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
826 brw_inst_set_scratch_block_size(devinfo
, inst
, ffs(num_regs
) - 1);
827 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
830 #define next_insn brw_next_insn
832 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
834 const struct brw_device_info
*devinfo
= p
->devinfo
;
837 if (p
->nr_insn
+ 1 > p
->store_size
) {
839 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
842 p
->next_insn_offset
+= 16;
843 insn
= &p
->store
[p
->nr_insn
++];
844 memcpy(insn
, p
->current
, sizeof(*insn
));
846 brw_inst_set_opcode(devinfo
, insn
, opcode
);
851 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
852 struct brw_reg dest
, struct brw_reg src
)
854 brw_inst
*insn
= next_insn(p
, opcode
);
855 brw_set_dest(p
, insn
, dest
);
856 brw_set_src0(p
, insn
, src
);
861 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
862 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
864 /* 64-bit immediates are only supported on 1-src instructions */
865 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
866 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
868 brw_inst
*insn
= next_insn(p
, opcode
);
869 brw_set_dest(p
, insn
, dest
);
870 brw_set_src0(p
, insn
, src0
);
871 brw_set_src1(p
, insn
, src1
);
876 get_3src_subreg_nr(struct brw_reg reg
)
878 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
879 * use 32-bit units (components 0..7). Since they only support F/D/UD
880 * types, this doesn't lose any flexibility, but uses fewer bits.
882 return reg
.subnr
/ 4;
886 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
887 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
889 const struct brw_device_info
*devinfo
= p
->devinfo
;
890 brw_inst
*inst
= next_insn(p
, opcode
);
892 gen7_convert_mrf_to_grf(p
, &dest
);
894 assert(brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
);
896 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
897 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
898 assert(dest
.nr
< 128);
899 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
900 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
901 dest
.type
== BRW_REGISTER_TYPE_DF
||
902 dest
.type
== BRW_REGISTER_TYPE_D
||
903 dest
.type
== BRW_REGISTER_TYPE_UD
);
904 if (devinfo
->gen
== 6) {
905 brw_inst_set_3src_dst_reg_file(devinfo
, inst
,
906 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
908 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
909 brw_inst_set_3src_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
910 brw_inst_set_3src_dst_writemask(devinfo
, inst
, dest
.writemask
);
912 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
913 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
914 assert(src0
.nr
< 128);
915 brw_inst_set_3src_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
916 brw_inst_set_3src_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
917 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
918 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
919 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
920 brw_inst_set_3src_src0_rep_ctrl(devinfo
, inst
,
921 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
923 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
924 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
925 assert(src1
.nr
< 128);
926 brw_inst_set_3src_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
927 brw_inst_set_3src_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
928 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
929 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
930 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
931 brw_inst_set_3src_src1_rep_ctrl(devinfo
, inst
,
932 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
934 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
935 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
936 assert(src2
.nr
< 128);
937 brw_inst_set_3src_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
938 brw_inst_set_3src_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
939 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
940 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
941 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
942 brw_inst_set_3src_src2_rep_ctrl(devinfo
, inst
,
943 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
945 if (devinfo
->gen
>= 7) {
946 /* Set both the source and destination types based on dest.type,
947 * ignoring the source register types. The MAD and LRP emitters ensure
948 * that all four types are float. The BFE and BFI2 emitters, however,
949 * may send us mixed D and UD types and want us to ignore that and use
950 * the destination type.
953 case BRW_REGISTER_TYPE_F
:
954 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
955 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
957 case BRW_REGISTER_TYPE_DF
:
958 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_DF
);
959 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_DF
);
961 case BRW_REGISTER_TYPE_D
:
962 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
963 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
965 case BRW_REGISTER_TYPE_UD
:
966 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
967 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
970 unreachable("not reached");
978 /***********************************************************************
979 * Convenience routines.
982 brw_inst *brw_##OP(struct brw_codegen *p, \
983 struct brw_reg dest, \
984 struct brw_reg src0) \
986 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
990 brw_inst *brw_##OP(struct brw_codegen *p, \
991 struct brw_reg dest, \
992 struct brw_reg src0, \
993 struct brw_reg src1) \
995 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
999 brw_inst *brw_##OP(struct brw_codegen *p, \
1000 struct brw_reg dest, \
1001 struct brw_reg src0, \
1002 struct brw_reg src1, \
1003 struct brw_reg src2) \
1005 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1009 brw_inst *brw_##OP(struct brw_codegen *p, \
1010 struct brw_reg dest, \
1011 struct brw_reg src0, \
1012 struct brw_reg src1, \
1013 struct brw_reg src2) \
1015 assert(dest.type == BRW_REGISTER_TYPE_F || \
1016 dest.type == BRW_REGISTER_TYPE_DF); \
1017 if (dest.type == BRW_REGISTER_TYPE_F) { \
1018 assert(src0.type == BRW_REGISTER_TYPE_F); \
1019 assert(src1.type == BRW_REGISTER_TYPE_F); \
1020 assert(src2.type == BRW_REGISTER_TYPE_F); \
1021 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
1022 assert(src0.type == BRW_REGISTER_TYPE_DF); \
1023 assert(src1.type == BRW_REGISTER_TYPE_DF); \
1024 assert(src2.type == BRW_REGISTER_TYPE_DF); \
1026 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1029 /* Rounding operations (other than RNDD) require two instructions - the first
1030 * stores a rounded value (possibly the wrong way) in the dest register, but
1031 * also sets a per-channel "increment bit" in the flag register. A predicated
1032 * add of 1.0 fixes dest to contain the desired result.
1034 * Sandybridge and later appear to round correctly without an ADD.
1037 void brw_##OP(struct brw_codegen *p, \
1038 struct brw_reg dest, \
1039 struct brw_reg src) \
1041 const struct brw_device_info *devinfo = p->devinfo; \
1042 brw_inst *rnd, *add; \
1043 rnd = next_insn(p, BRW_OPCODE_##OP); \
1044 brw_set_dest(p, rnd, dest); \
1045 brw_set_src0(p, rnd, src); \
1047 if (devinfo->gen < 6) { \
1048 /* turn on round-increments */ \
1049 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
1050 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1051 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1091 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1092 struct brw_reg src0
, struct brw_reg src1
)
1095 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1096 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1097 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1098 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1099 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1102 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1103 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1104 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1105 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1106 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1109 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1113 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1114 struct brw_reg src0
, struct brw_reg src1
)
1116 assert(dest
.type
== src0
.type
);
1117 assert(src0
.type
== src1
.type
);
1118 switch (src0
.type
) {
1119 case BRW_REGISTER_TYPE_B
:
1120 case BRW_REGISTER_TYPE_UB
:
1121 case BRW_REGISTER_TYPE_W
:
1122 case BRW_REGISTER_TYPE_UW
:
1123 case BRW_REGISTER_TYPE_D
:
1124 case BRW_REGISTER_TYPE_UD
:
1127 unreachable("Bad type for brw_AVG");
1130 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1134 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1135 struct brw_reg src0
, struct brw_reg src1
)
1138 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1139 src0
.type
== BRW_REGISTER_TYPE_UD
||
1140 src1
.type
== BRW_REGISTER_TYPE_D
||
1141 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1142 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1145 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1146 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1147 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1148 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1149 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1152 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1153 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1154 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1155 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1156 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1159 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1160 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1161 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1162 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1164 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1168 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1169 struct brw_reg src0
, struct brw_reg src1
)
1171 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1172 src0
.width
= BRW_WIDTH_1
;
1173 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1174 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1178 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1179 struct brw_reg src0
, struct brw_reg src1
)
1181 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1182 src0
.width
= BRW_WIDTH_1
;
1183 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1184 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1185 src1
.width
= BRW_WIDTH_8
;
1186 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1187 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1191 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1193 const struct brw_device_info
*devinfo
= p
->devinfo
;
1194 const bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1195 /* The F32TO16 instruction doesn't support 32-bit destination types in
1196 * Align1 mode, and neither does the Gen8 implementation in terms of a
1197 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1198 * an undocumented feature.
1200 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1201 (!align16
|| devinfo
->gen
>= 8));
1205 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1207 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1208 dst
.type
== BRW_REGISTER_TYPE_W
||
1209 dst
.type
== BRW_REGISTER_TYPE_UW
||
1210 dst
.type
== BRW_REGISTER_TYPE_HF
);
1213 brw_push_insn_state(p
);
1215 if (needs_zero_fill
) {
1216 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1217 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1220 if (devinfo
->gen
>= 8) {
1221 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1223 assert(devinfo
->gen
== 7);
1224 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1227 if (needs_zero_fill
) {
1228 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1229 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_ud(0u));
1230 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1233 brw_pop_insn_state(p
);
1238 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1240 const struct brw_device_info
*devinfo
= p
->devinfo
;
1241 bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1244 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1246 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1248 * Because this instruction does not have a 16-bit floating-point
1249 * type, the source data type must be Word (W). The destination type
1250 * must be F (Float).
1252 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1253 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1255 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1256 src
.type
== BRW_REGISTER_TYPE_UW
||
1257 src
.type
== BRW_REGISTER_TYPE_HF
);
1260 if (devinfo
->gen
>= 8) {
1261 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1263 assert(devinfo
->gen
== 7);
1264 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1269 void brw_NOP(struct brw_codegen
*p
)
1271 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1272 brw_inst_set_exec_size(p
->devinfo
, insn
, BRW_EXECUTE_1
);
1273 brw_set_dest(p
, insn
, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD
));
1274 brw_set_src0(p
, insn
, retype(brw_vec1_grf(0,0), BRW_REGISTER_TYPE_UD
));
1275 brw_set_src1(p
, insn
, brw_imm_ud(0x0));
1282 /***********************************************************************
1283 * Comparisons, if/else/endif
1287 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1288 unsigned predicate_control
)
1290 const struct brw_device_info
*devinfo
= p
->devinfo
;
1291 struct brw_reg ip
= brw_ip_reg();
1292 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1294 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_2
);
1295 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1296 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1297 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1303 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1305 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1307 p
->if_stack_depth
++;
1308 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1309 p
->if_stack_array_size
*= 2;
1310 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1311 p
->if_stack_array_size
);
1316 pop_if_stack(struct brw_codegen
*p
)
1318 p
->if_stack_depth
--;
1319 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1323 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1325 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1326 p
->loop_stack_array_size
*= 2;
1327 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1328 p
->loop_stack_array_size
);
1329 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1330 p
->loop_stack_array_size
);
1333 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1334 p
->loop_stack_depth
++;
1335 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1339 get_inner_do_insn(struct brw_codegen
*p
)
1341 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1344 /* EU takes the value from the flag register and pushes it onto some
1345 * sort of a stack (presumably merging with any flag value already on
1346 * the stack). Within an if block, the flags at the top of the stack
1347 * control execution on each channel of the unit, eg. on each of the
1348 * 16 pixel values in our wm programs.
1350 * When the matching 'else' instruction is reached (presumably by
1351 * countdown of the instruction count patched in by our ELSE/ENDIF
1352 * functions), the relevant flags are inverted.
1354 * When the matching 'endif' instruction is reached, the flags are
1355 * popped off. If the stack is now empty, normal execution resumes.
1358 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1360 const struct brw_device_info
*devinfo
= p
->devinfo
;
1363 insn
= next_insn(p
, BRW_OPCODE_IF
);
1365 /* Override the defaults for this instruction:
1367 if (devinfo
->gen
< 6) {
1368 brw_set_dest(p
, insn
, brw_ip_reg());
1369 brw_set_src0(p
, insn
, brw_ip_reg());
1370 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1371 } else if (devinfo
->gen
== 6) {
1372 brw_set_dest(p
, insn
, brw_imm_w(0));
1373 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1374 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1375 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1376 } else if (devinfo
->gen
== 7) {
1377 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1378 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1379 brw_set_src1(p
, insn
, brw_imm_w(0));
1380 brw_inst_set_jip(devinfo
, insn
, 0);
1381 brw_inst_set_uip(devinfo
, insn
, 0);
1383 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1384 brw_set_src0(p
, insn
, brw_imm_d(0));
1385 brw_inst_set_jip(devinfo
, insn
, 0);
1386 brw_inst_set_uip(devinfo
, insn
, 0);
1389 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1390 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1391 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1392 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1393 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1394 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1396 push_if_stack(p
, insn
);
1397 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1401 /* This function is only used for gen6-style IF instructions with an
1402 * embedded comparison (conditional modifier). It is not used on gen7.
1405 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1406 struct brw_reg src0
, struct brw_reg src1
)
1408 const struct brw_device_info
*devinfo
= p
->devinfo
;
1411 insn
= next_insn(p
, BRW_OPCODE_IF
);
1413 brw_set_dest(p
, insn
, brw_imm_w(0));
1414 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1416 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1417 brw_set_src0(p
, insn
, src0
);
1418 brw_set_src1(p
, insn
, src1
);
1420 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1421 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1422 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1424 push_if_stack(p
, insn
);
1429 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1432 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1433 brw_inst
*if_inst
, brw_inst
*else_inst
)
1435 const struct brw_device_info
*devinfo
= p
->devinfo
;
1437 /* The next instruction (where the ENDIF would be, if it existed) */
1438 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1440 assert(p
->single_program_flow
);
1441 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1442 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1443 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1445 /* Convert IF to an ADD instruction that moves the instruction pointer
1446 * to the first instruction of the ELSE block. If there is no ELSE
1447 * block, point to where ENDIF would be. Reverse the predicate.
1449 * There's no need to execute an ENDIF since we don't need to do any
1450 * stack operations, and if we're currently executing, we just want to
1451 * continue normally.
1453 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1454 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1456 if (else_inst
!= NULL
) {
1457 /* Convert ELSE to an ADD instruction that points where the ENDIF
1460 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1462 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1463 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1465 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1470 * Patch IF and ELSE instructions with appropriate jump targets.
1473 patch_IF_ELSE(struct brw_codegen
*p
,
1474 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1476 const struct brw_device_info
*devinfo
= p
->devinfo
;
1478 /* We shouldn't be patching IF and ELSE instructions in single program flow
1479 * mode when gen < 6, because in single program flow mode on those
1480 * platforms, we convert flow control instructions to conditional ADDs that
1481 * operate on IP (see brw_ENDIF).
1483 * However, on Gen6, writing to IP doesn't work in single program flow mode
1484 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1485 * not be updated by non-flow control instructions."). And on later
1486 * platforms, there is no significant benefit to converting control flow
1487 * instructions to conditional ADDs. So we do patch IF and ELSE
1488 * instructions in single program flow mode on those platforms.
1490 if (devinfo
->gen
< 6)
1491 assert(!p
->single_program_flow
);
1493 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1494 assert(endif_inst
!= NULL
);
1495 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1497 unsigned br
= brw_jump_scale(devinfo
);
1499 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1500 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1502 if (else_inst
== NULL
) {
1503 /* Patch IF -> ENDIF */
1504 if (devinfo
->gen
< 6) {
1505 /* Turn it into an IFF, which means no mask stack operations for
1506 * all-false and jumping past the ENDIF.
1508 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1509 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1510 br
* (endif_inst
- if_inst
+ 1));
1511 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1512 } else if (devinfo
->gen
== 6) {
1513 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1514 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1516 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1517 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1520 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1522 /* Patch IF -> ELSE */
1523 if (devinfo
->gen
< 6) {
1524 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1525 br
* (else_inst
- if_inst
));
1526 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1527 } else if (devinfo
->gen
== 6) {
1528 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1529 br
* (else_inst
- if_inst
+ 1));
1532 /* Patch ELSE -> ENDIF */
1533 if (devinfo
->gen
< 6) {
1534 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1537 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1538 br
* (endif_inst
- else_inst
+ 1));
1539 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1540 } else if (devinfo
->gen
== 6) {
1541 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1542 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1543 br
* (endif_inst
- else_inst
));
1545 /* The IF instruction's JIP should point just past the ELSE */
1546 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1547 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1548 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1549 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1550 if (devinfo
->gen
>= 8) {
1551 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1552 * should point to ENDIF.
1554 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1561 brw_ELSE(struct brw_codegen
*p
)
1563 const struct brw_device_info
*devinfo
= p
->devinfo
;
1566 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1568 if (devinfo
->gen
< 6) {
1569 brw_set_dest(p
, insn
, brw_ip_reg());
1570 brw_set_src0(p
, insn
, brw_ip_reg());
1571 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1572 } else if (devinfo
->gen
== 6) {
1573 brw_set_dest(p
, insn
, brw_imm_w(0));
1574 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1575 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1576 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1577 } else if (devinfo
->gen
== 7) {
1578 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1579 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1580 brw_set_src1(p
, insn
, brw_imm_w(0));
1581 brw_inst_set_jip(devinfo
, insn
, 0);
1582 brw_inst_set_uip(devinfo
, insn
, 0);
1584 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1585 brw_set_src0(p
, insn
, brw_imm_d(0));
1586 brw_inst_set_jip(devinfo
, insn
, 0);
1587 brw_inst_set_uip(devinfo
, insn
, 0);
1590 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1591 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1592 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1593 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1595 push_if_stack(p
, insn
);
1599 brw_ENDIF(struct brw_codegen
*p
)
1601 const struct brw_device_info
*devinfo
= p
->devinfo
;
1602 brw_inst
*insn
= NULL
;
1603 brw_inst
*else_inst
= NULL
;
1604 brw_inst
*if_inst
= NULL
;
1606 bool emit_endif
= true;
1608 /* In single program flow mode, we can express IF and ELSE instructions
1609 * equivalently as ADD instructions that operate on IP. On platforms prior
1610 * to Gen6, flow control instructions cause an implied thread switch, so
1611 * this is a significant savings.
1613 * However, on Gen6, writing to IP doesn't work in single program flow mode
1614 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1615 * not be updated by non-flow control instructions."). And on later
1616 * platforms, there is no significant benefit to converting control flow
1617 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1620 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1624 * A single next_insn() may change the base address of instruction store
1625 * memory(p->store), so call it first before referencing the instruction
1626 * store pointer from an index
1629 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1631 /* Pop the IF and (optional) ELSE instructions from the stack */
1632 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1633 tmp
= pop_if_stack(p
);
1634 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1636 tmp
= pop_if_stack(p
);
1641 /* ENDIF is useless; don't bother emitting it. */
1642 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1646 if (devinfo
->gen
< 6) {
1647 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1648 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1649 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1650 } else if (devinfo
->gen
== 6) {
1651 brw_set_dest(p
, insn
, brw_imm_w(0));
1652 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1653 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1654 } else if (devinfo
->gen
== 7) {
1655 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1656 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1657 brw_set_src1(p
, insn
, brw_imm_w(0));
1659 brw_set_src0(p
, insn
, brw_imm_d(0));
1662 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1663 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1664 if (devinfo
->gen
< 6)
1665 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1667 /* Also pop item off the stack in the endif instruction: */
1668 if (devinfo
->gen
< 6) {
1669 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1670 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1671 } else if (devinfo
->gen
== 6) {
1672 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1674 brw_inst_set_jip(devinfo
, insn
, 2);
1676 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1680 brw_BREAK(struct brw_codegen
*p
)
1682 const struct brw_device_info
*devinfo
= p
->devinfo
;
1685 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1686 if (devinfo
->gen
>= 8) {
1687 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1688 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1689 } else if (devinfo
->gen
>= 6) {
1690 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1691 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1692 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1694 brw_set_dest(p
, insn
, brw_ip_reg());
1695 brw_set_src0(p
, insn
, brw_ip_reg());
1696 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1697 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1698 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1700 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1701 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1708 brw_CONT(struct brw_codegen
*p
)
1710 const struct brw_device_info
*devinfo
= p
->devinfo
;
1713 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1714 brw_set_dest(p
, insn
, brw_ip_reg());
1715 if (devinfo
->gen
>= 8) {
1716 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1718 brw_set_src0(p
, insn
, brw_ip_reg());
1719 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1722 if (devinfo
->gen
< 6) {
1723 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1724 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1726 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1727 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1733 gen6_HALT(struct brw_codegen
*p
)
1735 const struct brw_device_info
*devinfo
= p
->devinfo
;
1738 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1739 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1740 if (devinfo
->gen
>= 8) {
1741 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1743 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1744 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1747 if (p
->compressed
) {
1748 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_16
);
1750 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1751 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_8
);
1758 * The DO/WHILE is just an unterminated loop -- break or continue are
1759 * used for control within the loop. We have a few ways they can be
1762 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1763 * jip and no DO instruction.
1765 * For non-uniform control flow pre-gen6, there's a DO instruction to
1766 * push the mask, and a WHILE to jump back, and BREAK to get out and
1769 * For gen6, there's no more mask stack, so no need for DO. WHILE
1770 * just points back to the first instruction of the loop.
1773 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1775 const struct brw_device_info
*devinfo
= p
->devinfo
;
1777 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1778 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1779 return &p
->store
[p
->nr_insn
];
1781 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1783 push_loop_stack(p
, insn
);
1785 /* Override the defaults for this instruction:
1787 brw_set_dest(p
, insn
, brw_null_reg());
1788 brw_set_src0(p
, insn
, brw_null_reg());
1789 brw_set_src1(p
, insn
, brw_null_reg());
1791 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1792 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1793 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1800 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1803 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1804 * nesting, since it can always just point to the end of the block/current loop.
1807 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1809 const struct brw_device_info
*devinfo
= p
->devinfo
;
1810 brw_inst
*do_inst
= get_inner_do_insn(p
);
1812 unsigned br
= brw_jump_scale(devinfo
);
1814 assert(devinfo
->gen
< 6);
1816 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1817 /* If the jump count is != 0, that means that this instruction has already
1818 * been patched because it's part of a loop inside of the one we're
1821 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1822 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1823 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1824 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1825 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1826 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1832 brw_WHILE(struct brw_codegen
*p
)
1834 const struct brw_device_info
*devinfo
= p
->devinfo
;
1835 brw_inst
*insn
, *do_insn
;
1836 unsigned br
= brw_jump_scale(devinfo
);
1838 if (devinfo
->gen
>= 6) {
1839 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1840 do_insn
= get_inner_do_insn(p
);
1842 if (devinfo
->gen
>= 8) {
1843 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1844 brw_set_src0(p
, insn
, brw_imm_d(0));
1845 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1846 } else if (devinfo
->gen
== 7) {
1847 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1848 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1849 brw_set_src1(p
, insn
, brw_imm_w(0));
1850 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1852 brw_set_dest(p
, insn
, brw_imm_w(0));
1853 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1854 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1855 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1858 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1861 if (p
->single_program_flow
) {
1862 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1863 do_insn
= get_inner_do_insn(p
);
1865 brw_set_dest(p
, insn
, brw_ip_reg());
1866 brw_set_src0(p
, insn
, brw_ip_reg());
1867 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1868 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1870 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1871 do_insn
= get_inner_do_insn(p
);
1873 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1875 brw_set_dest(p
, insn
, brw_ip_reg());
1876 brw_set_src0(p
, insn
, brw_ip_reg());
1877 brw_set_src1(p
, insn
, brw_imm_d(0));
1879 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1880 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1881 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1883 brw_patch_break_cont(p
, insn
);
1886 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1888 p
->loop_stack_depth
--;
1895 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1897 const struct brw_device_info
*devinfo
= p
->devinfo
;
1898 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1901 if (devinfo
->gen
>= 5)
1904 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1905 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1907 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1908 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1911 /* To integrate with the above, it makes sense that the comparison
1912 * instruction should populate the flag register. It might be simpler
1913 * just to use the flag reg for most WM tasks?
1915 void brw_CMP(struct brw_codegen
*p
,
1916 struct brw_reg dest
,
1917 unsigned conditional
,
1918 struct brw_reg src0
,
1919 struct brw_reg src1
)
1921 const struct brw_device_info
*devinfo
= p
->devinfo
;
1922 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1924 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1925 brw_set_dest(p
, insn
, dest
);
1926 brw_set_src0(p
, insn
, src0
);
1927 brw_set_src1(p
, insn
, src1
);
1929 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1931 * "Any CMP instruction with a null destination must use a {switch}."
1933 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1934 * mentioned on their work-arounds pages.
1936 if (devinfo
->gen
== 7) {
1937 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1938 dest
.nr
== BRW_ARF_NULL
) {
1939 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1944 /***********************************************************************
1945 * Helpers for the various SEND message types:
1948 /** Extended math function, float[8].
1950 void gen4_math(struct brw_codegen
*p
,
1951 struct brw_reg dest
,
1953 unsigned msg_reg_nr
,
1955 unsigned precision
)
1957 const struct brw_device_info
*devinfo
= p
->devinfo
;
1958 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1960 if (has_scalar_region(src
)) {
1961 data_type
= BRW_MATH_DATA_SCALAR
;
1963 data_type
= BRW_MATH_DATA_VECTOR
;
1966 assert(devinfo
->gen
< 6);
1968 /* Example code doesn't set predicate_control for send
1971 brw_inst_set_pred_control(devinfo
, insn
, 0);
1972 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1974 brw_set_dest(p
, insn
, dest
);
1975 brw_set_src0(p
, insn
, src
);
1976 brw_set_math_message(p
,
1979 src
.type
== BRW_REGISTER_TYPE_D
,
1984 void gen6_math(struct brw_codegen
*p
,
1985 struct brw_reg dest
,
1987 struct brw_reg src0
,
1988 struct brw_reg src1
)
1990 const struct brw_device_info
*devinfo
= p
->devinfo
;
1991 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1993 assert(devinfo
->gen
>= 6);
1995 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1996 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1997 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
1998 (devinfo
->gen
>= 8 && src0
.file
== BRW_IMMEDIATE_VALUE
));
2000 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2001 if (devinfo
->gen
== 6) {
2002 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2003 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2006 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2007 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2008 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2009 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2010 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2011 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2012 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2014 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
2015 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
2016 if (function
== BRW_MATH_FUNCTION_POW
) {
2017 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2018 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2020 assert(src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
2021 src1
.nr
== BRW_ARF_NULL
);
2025 /* Source modifiers are ignored for extended math instructions on Gen6. */
2026 if (devinfo
->gen
== 6) {
2027 assert(!src0
.negate
);
2029 assert(!src1
.negate
);
2033 brw_inst_set_math_function(devinfo
, insn
, function
);
2035 brw_set_dest(p
, insn
, dest
);
2036 brw_set_src0(p
, insn
, src0
);
2037 brw_set_src1(p
, insn
, src1
);
2041 * Return the right surface index to access the thread scratch space using
2042 * stateless dataport messages.
2045 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2047 /* The scratch space is thread-local so IA coherency is unnecessary. */
2048 if (p
->devinfo
->gen
>= 8)
2049 return GEN8_BTI_STATELESS_NON_COHERENT
;
2051 return BRW_BTI_STATELESS
;
2055 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2056 * using a constant offset per channel.
2058 * The offset must be aligned to oword size (16 bytes). Used for
2059 * register spilling.
2061 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2066 const struct brw_device_info
*devinfo
= p
->devinfo
;
2067 uint32_t msg_control
, msg_type
;
2070 if (devinfo
->gen
>= 6)
2073 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2075 if (num_regs
== 1) {
2076 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2079 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2083 /* Set up the message header. This is g0, with g0.2 filled with
2084 * the offset. We don't want to leave our offset around in g0 or
2085 * it'll screw up texture samples, so set it up inside the message
2089 brw_push_insn_state(p
);
2090 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2091 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2092 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2094 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2096 /* set message header global offset field (reg 0, element 2) */
2098 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2100 2), BRW_REGISTER_TYPE_UD
),
2101 brw_imm_ud(offset
));
2103 brw_pop_insn_state(p
);
2107 struct brw_reg dest
;
2108 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2109 int send_commit_msg
;
2110 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2111 BRW_REGISTER_TYPE_UW
);
2113 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_NONE
) {
2114 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2115 src_header
= vec16(src_header
);
2117 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2118 if (devinfo
->gen
< 6)
2119 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2121 /* Until gen6, writes followed by reads from the same location
2122 * are not guaranteed to be ordered unless write_commit is set.
2123 * If set, then a no-op write is issued to the destination
2124 * register to set a dependency, and a read from the destination
2125 * can be used to ensure the ordering.
2127 * For gen6, only writes between different threads need ordering
2128 * protection. Our use of DP writes is all about register
2129 * spilling within a thread.
2131 if (devinfo
->gen
>= 6) {
2132 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2133 send_commit_msg
= 0;
2136 send_commit_msg
= 1;
2139 brw_set_dest(p
, insn
, dest
);
2140 if (devinfo
->gen
>= 6) {
2141 brw_set_src0(p
, insn
, mrf
);
2143 brw_set_src0(p
, insn
, brw_null_reg());
2146 if (devinfo
->gen
>= 6)
2147 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2149 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2151 brw_set_dp_write_message(p
,
2153 brw_scratch_surface_idx(p
),
2157 true, /* header_present */
2158 0, /* not a render target */
2159 send_commit_msg
, /* response_length */
2167 * Read a block of owords (half a GRF each) from the scratch buffer
2168 * using a constant index per channel.
2170 * Offset must be aligned to oword size (16 bytes). Used for register
2174 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2175 struct brw_reg dest
,
2180 const struct brw_device_info
*devinfo
= p
->devinfo
;
2181 uint32_t msg_control
;
2184 if (devinfo
->gen
>= 6)
2187 if (p
->devinfo
->gen
>= 7) {
2188 /* On gen 7 and above, we no longer have message registers and we can
2189 * send from any register we want. By using the destination register
2190 * for the message, we guarantee that the implied message write won't
2191 * accidentally overwrite anything. This has been a problem because
2192 * the MRF registers and source for the final FB write are both fixed
2195 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2197 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2199 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2201 if (num_regs
== 1) {
2202 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2205 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2210 brw_push_insn_state(p
);
2211 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2212 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2213 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2215 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2217 /* set message header global offset field (reg 0, element 2) */
2218 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2220 brw_pop_insn_state(p
);
2224 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2226 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2227 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2229 brw_set_dest(p
, insn
, dest
); /* UW? */
2230 if (devinfo
->gen
>= 6) {
2231 brw_set_src0(p
, insn
, mrf
);
2233 brw_set_src0(p
, insn
, brw_null_reg());
2234 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2237 brw_set_dp_read_message(p
,
2239 brw_scratch_surface_idx(p
),
2241 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2242 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
,
2244 true, /* header_present */
2250 gen7_block_read_scratch(struct brw_codegen
*p
,
2251 struct brw_reg dest
,
2255 const struct brw_device_info
*devinfo
= p
->devinfo
;
2256 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2257 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2259 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2260 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2262 /* The HW requires that the header is present; this is to get the g0.5
2265 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2267 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2268 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2269 * is 32 bytes, which happens to be the size of a register.
2272 assert(offset
< (1 << 12));
2274 gen7_set_dp_scratch_message(p
, insn
,
2275 false, /* scratch read */
2277 false, /* invalidate after read */
2280 1, /* mlen: just g0 */
2281 num_regs
, /* rlen */
2282 true); /* header present */
2286 * Read a float[4] vector from the data port Data Cache (const buffer).
2287 * Location (in buffer) should be a multiple of 16.
2288 * Used for fetching shader constants.
2290 void brw_oword_block_read(struct brw_codegen
*p
,
2291 struct brw_reg dest
,
2294 uint32_t bind_table_index
)
2296 const struct brw_device_info
*devinfo
= p
->devinfo
;
2298 /* On newer hardware, offset is in units of owords. */
2299 if (devinfo
->gen
>= 6)
2302 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2304 brw_push_insn_state(p
);
2305 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2306 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2307 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2308 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2310 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2312 /* set message header global offset field (reg 0, element 2) */
2314 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2316 2), BRW_REGISTER_TYPE_UD
),
2317 brw_imm_ud(offset
));
2319 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2321 /* cast dest to a uword[8] vector */
2322 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2324 brw_set_dest(p
, insn
, dest
);
2325 if (devinfo
->gen
>= 6) {
2326 brw_set_src0(p
, insn
, mrf
);
2328 brw_set_src0(p
, insn
, brw_null_reg());
2329 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2332 brw_set_dp_read_message(p
,
2335 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW
,
2336 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2337 BRW_DATAPORT_READ_TARGET_DATA_CACHE
,
2339 true, /* header_present */
2340 1); /* response_length (1 reg, 2 owords!) */
2342 brw_pop_insn_state(p
);
2346 void brw_fb_WRITE(struct brw_codegen
*p
,
2348 struct brw_reg payload
,
2349 struct brw_reg implied_header
,
2350 unsigned msg_control
,
2351 unsigned binding_table_index
,
2352 unsigned msg_length
,
2353 unsigned response_length
,
2355 bool last_render_target
,
2356 bool header_present
)
2358 const struct brw_device_info
*devinfo
= p
->devinfo
;
2361 struct brw_reg dest
, src0
;
2363 if (dispatch_width
== 16)
2364 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2366 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2368 if (devinfo
->gen
>= 6) {
2369 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2371 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2373 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2375 if (devinfo
->gen
>= 6) {
2376 /* headerless version, just submit color payload */
2379 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2381 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2382 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2383 src0
= implied_header
;
2385 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2388 brw_set_dest(p
, insn
, dest
);
2389 brw_set_src0(p
, insn
, src0
);
2390 brw_set_dp_write_message(p
,
2392 binding_table_index
,
2400 0 /* send_commit_msg */);
2405 * Texture sample instruction.
2406 * Note: the msg_type plus msg_length values determine exactly what kind
2407 * of sampling operation is performed. See volume 4, page 161 of docs.
2409 void brw_SAMPLE(struct brw_codegen
*p
,
2410 struct brw_reg dest
,
2411 unsigned msg_reg_nr
,
2412 struct brw_reg src0
,
2413 unsigned binding_table_index
,
2416 unsigned response_length
,
2417 unsigned msg_length
,
2418 unsigned header_present
,
2420 unsigned return_format
)
2422 const struct brw_device_info
*devinfo
= p
->devinfo
;
2425 if (msg_reg_nr
!= -1)
2426 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2428 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2429 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2431 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2433 * "Instruction compression is not allowed for this instruction (that
2434 * is, send). The hardware behavior is undefined if this instruction is
2435 * set as compressed. However, compress control can be set to "SecHalf"
2436 * to affect the EMask generation."
2438 * No similar wording is found in later PRMs, but there are examples
2439 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2440 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2441 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2443 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_2NDHALF
)
2444 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2446 if (devinfo
->gen
< 6)
2447 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2449 brw_set_dest(p
, insn
, dest
);
2450 brw_set_src0(p
, insn
, src0
);
2451 brw_set_sampler_message(p
, insn
,
2452 binding_table_index
,
2462 /* Adjust the message header's sampler state pointer to
2463 * select the correct group of 16 samplers.
2465 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2466 struct brw_reg header
,
2467 struct brw_reg sampler_index
)
2469 /* The "Sampler Index" field can only store values between 0 and 15.
2470 * However, we can add an offset to the "Sampler State Pointer"
2471 * field, effectively selecting a different set of 16 samplers.
2473 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2474 * offset, and each sampler state is only 16-bytes, so we can't
2475 * exclusively use the offset - we have to use both.
2478 const struct brw_device_info
*devinfo
= p
->devinfo
;
2480 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2481 const int sampler_state_size
= 16; /* 16 bytes */
2482 uint32_t sampler
= sampler_index
.ud
;
2484 if (sampler
>= 16) {
2485 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2487 get_element_ud(header
, 3),
2488 get_element_ud(brw_vec8_grf(0, 0), 3),
2489 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2492 /* Non-const sampler array indexing case */
2493 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2497 struct brw_reg temp
= get_element_ud(header
, 3);
2499 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2500 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2502 get_element_ud(header
, 3),
2503 get_element_ud(brw_vec8_grf(0, 0), 3),
2508 /* All these variables are pretty confusing - we might be better off
2509 * using bitmasks and macros for this, in the old style. Or perhaps
2510 * just having the caller instantiate the fields in dword3 itself.
2512 void brw_urb_WRITE(struct brw_codegen
*p
,
2513 struct brw_reg dest
,
2514 unsigned msg_reg_nr
,
2515 struct brw_reg src0
,
2516 enum brw_urb_write_flags flags
,
2517 unsigned msg_length
,
2518 unsigned response_length
,
2522 const struct brw_device_info
*devinfo
= p
->devinfo
;
2525 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2527 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2528 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2529 brw_push_insn_state(p
);
2530 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2531 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2532 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2533 BRW_REGISTER_TYPE_UD
),
2534 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2535 brw_imm_ud(0xff00));
2536 brw_pop_insn_state(p
);
2539 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2541 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2543 brw_set_dest(p
, insn
, dest
);
2544 brw_set_src0(p
, insn
, src0
);
2545 brw_set_src1(p
, insn
, brw_imm_d(0));
2547 if (devinfo
->gen
< 6)
2548 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2550 brw_set_urb_message(p
,
2560 brw_send_indirect_message(struct brw_codegen
*p
,
2563 struct brw_reg payload
,
2564 struct brw_reg desc
)
2566 const struct brw_device_info
*devinfo
= p
->devinfo
;
2567 struct brw_inst
*send
;
2570 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2572 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2574 /* We hold on to the setup instruction (the SEND in the direct case, the OR
2575 * in the indirect case) by its index in the instruction store. The
2576 * pointer returned by next_insn() may become invalid if emitting the SEND
2577 * in the indirect case reallocs the store.
2580 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2582 send
= next_insn(p
, BRW_OPCODE_SEND
);
2583 brw_set_src1(p
, send
, desc
);
2586 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2588 brw_push_insn_state(p
);
2589 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2590 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2591 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2593 /* Load the indirect descriptor to an address register using OR so the
2594 * caller can specify additional descriptor bits with the usual
2595 * brw_set_*_message() helper functions.
2598 brw_OR(p
, addr
, desc
, brw_imm_ud(0));
2600 brw_pop_insn_state(p
);
2602 send
= next_insn(p
, BRW_OPCODE_SEND
);
2603 brw_set_src1(p
, send
, addr
);
2606 if (dst
.width
< BRW_EXECUTE_8
)
2607 brw_inst_set_exec_size(devinfo
, send
, dst
.width
);
2609 brw_set_dest(p
, send
, dst
);
2610 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2611 brw_inst_set_sfid(devinfo
, send
, sfid
);
2613 return &p
->store
[setup
];
2616 static struct brw_inst
*
2617 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2620 struct brw_reg payload
,
2621 struct brw_reg surface
,
2622 unsigned message_len
,
2623 unsigned response_len
,
2624 bool header_present
)
2626 const struct brw_device_info
*devinfo
= p
->devinfo
;
2627 struct brw_inst
*insn
;
2629 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2630 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2632 brw_push_insn_state(p
);
2633 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2634 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2635 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2637 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2638 * some surface array is accessed out of bounds.
2640 insn
= brw_AND(p
, addr
,
2641 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2642 BRW_GET_SWZ(surface
.swizzle
, 0)),
2645 brw_pop_insn_state(p
);
2650 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
);
2651 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2652 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2653 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2659 while_jumps_before_offset(const struct brw_device_info
*devinfo
,
2660 brw_inst
*insn
, int while_offset
, int start_offset
)
2662 int scale
= 16 / brw_jump_scale(devinfo
);
2663 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2664 : brw_inst_jip(devinfo
, insn
);
2665 return while_offset
+ jip
* scale
<= start_offset
;
2670 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2673 void *store
= p
->store
;
2674 const struct brw_device_info
*devinfo
= p
->devinfo
;
2678 for (offset
= next_offset(devinfo
, store
, start_offset
);
2679 offset
< p
->next_insn_offset
;
2680 offset
= next_offset(devinfo
, store
, offset
)) {
2681 brw_inst
*insn
= store
+ offset
;
2683 switch (brw_inst_opcode(devinfo
, insn
)) {
2687 case BRW_OPCODE_ENDIF
:
2692 case BRW_OPCODE_WHILE
:
2693 /* If the while doesn't jump before our instruction, it's the end
2694 * of a sibling do...while loop. Ignore it.
2696 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2699 case BRW_OPCODE_ELSE
:
2700 case BRW_OPCODE_HALT
:
2709 /* There is no DO instruction on gen6, so to find the end of the loop
2710 * we have to see if the loop is jumping back before our start
2714 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2716 const struct brw_device_info
*devinfo
= p
->devinfo
;
2718 void *store
= p
->store
;
2720 assert(devinfo
->gen
>= 6);
2722 /* Always start after the instruction (such as a WHILE) we're trying to fix
2725 for (offset
= next_offset(devinfo
, store
, start_offset
);
2726 offset
< p
->next_insn_offset
;
2727 offset
= next_offset(devinfo
, store
, offset
)) {
2728 brw_inst
*insn
= store
+ offset
;
2730 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2731 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2735 assert(!"not reached");
2736 return start_offset
;
2739 /* After program generation, go back and update the UIP and JIP of
2740 * BREAK, CONT, and HALT instructions to their correct locations.
2743 brw_set_uip_jip(struct brw_codegen
*p
)
2745 const struct brw_device_info
*devinfo
= p
->devinfo
;
2747 int br
= brw_jump_scale(devinfo
);
2748 int scale
= 16 / br
;
2749 void *store
= p
->store
;
2751 if (devinfo
->gen
< 6)
2754 for (offset
= 0; offset
< p
->next_insn_offset
;
2755 offset
= next_offset(devinfo
, store
, offset
)) {
2756 brw_inst
*insn
= store
+ offset
;
2758 if (brw_inst_cmpt_control(devinfo
, insn
)) {
2759 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2760 assert(brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_BREAK
&&
2761 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_CONTINUE
&&
2762 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_HALT
);
2766 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2767 switch (brw_inst_opcode(devinfo
, insn
)) {
2768 case BRW_OPCODE_BREAK
:
2769 assert(block_end_offset
!= 0);
2770 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2771 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2772 brw_inst_set_uip(devinfo
, insn
,
2773 (brw_find_loop_end(p
, offset
) - offset
+
2774 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2776 case BRW_OPCODE_CONTINUE
:
2777 assert(block_end_offset
!= 0);
2778 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2779 brw_inst_set_uip(devinfo
, insn
,
2780 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2782 assert(brw_inst_uip(devinfo
, insn
) != 0);
2783 assert(brw_inst_jip(devinfo
, insn
) != 0);
2786 case BRW_OPCODE_ENDIF
: {
2787 int32_t jump
= (block_end_offset
== 0) ?
2788 1 * br
: (block_end_offset
- offset
) / scale
;
2789 if (devinfo
->gen
>= 7)
2790 brw_inst_set_jip(devinfo
, insn
, jump
);
2792 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2796 case BRW_OPCODE_HALT
:
2797 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2799 * "In case of the halt instruction not inside any conditional
2800 * code block, the value of <JIP> and <UIP> should be the
2801 * same. In case of the halt instruction inside conditional code
2802 * block, the <UIP> should be the end of the program, and the
2803 * <JIP> should be end of the most inner conditional code block."
2805 * The uip will have already been set by whoever set up the
2808 if (block_end_offset
== 0) {
2809 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2811 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2813 assert(brw_inst_uip(devinfo
, insn
) != 0);
2814 assert(brw_inst_jip(devinfo
, insn
) != 0);
2820 void brw_ff_sync(struct brw_codegen
*p
,
2821 struct brw_reg dest
,
2822 unsigned msg_reg_nr
,
2823 struct brw_reg src0
,
2825 unsigned response_length
,
2828 const struct brw_device_info
*devinfo
= p
->devinfo
;
2831 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2833 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2834 brw_set_dest(p
, insn
, dest
);
2835 brw_set_src0(p
, insn
, src0
);
2836 brw_set_src1(p
, insn
, brw_imm_d(0));
2838 if (devinfo
->gen
< 6)
2839 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2841 brw_set_ff_sync_message(p
,
2849 * Emit the SEND instruction necessary to generate stream output data on Gen6
2850 * (for transform feedback).
2852 * If send_commit_msg is true, this is the last piece of stream output data
2853 * from this thread, so send the data as a committed write. According to the
2854 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2856 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2857 * writes are complete by sending the final write as a committed write."
2860 brw_svb_write(struct brw_codegen
*p
,
2861 struct brw_reg dest
,
2862 unsigned msg_reg_nr
,
2863 struct brw_reg src0
,
2864 unsigned binding_table_index
,
2865 bool send_commit_msg
)
2869 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2871 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2872 brw_set_dest(p
, insn
, dest
);
2873 brw_set_src0(p
, insn
, src0
);
2874 brw_set_src1(p
, insn
, brw_imm_d(0));
2875 brw_set_dp_write_message(p
, insn
,
2876 binding_table_index
,
2877 0, /* msg_control: ignored */
2878 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2880 true, /* header_present */
2881 0, /* last_render_target: ignored */
2882 send_commit_msg
, /* response_length */
2883 0, /* end_of_thread */
2884 send_commit_msg
); /* send_commit_msg */
2888 brw_surface_payload_size(struct brw_codegen
*p
,
2889 unsigned num_channels
,
2893 if (has_simd4x2
&& brw_inst_access_mode(p
->devinfo
, p
->current
) == BRW_ALIGN_16
)
2895 else if (has_simd16
&& p
->compressed
)
2896 return 2 * num_channels
;
2898 return num_channels
;
2902 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2905 bool response_expected
)
2907 const struct brw_device_info
*devinfo
= p
->devinfo
;
2908 unsigned msg_control
=
2909 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2910 (response_expected
? 1 << 5 : 0); /* Return data expected */
2912 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2913 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2915 msg_control
|= 1 << 4; /* SIMD8 mode */
2917 brw_inst_set_dp_msg_type(devinfo
, insn
,
2918 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2920 brw_inst_set_dp_msg_type(devinfo
, insn
,
2921 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2924 brw_inst_set_dp_msg_type(devinfo
, insn
,
2925 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2928 msg_control
|= 1 << 4; /* SIMD8 mode */
2931 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2935 brw_untyped_atomic(struct brw_codegen
*p
,
2937 struct brw_reg payload
,
2938 struct brw_reg surface
,
2940 unsigned msg_length
,
2941 bool response_expected
)
2943 const struct brw_device_info
*devinfo
= p
->devinfo
;
2944 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2945 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2946 GEN7_SFID_DATAPORT_DATA_CACHE
);
2947 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2948 /* Mask out unused components -- This is especially important in Align16
2949 * mode on generations that don't have native support for SIMD4x2 atomics,
2950 * because unused but enabled components will cause the dataport to perform
2951 * additional atomic operations on the addresses that happen to be in the
2952 * uninitialized Y, Z and W coordinates of the payload.
2954 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2955 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2956 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2957 brw_surface_payload_size(p
, response_expected
,
2958 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2961 brw_set_dp_untyped_atomic_message(
2962 p
, insn
, atomic_op
, response_expected
);
2966 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
2967 struct brw_inst
*insn
,
2968 unsigned num_channels
)
2970 const struct brw_device_info
*devinfo
= p
->devinfo
;
2971 /* Set mask of 32-bit channels to drop. */
2972 unsigned msg_control
= 0xf & (0xf << num_channels
);
2974 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2976 msg_control
|= 1 << 4; /* SIMD16 mode */
2978 msg_control
|= 2 << 4; /* SIMD8 mode */
2981 brw_inst_set_dp_msg_type(devinfo
, insn
,
2982 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2983 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2984 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
2985 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2989 brw_untyped_surface_read(struct brw_codegen
*p
,
2991 struct brw_reg payload
,
2992 struct brw_reg surface
,
2993 unsigned msg_length
,
2994 unsigned num_channels
)
2996 const struct brw_device_info
*devinfo
= p
->devinfo
;
2997 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2998 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2999 GEN7_SFID_DATAPORT_DATA_CACHE
);
3000 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3001 p
, sfid
, dst
, payload
, surface
, msg_length
,
3002 brw_surface_payload_size(p
, num_channels
, true, true),
3005 brw_set_dp_untyped_surface_read_message(
3006 p
, insn
, num_channels
);
3010 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
3011 struct brw_inst
*insn
,
3012 unsigned num_channels
)
3014 const struct brw_device_info
*devinfo
= p
->devinfo
;
3015 /* Set mask of 32-bit channels to drop. */
3016 unsigned msg_control
= 0xf & (0xf << num_channels
);
3018 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3020 msg_control
|= 1 << 4; /* SIMD16 mode */
3022 msg_control
|= 2 << 4; /* SIMD8 mode */
3024 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
3025 msg_control
|= 0 << 4; /* SIMD4x2 mode */
3027 msg_control
|= 2 << 4; /* SIMD8 mode */
3030 brw_inst_set_dp_msg_type(devinfo
, insn
,
3031 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3032 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
3033 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
3034 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3038 brw_untyped_surface_write(struct brw_codegen
*p
,
3039 struct brw_reg payload
,
3040 struct brw_reg surface
,
3041 unsigned msg_length
,
3042 unsigned num_channels
)
3044 const struct brw_device_info
*devinfo
= p
->devinfo
;
3045 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3046 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3047 GEN7_SFID_DATAPORT_DATA_CACHE
);
3048 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3049 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3050 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3051 WRITEMASK_X
: WRITEMASK_XYZW
;
3052 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3053 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3054 payload
, surface
, msg_length
, 0, align1
);
3056 brw_set_dp_untyped_surface_write_message(
3057 p
, insn
, num_channels
);
3061 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
3062 struct brw_inst
*insn
,
3064 bool response_expected
)
3066 const struct brw_device_info
*devinfo
= p
->devinfo
;
3067 unsigned msg_control
=
3068 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3069 (response_expected
? 1 << 5 : 0); /* Return data expected */
3071 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3072 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3073 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3074 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3076 brw_inst_set_dp_msg_type(devinfo
, insn
,
3077 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
3079 brw_inst_set_dp_msg_type(devinfo
, insn
,
3080 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
3084 brw_inst_set_dp_msg_type(devinfo
, insn
,
3085 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
3087 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3088 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3091 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3095 brw_typed_atomic(struct brw_codegen
*p
,
3097 struct brw_reg payload
,
3098 struct brw_reg surface
,
3100 unsigned msg_length
,
3101 bool response_expected
) {
3102 const struct brw_device_info
*devinfo
= p
->devinfo
;
3103 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3104 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3105 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3106 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3107 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3108 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3109 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3110 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
3111 brw_surface_payload_size(p
, response_expected
,
3112 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3115 brw_set_dp_typed_atomic_message(
3116 p
, insn
, atomic_op
, response_expected
);
3120 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3121 struct brw_inst
*insn
,
3122 unsigned num_channels
)
3124 const struct brw_device_info
*devinfo
= p
->devinfo
;
3125 /* Set mask of unused channels. */
3126 unsigned msg_control
= 0xf & (0xf << num_channels
);
3128 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3129 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3130 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3131 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3133 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3136 brw_inst_set_dp_msg_type(devinfo
, insn
,
3137 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3139 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3140 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3141 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3144 brw_inst_set_dp_msg_type(devinfo
, insn
,
3145 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3148 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3152 brw_typed_surface_read(struct brw_codegen
*p
,
3154 struct brw_reg payload
,
3155 struct brw_reg surface
,
3156 unsigned msg_length
,
3157 unsigned num_channels
)
3159 const struct brw_device_info
*devinfo
= p
->devinfo
;
3160 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3161 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3162 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3163 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3164 p
, sfid
, dst
, payload
, surface
, msg_length
,
3165 brw_surface_payload_size(p
, num_channels
,
3166 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3169 brw_set_dp_typed_surface_read_message(
3170 p
, insn
, num_channels
);
3174 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3175 struct brw_inst
*insn
,
3176 unsigned num_channels
)
3178 const struct brw_device_info
*devinfo
= p
->devinfo
;
3179 /* Set mask of unused channels. */
3180 unsigned msg_control
= 0xf & (0xf << num_channels
);
3182 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3183 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3184 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3185 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3187 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3190 brw_inst_set_dp_msg_type(devinfo
, insn
,
3191 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3194 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3195 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3196 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3199 brw_inst_set_dp_msg_type(devinfo
, insn
,
3200 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3203 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3207 brw_typed_surface_write(struct brw_codegen
*p
,
3208 struct brw_reg payload
,
3209 struct brw_reg surface
,
3210 unsigned msg_length
,
3211 unsigned num_channels
)
3213 const struct brw_device_info
*devinfo
= p
->devinfo
;
3214 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3215 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3216 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3217 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3218 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3219 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3220 WRITEMASK_X
: WRITEMASK_XYZW
);
3221 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3222 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3223 payload
, surface
, msg_length
, 0, true);
3225 brw_set_dp_typed_surface_write_message(
3226 p
, insn
, num_channels
);
3230 brw_set_memory_fence_message(struct brw_codegen
*p
,
3231 struct brw_inst
*insn
,
3232 enum brw_message_target sfid
,
3235 const struct brw_device_info
*devinfo
= p
->devinfo
;
3237 brw_set_message_descriptor(p
, insn
, sfid
,
3238 1 /* message length */,
3239 (commit_enable
? 1 : 0) /* response length */,
3240 true /* header present */,
3244 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3245 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3247 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3248 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3251 unreachable("Not reached");
3255 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3259 brw_memory_fence(struct brw_codegen
*p
,
3262 const struct brw_device_info
*devinfo
= p
->devinfo
;
3263 const bool commit_enable
= devinfo
->gen
== 7 && !devinfo
->is_haswell
;
3264 struct brw_inst
*insn
;
3266 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3267 * message doesn't write anything back.
3269 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3270 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3271 brw_set_dest(p
, insn
, dst
);
3272 brw_set_src0(p
, insn
, dst
);
3273 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3276 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3277 /* IVB does typed surface access through the render cache, so we need to
3278 * flush it too. Use a different register so both flushes can be
3279 * pipelined by the hardware.
3281 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3282 brw_set_dest(p
, insn
, offset(dst
, 1));
3283 brw_set_src0(p
, insn
, offset(dst
, 1));
3284 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3287 /* Now write the response of the second message into the response of the
3288 * first to trigger a pipeline stall -- This way future render and data
3289 * cache messages will be properly ordered with respect to past data and
3290 * render cache messages.
3292 brw_push_insn_state(p
);
3293 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3294 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3295 brw_MOV(p
, dst
, offset(dst
, 1));
3296 brw_pop_insn_state(p
);
3301 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3302 struct brw_reg dest
,
3306 struct brw_reg data
,
3307 unsigned msg_length
,
3308 unsigned response_length
)
3310 const struct brw_device_info
*devinfo
= p
->devinfo
;
3311 struct brw_inst
*insn
;
3312 const uint16_t exec_size
= brw_inst_exec_size(devinfo
, p
->current
);
3314 /* brw_send_indirect_message will automatically use a direct send message
3315 * if data is actually immediate.
3317 insn
= brw_send_indirect_message(p
,
3318 GEN7_SFID_PIXEL_INTERPOLATOR
,
3322 brw_inst_set_mlen(devinfo
, insn
, msg_length
);
3323 brw_inst_set_rlen(devinfo
, insn
, response_length
);
3325 brw_inst_set_pi_simd_mode(devinfo
, insn
, exec_size
== BRW_EXECUTE_16
);
3326 brw_inst_set_pi_slot_group(devinfo
, insn
, 0); /* zero unless 32/64px dispatch */
3327 brw_inst_set_pi_nopersp(devinfo
, insn
, noperspective
);
3328 brw_inst_set_pi_message_type(devinfo
, insn
, mode
);
3332 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
)
3334 const struct brw_device_info
*devinfo
= p
->devinfo
;
3337 assert(devinfo
->gen
>= 7);
3339 brw_push_insn_state(p
);
3341 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3342 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3344 if (devinfo
->gen
>= 8) {
3345 /* Getting the first active channel index is easy on Gen8: Just find
3346 * the first bit set in the mask register. The same register exists
3347 * on HSW already but it reads back as all ones when the current
3348 * instruction has execution masking disabled, so it's kind of
3351 inst
= brw_FBL(p
, vec1(dst
),
3352 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
));
3354 /* Quarter control has the effect of magically shifting the value of
3355 * this register. Make sure it's set to zero.
3357 brw_inst_set_qtr_control(devinfo
, inst
, GEN6_COMPRESSION_1Q
);
3359 const struct brw_reg flag
= retype(brw_flag_reg(1, 0),
3360 BRW_REGISTER_TYPE_UD
);
3362 brw_MOV(p
, flag
, brw_imm_ud(0));
3364 /* Run a 16-wide instruction returning zero with execution masking
3365 * and a conditional modifier enabled in order to get the current
3366 * execution mask in f1.0.
3368 inst
= brw_MOV(p
, brw_null_reg(), brw_imm_ud(0));
3369 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_16
);
3370 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3371 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3372 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3374 brw_FBL(p
, vec1(dst
), flag
);
3377 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3379 if (devinfo
->gen
>= 8) {
3380 /* In SIMD4x2 mode the first active channel index is just the
3381 * negation of the first bit of the mask register.
3383 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3384 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3388 /* Overwrite the destination without and with execution masking to
3389 * find out which of the channels is active.
3391 brw_push_insn_state(p
);
3392 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3393 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3396 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3398 brw_pop_insn_state(p
);
3399 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3403 brw_pop_insn_state(p
);
3407 brw_broadcast(struct brw_codegen
*p
,
3412 const struct brw_device_info
*devinfo
= p
->devinfo
;
3413 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3416 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3417 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3419 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3420 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3421 /* Trivial, the source is already uniform or the index is a constant.
3422 * We will typically not get here if the optimizer is doing its job, but
3423 * asserting would be mean.
3425 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3427 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3428 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3431 const struct brw_reg addr
=
3432 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3433 const unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3434 /* Limit in bytes of the signed indirect addressing immediate. */
3435 const unsigned limit
= 512;
3437 brw_push_insn_state(p
);
3438 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3439 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3441 /* Take into account the component size and horizontal stride. */
3442 assert(src
.vstride
== src
.hstride
+ src
.width
);
3443 brw_SHL(p
, addr
, vec1(idx
),
3444 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3447 /* We can only address up to limit bytes using the indirect
3448 * addressing immediate, account for the difference if the source
3449 * register is above this limit.
3451 if (offset
>= limit
)
3452 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3454 brw_pop_insn_state(p
);
3456 /* Use indirect addressing to fetch the specified component. */
3458 retype(brw_vec1_indirect(addr
.subnr
, offset
% limit
),
3461 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3462 * to all bits of a flag register,
3466 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 0, 4, 1));
3467 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3468 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3469 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3471 /* and use predicated SEL to pick the right channel. */
3472 inst
= brw_SEL(p
, dst
,
3473 stride(suboffset(src
, 4), 0, 4, 1),
3474 stride(src
, 0, 4, 1));
3475 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3476 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3482 * This instruction is generated as a single-channel align1 instruction by
3483 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3485 * We can't use the typed atomic op in the FS because that has the execution
3486 * mask ANDed with the pixel mask, but we just want to write the one dword for
3489 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3490 * one u32. So we use the same untyped atomic write message as the pixel
3493 * The untyped atomic operation requires a BUFFER surface type with RAW
3494 * format, and is only accessible through the legacy DATA_CACHE dataport
3497 void brw_shader_time_add(struct brw_codegen
*p
,
3498 struct brw_reg payload
,
3499 uint32_t surf_index
)
3501 const unsigned sfid
= (p
->devinfo
->gen
>= 8 || p
->devinfo
->is_haswell
?
3502 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3503 GEN7_SFID_DATAPORT_DATA_CACHE
);
3504 assert(p
->devinfo
->gen
>= 7);
3506 brw_push_insn_state(p
);
3507 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3508 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3509 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3510 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3512 /* We use brw_vec1_reg and unmasked because we want to increment the given
3515 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3517 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3519 brw_set_src1(p
, send
, brw_imm_ud(0));
3520 brw_set_message_descriptor(p
, send
, sfid
, 2, 0, false, false);
3521 brw_inst_set_binding_table_index(p
->devinfo
, send
, surf_index
);
3522 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3524 brw_pop_insn_state(p
);
3529 * Emit the SEND message for a barrier
3532 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3534 const struct brw_device_info
*devinfo
= p
->devinfo
;
3535 struct brw_inst
*inst
;
3537 assert(devinfo
->gen
>= 7);
3539 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3540 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3541 brw_set_src0(p
, inst
, src
);
3542 brw_set_src1(p
, inst
, brw_null_reg());
3544 brw_set_message_descriptor(p
, inst
, BRW_SFID_MESSAGE_GATEWAY
,
3546 0 /* response_length */,
3547 false /* header_present */,
3548 false /* end_of_thread */);
3550 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3551 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3552 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3554 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3559 * Emit the wait instruction for a barrier
3562 brw_WAIT(struct brw_codegen
*p
)
3564 const struct brw_device_info
*devinfo
= p
->devinfo
;
3565 struct brw_inst
*insn
;
3567 struct brw_reg src
= brw_notification_reg();
3569 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3570 brw_set_dest(p
, insn
, src
);
3571 brw_set_src0(p
, insn
, src
);
3572 brw_set_src1(p
, insn
, brw_null_reg());
3574 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3575 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);