2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 * Convert a brw_reg_type enumeration value into the hardware representation.
90 * The hardware encoding may depend on whether the value is an immediate.
93 brw_reg_type_to_hw_type(const struct gen_device_info
*devinfo
,
94 enum brw_reg_type type
, enum brw_reg_file file
)
96 if (file
== BRW_IMMEDIATE_VALUE
) {
97 static const int imm_hw_types
[] = {
98 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
99 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
100 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
101 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
102 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
103 [BRW_REGISTER_TYPE_UB
] = -1,
104 [BRW_REGISTER_TYPE_B
] = -1,
105 [BRW_REGISTER_TYPE_UV
] = BRW_HW_REG_IMM_TYPE_UV
,
106 [BRW_REGISTER_TYPE_VF
] = BRW_HW_REG_IMM_TYPE_VF
,
107 [BRW_REGISTER_TYPE_V
] = BRW_HW_REG_IMM_TYPE_V
,
108 [BRW_REGISTER_TYPE_DF
] = GEN8_HW_REG_IMM_TYPE_DF
,
109 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_IMM_TYPE_HF
,
110 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
111 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
113 assert(type
< ARRAY_SIZE(imm_hw_types
));
114 assert(imm_hw_types
[type
] != -1);
115 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_DF
);
116 return imm_hw_types
[type
];
118 /* Non-immediate registers */
119 static const int hw_types
[] = {
120 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
121 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
122 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
123 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
124 [BRW_REGISTER_TYPE_UB
] = BRW_HW_REG_NON_IMM_TYPE_UB
,
125 [BRW_REGISTER_TYPE_B
] = BRW_HW_REG_NON_IMM_TYPE_B
,
126 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
127 [BRW_REGISTER_TYPE_UV
] = -1,
128 [BRW_REGISTER_TYPE_VF
] = -1,
129 [BRW_REGISTER_TYPE_V
] = -1,
130 [BRW_REGISTER_TYPE_DF
] = GEN7_HW_REG_NON_IMM_TYPE_DF
,
131 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_NON_IMM_TYPE_HF
,
132 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
133 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
135 assert(type
< ARRAY_SIZE(hw_types
));
136 assert(hw_types
[type
] != -1);
137 assert(devinfo
->gen
>= 7 || type
< BRW_REGISTER_TYPE_DF
);
138 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_Q
);
139 return hw_types
[type
];
144 * Return the element size given a hardware register type and file.
146 * The hardware encoding may depend on whether the value is an immediate.
149 brw_hw_reg_type_to_size(const struct gen_device_info
*devinfo
,
150 unsigned type
, enum brw_reg_file file
)
152 if (file
== BRW_IMMEDIATE_VALUE
) {
153 static const unsigned imm_hw_sizes
[] = {
154 [BRW_HW_REG_TYPE_UD
] = 4,
155 [BRW_HW_REG_TYPE_D
] = 4,
156 [BRW_HW_REG_TYPE_UW
] = 2,
157 [BRW_HW_REG_TYPE_W
] = 2,
158 [BRW_HW_REG_IMM_TYPE_UV
] = 2,
159 [BRW_HW_REG_IMM_TYPE_VF
] = 4,
160 [BRW_HW_REG_IMM_TYPE_V
] = 2,
161 [BRW_HW_REG_TYPE_F
] = 4,
162 [GEN8_HW_REG_TYPE_UQ
] = 8,
163 [GEN8_HW_REG_TYPE_Q
] = 8,
164 [GEN8_HW_REG_IMM_TYPE_DF
] = 8,
165 [GEN8_HW_REG_IMM_TYPE_HF
] = 2,
167 assert(type
< ARRAY_SIZE(imm_hw_sizes
));
168 assert(devinfo
->gen
>= 6 || type
!= BRW_HW_REG_IMM_TYPE_UV
);
169 assert(devinfo
->gen
>= 8 || type
<= BRW_HW_REG_TYPE_F
);
170 return imm_hw_sizes
[type
];
172 /* Non-immediate registers */
173 static const unsigned hw_sizes
[] = {
174 [BRW_HW_REG_TYPE_UD
] = 4,
175 [BRW_HW_REG_TYPE_D
] = 4,
176 [BRW_HW_REG_TYPE_UW
] = 2,
177 [BRW_HW_REG_TYPE_W
] = 2,
178 [BRW_HW_REG_NON_IMM_TYPE_UB
] = 1,
179 [BRW_HW_REG_NON_IMM_TYPE_B
] = 1,
180 [GEN7_HW_REG_NON_IMM_TYPE_DF
] = 8,
181 [BRW_HW_REG_TYPE_F
] = 4,
182 [GEN8_HW_REG_TYPE_UQ
] = 8,
183 [GEN8_HW_REG_TYPE_Q
] = 8,
184 [GEN8_HW_REG_NON_IMM_TYPE_HF
] = 2,
186 assert(type
< ARRAY_SIZE(hw_sizes
));
187 assert(devinfo
->gen
>= 7 ||
188 (type
< GEN7_HW_REG_NON_IMM_TYPE_DF
|| type
== BRW_HW_REG_TYPE_F
));
189 assert(devinfo
->gen
>= 8 || type
<= BRW_HW_REG_TYPE_F
);
190 return hw_sizes
[type
];
195 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
197 const struct gen_device_info
*devinfo
= p
->devinfo
;
199 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
200 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
201 else if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
202 assert(dest
.nr
< 128);
204 gen7_convert_mrf_to_grf(p
, &dest
);
206 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
207 brw_inst_set_dst_reg_type(devinfo
, inst
,
208 brw_reg_type_to_hw_type(devinfo
, dest
.type
,
210 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
212 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
213 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
215 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
216 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
217 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
218 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
219 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
221 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
222 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
223 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
224 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
225 assert(dest
.writemask
!= 0);
227 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
228 * Although Dst.HorzStride is a don't care for Align16, HW needs
229 * this to be programmed as "01".
231 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
234 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
236 /* These are different sizes in align1 vs align16:
238 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
239 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
240 dest
.indirect_offset
);
241 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
242 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
243 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
245 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
246 dest
.indirect_offset
);
247 /* even ignored in da16, still need to set as '01' */
248 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
252 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
253 * or 16 (SIMD16), as that's normally correct. However, when dealing with
254 * small registers, we automatically reduce it to match the register size.
256 * In platforms that support fp64 we can emit instructions with a width of
257 * 4 that need two SIMD8 registers and an exec_size of 8 or 16. In these
258 * cases we need to make sure that these instructions have their exec sizes
259 * set properly when they are emitted and we can't rely on this code to fix
263 if (devinfo
->gen
>= 6)
264 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
266 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
269 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
273 validate_reg(const struct gen_device_info
*devinfo
,
274 brw_inst
*inst
, struct brw_reg reg
)
276 const int hstride_for_reg
[] = {0, 1, 2, 4};
277 const int vstride_for_reg
[] = {0, 1, 2, 4, 8, 16, 32};
278 const int width_for_reg
[] = {1, 2, 4, 8, 16};
279 const int execsize_for_reg
[] = {1, 2, 4, 8, 16, 32};
280 int width
, hstride
, vstride
, execsize
;
282 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
283 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
284 * mean the destination has to be 128-bit aligned and the
285 * destination horiz stride has to be a word.
287 if (reg
.type
== BRW_REGISTER_TYPE_V
) {
288 unsigned UNUSED elem_size
= brw_element_size(devinfo
, inst
, dst
);
289 assert(hstride_for_reg
[brw_inst_dst_hstride(devinfo
, inst
)] *
296 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
297 reg
.file
== BRW_ARF_NULL
)
300 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
302 * "Swizzling is not allowed when an accumulator is used as an implicit
303 * source or an explicit source in an instruction."
305 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
306 reg
.nr
== BRW_ARF_ACCUMULATOR
)
307 assert(reg
.swizzle
== BRW_SWIZZLE_XYZW
);
309 assert(reg
.hstride
< ARRAY_SIZE(hstride_for_reg
));
310 hstride
= hstride_for_reg
[reg
.hstride
];
312 if (reg
.vstride
== 0xf) {
315 assert(reg
.vstride
>= 0 && reg
.vstride
< ARRAY_SIZE(vstride_for_reg
));
316 vstride
= vstride_for_reg
[reg
.vstride
];
319 assert(reg
.width
>= 0 && reg
.width
< ARRAY_SIZE(width_for_reg
));
320 width
= width_for_reg
[reg
.width
];
322 assert(brw_inst_exec_size(devinfo
, inst
) >= 0 &&
323 brw_inst_exec_size(devinfo
, inst
) < ARRAY_SIZE(execsize_for_reg
));
324 execsize
= execsize_for_reg
[brw_inst_exec_size(devinfo
, inst
)];
326 /* Restrictions from 3.3.10: Register Region Restrictions. */
328 assert(execsize
>= width
);
331 if (execsize
== width
&& hstride
!= 0) {
332 assert(vstride
== -1 || vstride
== width
* hstride
);
336 if (execsize
== width
&& hstride
== 0) {
337 /* no restriction on vstride. */
342 assert(hstride
== 0);
346 if (execsize
== 1 && width
== 1) {
347 assert(hstride
== 0);
348 assert(vstride
== 0);
352 if (vstride
== 0 && hstride
== 0) {
356 /* 10. Check destination issues. */
360 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
362 const struct gen_device_info
*devinfo
= p
->devinfo
;
364 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
365 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
366 else if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
367 assert(reg
.nr
< 128);
369 gen7_convert_mrf_to_grf(p
, ®
);
371 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
372 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
373 /* Any source modifiers or regions will be ignored, since this just
374 * identifies the MRF/GRF to start reading the message contents from.
375 * Check for some likely failures.
379 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
382 validate_reg(devinfo
, inst
, reg
);
384 brw_inst_set_src0_reg_file(devinfo
, inst
, reg
.file
);
385 brw_inst_set_src0_reg_type(devinfo
, inst
,
386 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
387 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
388 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
389 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
391 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
392 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
393 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
394 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
395 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
396 reg
.type
== BRW_REGISTER_TYPE_Q
)
397 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
399 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
401 if (type_sz(reg
.type
) < 8) {
402 brw_inst_set_src1_reg_file(devinfo
, inst
,
403 BRW_ARCHITECTURE_REGISTER_FILE
);
404 brw_inst_set_src1_reg_type(devinfo
, inst
,
405 brw_inst_src0_reg_type(devinfo
, inst
));
408 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
409 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
410 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
411 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
413 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
416 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
418 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
419 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
421 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
425 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
426 if (reg
.width
== BRW_WIDTH_1
&&
427 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
428 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
429 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
430 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
432 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
433 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
434 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
437 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
438 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
439 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
440 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
441 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
442 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
443 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
444 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
446 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
447 /* This is an oddity of the fact we're using the same
448 * descriptions for registers in align_16 as align_1:
450 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
451 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
452 reg
.type
== BRW_REGISTER_TYPE_DF
&&
453 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
456 * "For Align16 access mode, only encodings of 0000 and 0011
457 * are allowed. Other codes are reserved."
459 * Presumably the DevSNB behavior applies to IVB as well.
461 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
463 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
471 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
473 const struct gen_device_info
*devinfo
= p
->devinfo
;
475 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
476 assert(reg
.nr
< 128);
478 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
480 * "Accumulator registers may be accessed explicitly as src0
483 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
484 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
486 gen7_convert_mrf_to_grf(p
, ®
);
487 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
489 validate_reg(devinfo
, inst
, reg
);
491 brw_inst_set_src1_reg_file(devinfo
, inst
, reg
.file
);
492 brw_inst_set_src1_reg_type(devinfo
, inst
,
493 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
494 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
495 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
497 /* Only src1 can be immediate in two-argument instructions.
499 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
501 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
502 /* two-argument instructions can only use 32-bit immediates */
503 assert(type_sz(reg
.type
) < 8);
504 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
506 /* This is a hardware restriction, which may or may not be lifted
509 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
510 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
512 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
513 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
514 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
516 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
519 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
520 if (reg
.width
== BRW_WIDTH_1
&&
521 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
522 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
523 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
524 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
526 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
527 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
528 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
531 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
532 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
533 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
534 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
535 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
536 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
537 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
538 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
540 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
541 /* This is an oddity of the fact we're using the same
542 * descriptions for registers in align_16 as align_1:
544 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
545 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
546 reg
.type
== BRW_REGISTER_TYPE_DF
&&
547 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
550 * "For Align16 access mode, only encodings of 0000 and 0011
551 * are allowed. Other codes are reserved."
553 * Presumably the DevSNB behavior applies to IVB as well.
555 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
557 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
564 * Set the Message Descriptor and Extended Message Descriptor fields
567 * \note This zeroes out the Function Control bits, so it must be called
568 * \b before filling out any message-specific data. Callers can
569 * choose not to fill in irrelevant bits; they will be zero.
572 brw_set_message_descriptor(struct brw_codegen
*p
,
574 enum brw_message_target sfid
,
576 unsigned response_length
,
580 const struct gen_device_info
*devinfo
= p
->devinfo
;
582 brw_set_src1(p
, inst
, brw_imm_d(0));
584 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
585 * itself; instead, it will be a MOV/OR into the address register.
587 * In this case, we avoid setting the extended message descriptor bits,
588 * since they go on the later SEND/SENDC instead and if set here would
589 * instead clobber the conditionalmod bits.
591 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
592 if (opcode
== BRW_OPCODE_SEND
|| opcode
== BRW_OPCODE_SENDC
) {
593 brw_inst_set_sfid(devinfo
, inst
, sfid
);
596 brw_inst_set_mlen(devinfo
, inst
, msg_length
);
597 brw_inst_set_rlen(devinfo
, inst
, response_length
);
598 brw_inst_set_eot(devinfo
, inst
, end_of_thread
);
600 if (devinfo
->gen
>= 5) {
601 brw_inst_set_header_present(devinfo
, inst
, header_present
);
605 static void brw_set_math_message( struct brw_codegen
*p
,
608 unsigned integer_type
,
612 const struct gen_device_info
*devinfo
= p
->devinfo
;
614 unsigned response_length
;
616 /* Infer message length from the function */
618 case BRW_MATH_FUNCTION_POW
:
619 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
620 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
621 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
629 /* Infer response length from the function */
631 case BRW_MATH_FUNCTION_SINCOS
:
632 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
641 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
642 msg_length
, response_length
, false, false);
643 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
644 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
645 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
646 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
647 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
648 brw_inst_set_saturate(devinfo
, inst
, 0);
652 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
655 unsigned response_length
,
658 const struct gen_device_info
*devinfo
= p
->devinfo
;
660 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
661 1, response_length
, true, end_of_thread
);
662 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
663 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
664 /* The following fields are not used by FF_SYNC: */
665 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
666 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
667 brw_inst_set_urb_used(devinfo
, insn
, 0);
668 brw_inst_set_urb_complete(devinfo
, insn
, 0);
671 static void brw_set_urb_message( struct brw_codegen
*p
,
673 enum brw_urb_write_flags flags
,
675 unsigned response_length
,
677 unsigned swizzle_control
)
679 const struct gen_device_info
*devinfo
= p
->devinfo
;
681 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
682 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
683 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
685 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
686 msg_length
, response_length
, true,
687 flags
& BRW_URB_WRITE_EOT
);
689 if (flags
& BRW_URB_WRITE_OWORD
) {
690 assert(msg_length
== 2); /* header + one OWORD of data */
691 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
693 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
696 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
697 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
699 if (devinfo
->gen
< 8) {
700 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
703 if (devinfo
->gen
< 7) {
704 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
705 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
707 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
708 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
713 brw_set_dp_write_message(struct brw_codegen
*p
,
715 unsigned binding_table_index
,
716 unsigned msg_control
,
718 unsigned target_cache
,
721 unsigned last_render_target
,
722 unsigned response_length
,
723 unsigned end_of_thread
,
724 unsigned send_commit_msg
)
726 const struct gen_device_info
*devinfo
= p
->devinfo
;
727 const unsigned sfid
= (devinfo
->gen
>= 6 ? target_cache
:
728 BRW_SFID_DATAPORT_WRITE
);
730 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
731 header_present
, end_of_thread
);
733 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
734 brw_inst_set_dp_write_msg_type(devinfo
, insn
, msg_type
);
735 brw_inst_set_dp_write_msg_control(devinfo
, insn
, msg_control
);
736 brw_inst_set_rt_last(devinfo
, insn
, last_render_target
);
737 if (devinfo
->gen
< 7) {
738 brw_inst_set_dp_write_commit(devinfo
, insn
, send_commit_msg
);
743 brw_set_dp_read_message(struct brw_codegen
*p
,
745 unsigned binding_table_index
,
746 unsigned msg_control
,
748 unsigned target_cache
,
751 unsigned response_length
)
753 const struct gen_device_info
*devinfo
= p
->devinfo
;
754 const unsigned sfid
= (devinfo
->gen
>= 6 ? target_cache
:
755 BRW_SFID_DATAPORT_READ
);
757 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
758 header_present
, false);
760 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
761 brw_inst_set_dp_read_msg_type(devinfo
, insn
, msg_type
);
762 brw_inst_set_dp_read_msg_control(devinfo
, insn
, msg_control
);
763 if (devinfo
->gen
< 6)
764 brw_inst_set_dp_read_target_cache(devinfo
, insn
, target_cache
);
768 brw_set_sampler_message(struct brw_codegen
*p
,
770 unsigned binding_table_index
,
773 unsigned response_length
,
775 unsigned header_present
,
777 unsigned return_format
)
779 const struct gen_device_info
*devinfo
= p
->devinfo
;
781 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
782 response_length
, header_present
, false);
784 brw_inst_set_binding_table_index(devinfo
, inst
, binding_table_index
);
785 brw_inst_set_sampler(devinfo
, inst
, sampler
);
786 brw_inst_set_sampler_msg_type(devinfo
, inst
, msg_type
);
787 if (devinfo
->gen
>= 5) {
788 brw_inst_set_sampler_simd_mode(devinfo
, inst
, simd_mode
);
789 } else if (devinfo
->gen
== 4 && !devinfo
->is_g4x
) {
790 brw_inst_set_sampler_return_format(devinfo
, inst
, return_format
);
795 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
799 bool invalidate_after_read
,
801 unsigned addr_offset
,
806 const struct gen_device_info
*devinfo
= p
->devinfo
;
807 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
808 (devinfo
->gen
>= 8 && num_regs
== 8));
809 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
812 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
813 mlen
, rlen
, header_present
, false);
814 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
815 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
816 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
817 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
818 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
819 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
822 #define next_insn brw_next_insn
824 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
826 const struct gen_device_info
*devinfo
= p
->devinfo
;
829 if (p
->nr_insn
+ 1 > p
->store_size
) {
831 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
834 p
->next_insn_offset
+= 16;
835 insn
= &p
->store
[p
->nr_insn
++];
836 memcpy(insn
, p
->current
, sizeof(*insn
));
838 brw_inst_set_opcode(devinfo
, insn
, opcode
);
843 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
844 struct brw_reg dest
, struct brw_reg src
)
846 brw_inst
*insn
= next_insn(p
, opcode
);
847 brw_set_dest(p
, insn
, dest
);
848 brw_set_src0(p
, insn
, src
);
853 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
854 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
856 /* 64-bit immediates are only supported on 1-src instructions */
857 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
858 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
860 brw_inst
*insn
= next_insn(p
, opcode
);
861 brw_set_dest(p
, insn
, dest
);
862 brw_set_src0(p
, insn
, src0
);
863 brw_set_src1(p
, insn
, src1
);
868 get_3src_subreg_nr(struct brw_reg reg
)
870 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
871 * use 32-bit units (components 0..7). Since they only support F/D/UD
872 * types, this doesn't lose any flexibility, but uses fewer bits.
874 return reg
.subnr
/ 4;
878 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
879 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
881 const struct gen_device_info
*devinfo
= p
->devinfo
;
882 brw_inst
*inst
= next_insn(p
, opcode
);
884 gen7_convert_mrf_to_grf(p
, &dest
);
886 assert(brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
);
888 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
889 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
890 assert(dest
.nr
< 128);
891 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
892 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
893 dest
.type
== BRW_REGISTER_TYPE_DF
||
894 dest
.type
== BRW_REGISTER_TYPE_D
||
895 dest
.type
== BRW_REGISTER_TYPE_UD
);
896 if (devinfo
->gen
== 6) {
897 brw_inst_set_3src_dst_reg_file(devinfo
, inst
,
898 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
900 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
901 brw_inst_set_3src_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
902 brw_inst_set_3src_dst_writemask(devinfo
, inst
, dest
.writemask
);
904 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
905 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
906 assert(src0
.nr
< 128);
907 brw_inst_set_3src_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
908 brw_inst_set_3src_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
909 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
910 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
911 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
912 brw_inst_set_3src_src0_rep_ctrl(devinfo
, inst
,
913 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
915 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
916 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
917 assert(src1
.nr
< 128);
918 brw_inst_set_3src_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
919 brw_inst_set_3src_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
920 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
921 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
922 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
923 brw_inst_set_3src_src1_rep_ctrl(devinfo
, inst
,
924 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
926 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
927 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
928 assert(src2
.nr
< 128);
929 brw_inst_set_3src_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
930 brw_inst_set_3src_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
931 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
932 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
933 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
934 brw_inst_set_3src_src2_rep_ctrl(devinfo
, inst
,
935 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
937 if (devinfo
->gen
>= 7) {
938 /* Set both the source and destination types based on dest.type,
939 * ignoring the source register types. The MAD and LRP emitters ensure
940 * that all four types are float. The BFE and BFI2 emitters, however,
941 * may send us mixed D and UD types and want us to ignore that and use
942 * the destination type.
945 case BRW_REGISTER_TYPE_F
:
946 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
947 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
949 case BRW_REGISTER_TYPE_DF
:
950 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_DF
);
951 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_DF
);
953 case BRW_REGISTER_TYPE_D
:
954 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
955 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
957 case BRW_REGISTER_TYPE_UD
:
958 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
959 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
962 unreachable("not reached");
970 /***********************************************************************
971 * Convenience routines.
974 brw_inst *brw_##OP(struct brw_codegen *p, \
975 struct brw_reg dest, \
976 struct brw_reg src0) \
978 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
982 brw_inst *brw_##OP(struct brw_codegen *p, \
983 struct brw_reg dest, \
984 struct brw_reg src0, \
985 struct brw_reg src1) \
987 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
991 brw_inst *brw_##OP(struct brw_codegen *p, \
992 struct brw_reg dest, \
993 struct brw_reg src0, \
994 struct brw_reg src1, \
995 struct brw_reg src2) \
997 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1001 brw_inst *brw_##OP(struct brw_codegen *p, \
1002 struct brw_reg dest, \
1003 struct brw_reg src0, \
1004 struct brw_reg src1, \
1005 struct brw_reg src2) \
1007 assert(dest.type == BRW_REGISTER_TYPE_F || \
1008 dest.type == BRW_REGISTER_TYPE_DF); \
1009 if (dest.type == BRW_REGISTER_TYPE_F) { \
1010 assert(src0.type == BRW_REGISTER_TYPE_F); \
1011 assert(src1.type == BRW_REGISTER_TYPE_F); \
1012 assert(src2.type == BRW_REGISTER_TYPE_F); \
1013 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
1014 assert(src0.type == BRW_REGISTER_TYPE_DF); \
1015 assert(src1.type == BRW_REGISTER_TYPE_DF); \
1016 assert(src2.type == BRW_REGISTER_TYPE_DF); \
1018 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1021 /* Rounding operations (other than RNDD) require two instructions - the first
1022 * stores a rounded value (possibly the wrong way) in the dest register, but
1023 * also sets a per-channel "increment bit" in the flag register. A predicated
1024 * add of 1.0 fixes dest to contain the desired result.
1026 * Sandybridge and later appear to round correctly without an ADD.
1029 void brw_##OP(struct brw_codegen *p, \
1030 struct brw_reg dest, \
1031 struct brw_reg src) \
1033 const struct gen_device_info *devinfo = p->devinfo; \
1034 brw_inst *rnd, *add; \
1035 rnd = next_insn(p, BRW_OPCODE_##OP); \
1036 brw_set_dest(p, rnd, dest); \
1037 brw_set_src0(p, rnd, src); \
1039 if (devinfo->gen < 6) { \
1040 /* turn on round-increments */ \
1041 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
1042 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1043 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1082 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
1084 const struct gen_device_info
*devinfo
= p
->devinfo
;
1086 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
1087 * To avoid the problems that causes, we use a <1,2,0> source region to read
1088 * each element twice.
1090 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1091 brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
&&
1092 dest
.type
== BRW_REGISTER_TYPE_DF
&&
1093 (src0
.type
== BRW_REGISTER_TYPE_F
||
1094 src0
.type
== BRW_REGISTER_TYPE_D
||
1095 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
1096 !has_scalar_region(src0
)) {
1097 assert(src0
.vstride
== BRW_VERTICAL_STRIDE_4
&&
1098 src0
.width
== BRW_WIDTH_4
&&
1099 src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1101 src0
.vstride
= BRW_VERTICAL_STRIDE_1
;
1102 src0
.width
= BRW_WIDTH_2
;
1103 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1106 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1110 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1111 struct brw_reg src0
, struct brw_reg src1
)
1114 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1115 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1116 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1117 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1118 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1121 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1122 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1123 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1124 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1125 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1128 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1132 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1133 struct brw_reg src0
, struct brw_reg src1
)
1135 assert(dest
.type
== src0
.type
);
1136 assert(src0
.type
== src1
.type
);
1137 switch (src0
.type
) {
1138 case BRW_REGISTER_TYPE_B
:
1139 case BRW_REGISTER_TYPE_UB
:
1140 case BRW_REGISTER_TYPE_W
:
1141 case BRW_REGISTER_TYPE_UW
:
1142 case BRW_REGISTER_TYPE_D
:
1143 case BRW_REGISTER_TYPE_UD
:
1146 unreachable("Bad type for brw_AVG");
1149 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1153 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1154 struct brw_reg src0
, struct brw_reg src1
)
1157 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1158 src0
.type
== BRW_REGISTER_TYPE_UD
||
1159 src1
.type
== BRW_REGISTER_TYPE_D
||
1160 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1161 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1164 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1165 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1166 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1167 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1168 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1171 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1172 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1173 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1174 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1175 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1178 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1179 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1180 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1181 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1183 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1187 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1188 struct brw_reg src0
, struct brw_reg src1
)
1190 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1191 src0
.width
= BRW_WIDTH_1
;
1192 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1193 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1197 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1198 struct brw_reg src0
, struct brw_reg src1
)
1200 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1201 src0
.width
= BRW_WIDTH_1
;
1202 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1203 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1204 src1
.width
= BRW_WIDTH_8
;
1205 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1206 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1210 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1212 const struct gen_device_info
*devinfo
= p
->devinfo
;
1213 const bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1214 /* The F32TO16 instruction doesn't support 32-bit destination types in
1215 * Align1 mode, and neither does the Gen8 implementation in terms of a
1216 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1217 * an undocumented feature.
1219 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1220 (!align16
|| devinfo
->gen
>= 8));
1224 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1226 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1227 dst
.type
== BRW_REGISTER_TYPE_W
||
1228 dst
.type
== BRW_REGISTER_TYPE_UW
||
1229 dst
.type
== BRW_REGISTER_TYPE_HF
);
1232 brw_push_insn_state(p
);
1234 if (needs_zero_fill
) {
1235 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1236 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1239 if (devinfo
->gen
>= 8) {
1240 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1242 assert(devinfo
->gen
== 7);
1243 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1246 if (needs_zero_fill
) {
1247 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1248 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1249 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1252 brw_pop_insn_state(p
);
1257 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1259 const struct gen_device_info
*devinfo
= p
->devinfo
;
1260 bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1263 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1265 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1267 * Because this instruction does not have a 16-bit floating-point
1268 * type, the source data type must be Word (W). The destination type
1269 * must be F (Float).
1271 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1272 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1274 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1275 src
.type
== BRW_REGISTER_TYPE_UW
||
1276 src
.type
== BRW_REGISTER_TYPE_HF
);
1279 if (devinfo
->gen
>= 8) {
1280 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1282 assert(devinfo
->gen
== 7);
1283 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1288 void brw_NOP(struct brw_codegen
*p
)
1290 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1291 memset(insn
, 0, sizeof(*insn
));
1292 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1299 /***********************************************************************
1300 * Comparisons, if/else/endif
1304 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1305 unsigned predicate_control
)
1307 const struct gen_device_info
*devinfo
= p
->devinfo
;
1308 struct brw_reg ip
= brw_ip_reg();
1309 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1311 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_2
);
1312 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1313 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1314 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1320 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1322 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1324 p
->if_stack_depth
++;
1325 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1326 p
->if_stack_array_size
*= 2;
1327 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1328 p
->if_stack_array_size
);
1333 pop_if_stack(struct brw_codegen
*p
)
1335 p
->if_stack_depth
--;
1336 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1340 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1342 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1343 p
->loop_stack_array_size
*= 2;
1344 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1345 p
->loop_stack_array_size
);
1346 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1347 p
->loop_stack_array_size
);
1350 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1351 p
->loop_stack_depth
++;
1352 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1356 get_inner_do_insn(struct brw_codegen
*p
)
1358 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1361 /* EU takes the value from the flag register and pushes it onto some
1362 * sort of a stack (presumably merging with any flag value already on
1363 * the stack). Within an if block, the flags at the top of the stack
1364 * control execution on each channel of the unit, eg. on each of the
1365 * 16 pixel values in our wm programs.
1367 * When the matching 'else' instruction is reached (presumably by
1368 * countdown of the instruction count patched in by our ELSE/ENDIF
1369 * functions), the relevant flags are inverted.
1371 * When the matching 'endif' instruction is reached, the flags are
1372 * popped off. If the stack is now empty, normal execution resumes.
1375 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1377 const struct gen_device_info
*devinfo
= p
->devinfo
;
1380 insn
= next_insn(p
, BRW_OPCODE_IF
);
1382 /* Override the defaults for this instruction:
1384 if (devinfo
->gen
< 6) {
1385 brw_set_dest(p
, insn
, brw_ip_reg());
1386 brw_set_src0(p
, insn
, brw_ip_reg());
1387 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1388 } else if (devinfo
->gen
== 6) {
1389 brw_set_dest(p
, insn
, brw_imm_w(0));
1390 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1391 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1392 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1393 } else if (devinfo
->gen
== 7) {
1394 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1395 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1396 brw_set_src1(p
, insn
, brw_imm_w(0));
1397 brw_inst_set_jip(devinfo
, insn
, 0);
1398 brw_inst_set_uip(devinfo
, insn
, 0);
1400 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1401 brw_set_src0(p
, insn
, brw_imm_d(0));
1402 brw_inst_set_jip(devinfo
, insn
, 0);
1403 brw_inst_set_uip(devinfo
, insn
, 0);
1406 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1407 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1408 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1409 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1410 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1411 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1413 push_if_stack(p
, insn
);
1414 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1418 /* This function is only used for gen6-style IF instructions with an
1419 * embedded comparison (conditional modifier). It is not used on gen7.
1422 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1423 struct brw_reg src0
, struct brw_reg src1
)
1425 const struct gen_device_info
*devinfo
= p
->devinfo
;
1428 insn
= next_insn(p
, BRW_OPCODE_IF
);
1430 brw_set_dest(p
, insn
, brw_imm_w(0));
1431 brw_inst_set_exec_size(devinfo
, insn
,
1432 brw_inst_exec_size(devinfo
, p
->current
));
1433 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1434 brw_set_src0(p
, insn
, src0
);
1435 brw_set_src1(p
, insn
, src1
);
1437 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1438 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1439 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1441 push_if_stack(p
, insn
);
1446 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1449 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1450 brw_inst
*if_inst
, brw_inst
*else_inst
)
1452 const struct gen_device_info
*devinfo
= p
->devinfo
;
1454 /* The next instruction (where the ENDIF would be, if it existed) */
1455 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1457 assert(p
->single_program_flow
);
1458 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1459 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1460 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1462 /* Convert IF to an ADD instruction that moves the instruction pointer
1463 * to the first instruction of the ELSE block. If there is no ELSE
1464 * block, point to where ENDIF would be. Reverse the predicate.
1466 * There's no need to execute an ENDIF since we don't need to do any
1467 * stack operations, and if we're currently executing, we just want to
1468 * continue normally.
1470 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1471 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1473 if (else_inst
!= NULL
) {
1474 /* Convert ELSE to an ADD instruction that points where the ENDIF
1477 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1479 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1480 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1482 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1487 * Patch IF and ELSE instructions with appropriate jump targets.
1490 patch_IF_ELSE(struct brw_codegen
*p
,
1491 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1493 const struct gen_device_info
*devinfo
= p
->devinfo
;
1495 /* We shouldn't be patching IF and ELSE instructions in single program flow
1496 * mode when gen < 6, because in single program flow mode on those
1497 * platforms, we convert flow control instructions to conditional ADDs that
1498 * operate on IP (see brw_ENDIF).
1500 * However, on Gen6, writing to IP doesn't work in single program flow mode
1501 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1502 * not be updated by non-flow control instructions."). And on later
1503 * platforms, there is no significant benefit to converting control flow
1504 * instructions to conditional ADDs. So we do patch IF and ELSE
1505 * instructions in single program flow mode on those platforms.
1507 if (devinfo
->gen
< 6)
1508 assert(!p
->single_program_flow
);
1510 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1511 assert(endif_inst
!= NULL
);
1512 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1514 unsigned br
= brw_jump_scale(devinfo
);
1516 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1517 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1519 if (else_inst
== NULL
) {
1520 /* Patch IF -> ENDIF */
1521 if (devinfo
->gen
< 6) {
1522 /* Turn it into an IFF, which means no mask stack operations for
1523 * all-false and jumping past the ENDIF.
1525 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1526 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1527 br
* (endif_inst
- if_inst
+ 1));
1528 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1529 } else if (devinfo
->gen
== 6) {
1530 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1531 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1533 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1534 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1537 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1539 /* Patch IF -> ELSE */
1540 if (devinfo
->gen
< 6) {
1541 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1542 br
* (else_inst
- if_inst
));
1543 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1544 } else if (devinfo
->gen
== 6) {
1545 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1546 br
* (else_inst
- if_inst
+ 1));
1549 /* Patch ELSE -> ENDIF */
1550 if (devinfo
->gen
< 6) {
1551 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1554 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1555 br
* (endif_inst
- else_inst
+ 1));
1556 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1557 } else if (devinfo
->gen
== 6) {
1558 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1559 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1560 br
* (endif_inst
- else_inst
));
1562 /* The IF instruction's JIP should point just past the ELSE */
1563 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1564 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1565 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1566 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1567 if (devinfo
->gen
>= 8) {
1568 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1569 * should point to ENDIF.
1571 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1578 brw_ELSE(struct brw_codegen
*p
)
1580 const struct gen_device_info
*devinfo
= p
->devinfo
;
1583 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1585 if (devinfo
->gen
< 6) {
1586 brw_set_dest(p
, insn
, brw_ip_reg());
1587 brw_set_src0(p
, insn
, brw_ip_reg());
1588 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1589 } else if (devinfo
->gen
== 6) {
1590 brw_set_dest(p
, insn
, brw_imm_w(0));
1591 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1592 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1593 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1594 } else if (devinfo
->gen
== 7) {
1595 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1596 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1597 brw_set_src1(p
, insn
, brw_imm_w(0));
1598 brw_inst_set_jip(devinfo
, insn
, 0);
1599 brw_inst_set_uip(devinfo
, insn
, 0);
1601 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1602 brw_set_src0(p
, insn
, brw_imm_d(0));
1603 brw_inst_set_jip(devinfo
, insn
, 0);
1604 brw_inst_set_uip(devinfo
, insn
, 0);
1607 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1608 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1609 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1610 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1612 push_if_stack(p
, insn
);
1616 brw_ENDIF(struct brw_codegen
*p
)
1618 const struct gen_device_info
*devinfo
= p
->devinfo
;
1619 brw_inst
*insn
= NULL
;
1620 brw_inst
*else_inst
= NULL
;
1621 brw_inst
*if_inst
= NULL
;
1623 bool emit_endif
= true;
1625 /* In single program flow mode, we can express IF and ELSE instructions
1626 * equivalently as ADD instructions that operate on IP. On platforms prior
1627 * to Gen6, flow control instructions cause an implied thread switch, so
1628 * this is a significant savings.
1630 * However, on Gen6, writing to IP doesn't work in single program flow mode
1631 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1632 * not be updated by non-flow control instructions."). And on later
1633 * platforms, there is no significant benefit to converting control flow
1634 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1637 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1641 * A single next_insn() may change the base address of instruction store
1642 * memory(p->store), so call it first before referencing the instruction
1643 * store pointer from an index
1646 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1648 /* Pop the IF and (optional) ELSE instructions from the stack */
1649 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1650 tmp
= pop_if_stack(p
);
1651 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1653 tmp
= pop_if_stack(p
);
1658 /* ENDIF is useless; don't bother emitting it. */
1659 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1663 if (devinfo
->gen
< 6) {
1664 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1665 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1666 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1667 } else if (devinfo
->gen
== 6) {
1668 brw_set_dest(p
, insn
, brw_imm_w(0));
1669 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1670 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1671 } else if (devinfo
->gen
== 7) {
1672 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1673 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1674 brw_set_src1(p
, insn
, brw_imm_w(0));
1676 brw_set_src0(p
, insn
, brw_imm_d(0));
1679 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1680 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1681 if (devinfo
->gen
< 6)
1682 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1684 /* Also pop item off the stack in the endif instruction: */
1685 if (devinfo
->gen
< 6) {
1686 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1687 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1688 } else if (devinfo
->gen
== 6) {
1689 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1691 brw_inst_set_jip(devinfo
, insn
, 2);
1693 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1697 brw_BREAK(struct brw_codegen
*p
)
1699 const struct gen_device_info
*devinfo
= p
->devinfo
;
1702 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1703 if (devinfo
->gen
>= 8) {
1704 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1705 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1706 } else if (devinfo
->gen
>= 6) {
1707 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1708 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1709 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1711 brw_set_dest(p
, insn
, brw_ip_reg());
1712 brw_set_src0(p
, insn
, brw_ip_reg());
1713 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1714 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1715 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1717 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1718 brw_inst_set_exec_size(devinfo
, insn
,
1719 brw_inst_exec_size(devinfo
, p
->current
));
1725 brw_CONT(struct brw_codegen
*p
)
1727 const struct gen_device_info
*devinfo
= p
->devinfo
;
1730 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1731 brw_set_dest(p
, insn
, brw_ip_reg());
1732 if (devinfo
->gen
>= 8) {
1733 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1735 brw_set_src0(p
, insn
, brw_ip_reg());
1736 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1739 if (devinfo
->gen
< 6) {
1740 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1741 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1743 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1744 brw_inst_set_exec_size(devinfo
, insn
,
1745 brw_inst_exec_size(devinfo
, p
->current
));
1750 gen6_HALT(struct brw_codegen
*p
)
1752 const struct gen_device_info
*devinfo
= p
->devinfo
;
1755 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1756 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1757 if (devinfo
->gen
>= 8) {
1758 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1760 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1761 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1764 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1765 brw_inst_set_exec_size(devinfo
, insn
,
1766 brw_inst_exec_size(devinfo
, p
->current
));
1772 * The DO/WHILE is just an unterminated loop -- break or continue are
1773 * used for control within the loop. We have a few ways they can be
1776 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1777 * jip and no DO instruction.
1779 * For non-uniform control flow pre-gen6, there's a DO instruction to
1780 * push the mask, and a WHILE to jump back, and BREAK to get out and
1783 * For gen6, there's no more mask stack, so no need for DO. WHILE
1784 * just points back to the first instruction of the loop.
1787 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1789 const struct gen_device_info
*devinfo
= p
->devinfo
;
1791 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1792 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1793 return &p
->store
[p
->nr_insn
];
1795 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1797 push_loop_stack(p
, insn
);
1799 /* Override the defaults for this instruction:
1801 brw_set_dest(p
, insn
, brw_null_reg());
1802 brw_set_src0(p
, insn
, brw_null_reg());
1803 brw_set_src1(p
, insn
, brw_null_reg());
1805 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1806 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1807 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1814 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1817 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1818 * nesting, since it can always just point to the end of the block/current loop.
1821 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1823 const struct gen_device_info
*devinfo
= p
->devinfo
;
1824 brw_inst
*do_inst
= get_inner_do_insn(p
);
1826 unsigned br
= brw_jump_scale(devinfo
);
1828 assert(devinfo
->gen
< 6);
1830 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1831 /* If the jump count is != 0, that means that this instruction has already
1832 * been patched because it's part of a loop inside of the one we're
1835 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1836 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1837 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1838 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1839 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1840 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1846 brw_WHILE(struct brw_codegen
*p
)
1848 const struct gen_device_info
*devinfo
= p
->devinfo
;
1849 brw_inst
*insn
, *do_insn
;
1850 unsigned br
= brw_jump_scale(devinfo
);
1852 if (devinfo
->gen
>= 6) {
1853 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1854 do_insn
= get_inner_do_insn(p
);
1856 if (devinfo
->gen
>= 8) {
1857 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1858 brw_set_src0(p
, insn
, brw_imm_d(0));
1859 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1860 } else if (devinfo
->gen
== 7) {
1861 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1862 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1863 brw_set_src1(p
, insn
, brw_imm_w(0));
1864 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1866 brw_set_dest(p
, insn
, brw_imm_w(0));
1867 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1868 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1869 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1872 brw_inst_set_exec_size(devinfo
, insn
,
1873 brw_inst_exec_size(devinfo
, p
->current
));
1876 if (p
->single_program_flow
) {
1877 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1878 do_insn
= get_inner_do_insn(p
);
1880 brw_set_dest(p
, insn
, brw_ip_reg());
1881 brw_set_src0(p
, insn
, brw_ip_reg());
1882 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1883 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1885 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1886 do_insn
= get_inner_do_insn(p
);
1888 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1890 brw_set_dest(p
, insn
, brw_ip_reg());
1891 brw_set_src0(p
, insn
, brw_ip_reg());
1892 brw_set_src1(p
, insn
, brw_imm_d(0));
1894 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1895 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1896 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1898 brw_patch_break_cont(p
, insn
);
1901 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1903 p
->loop_stack_depth
--;
1910 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1912 const struct gen_device_info
*devinfo
= p
->devinfo
;
1913 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1916 if (devinfo
->gen
>= 5)
1919 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1920 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1922 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1923 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1926 /* To integrate with the above, it makes sense that the comparison
1927 * instruction should populate the flag register. It might be simpler
1928 * just to use the flag reg for most WM tasks?
1930 void brw_CMP(struct brw_codegen
*p
,
1931 struct brw_reg dest
,
1932 unsigned conditional
,
1933 struct brw_reg src0
,
1934 struct brw_reg src1
)
1936 const struct gen_device_info
*devinfo
= p
->devinfo
;
1937 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1939 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1940 brw_set_dest(p
, insn
, dest
);
1941 brw_set_src0(p
, insn
, src0
);
1942 brw_set_src1(p
, insn
, src1
);
1944 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1946 * "Any CMP instruction with a null destination must use a {switch}."
1948 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1949 * mentioned on their work-arounds pages.
1951 if (devinfo
->gen
== 7) {
1952 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1953 dest
.nr
== BRW_ARF_NULL
) {
1954 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1959 /***********************************************************************
1960 * Helpers for the various SEND message types:
1963 /** Extended math function, float[8].
1965 void gen4_math(struct brw_codegen
*p
,
1966 struct brw_reg dest
,
1968 unsigned msg_reg_nr
,
1970 unsigned precision
)
1972 const struct gen_device_info
*devinfo
= p
->devinfo
;
1973 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1975 if (has_scalar_region(src
)) {
1976 data_type
= BRW_MATH_DATA_SCALAR
;
1978 data_type
= BRW_MATH_DATA_VECTOR
;
1981 assert(devinfo
->gen
< 6);
1983 /* Example code doesn't set predicate_control for send
1986 brw_inst_set_pred_control(devinfo
, insn
, 0);
1987 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1989 brw_set_dest(p
, insn
, dest
);
1990 brw_set_src0(p
, insn
, src
);
1991 brw_set_math_message(p
,
1994 src
.type
== BRW_REGISTER_TYPE_D
,
1999 void gen6_math(struct brw_codegen
*p
,
2000 struct brw_reg dest
,
2002 struct brw_reg src0
,
2003 struct brw_reg src1
)
2005 const struct gen_device_info
*devinfo
= p
->devinfo
;
2006 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
2008 assert(devinfo
->gen
>= 6);
2010 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
2011 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
2013 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2014 if (devinfo
->gen
== 6) {
2015 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2016 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2019 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2020 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2021 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2022 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2023 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2024 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2025 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2027 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
2028 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
2031 /* Source modifiers are ignored for extended math instructions on Gen6. */
2032 if (devinfo
->gen
== 6) {
2033 assert(!src0
.negate
);
2035 assert(!src1
.negate
);
2039 brw_inst_set_math_function(devinfo
, insn
, function
);
2041 brw_set_dest(p
, insn
, dest
);
2042 brw_set_src0(p
, insn
, src0
);
2043 brw_set_src1(p
, insn
, src1
);
2047 * Return the right surface index to access the thread scratch space using
2048 * stateless dataport messages.
2051 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2053 /* The scratch space is thread-local so IA coherency is unnecessary. */
2054 if (p
->devinfo
->gen
>= 8)
2055 return GEN8_BTI_STATELESS_NON_COHERENT
;
2057 return BRW_BTI_STATELESS
;
2061 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2062 * using a constant offset per channel.
2064 * The offset must be aligned to oword size (16 bytes). Used for
2065 * register spilling.
2067 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2072 const struct gen_device_info
*devinfo
= p
->devinfo
;
2073 const unsigned target_cache
=
2074 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2075 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2076 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2079 if (devinfo
->gen
>= 6)
2082 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2084 const unsigned mlen
= 1 + num_regs
;
2086 /* Set up the message header. This is g0, with g0.2 filled with
2087 * the offset. We don't want to leave our offset around in g0 or
2088 * it'll screw up texture samples, so set it up inside the message
2092 brw_push_insn_state(p
);
2093 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2094 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2095 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2097 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2099 /* set message header global offset field (reg 0, element 2) */
2101 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2103 2), BRW_REGISTER_TYPE_UD
),
2104 brw_imm_ud(offset
));
2106 brw_pop_insn_state(p
);
2110 struct brw_reg dest
;
2111 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2112 int send_commit_msg
;
2113 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2114 BRW_REGISTER_TYPE_UW
);
2116 brw_inst_set_compression(devinfo
, insn
, false);
2118 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2119 src_header
= vec16(src_header
);
2121 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2122 if (devinfo
->gen
< 6)
2123 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2125 /* Until gen6, writes followed by reads from the same location
2126 * are not guaranteed to be ordered unless write_commit is set.
2127 * If set, then a no-op write is issued to the destination
2128 * register to set a dependency, and a read from the destination
2129 * can be used to ensure the ordering.
2131 * For gen6, only writes between different threads need ordering
2132 * protection. Our use of DP writes is all about register
2133 * spilling within a thread.
2135 if (devinfo
->gen
>= 6) {
2136 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2137 send_commit_msg
= 0;
2140 send_commit_msg
= 1;
2143 brw_set_dest(p
, insn
, dest
);
2144 if (devinfo
->gen
>= 6) {
2145 brw_set_src0(p
, insn
, mrf
);
2147 brw_set_src0(p
, insn
, brw_null_reg());
2150 if (devinfo
->gen
>= 6)
2151 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2153 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2155 brw_set_dp_write_message(p
,
2157 brw_scratch_surface_idx(p
),
2158 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2162 true, /* header_present */
2163 0, /* not a render target */
2164 send_commit_msg
, /* response_length */
2172 * Read a block of owords (half a GRF each) from the scratch buffer
2173 * using a constant index per channel.
2175 * Offset must be aligned to oword size (16 bytes). Used for register
2179 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2180 struct brw_reg dest
,
2185 const struct gen_device_info
*devinfo
= p
->devinfo
;
2187 if (devinfo
->gen
>= 6)
2190 if (p
->devinfo
->gen
>= 7) {
2191 /* On gen 7 and above, we no longer have message registers and we can
2192 * send from any register we want. By using the destination register
2193 * for the message, we guarantee that the implied message write won't
2194 * accidentally overwrite anything. This has been a problem because
2195 * the MRF registers and source for the final FB write are both fixed
2198 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2200 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2202 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2204 const unsigned rlen
= num_regs
;
2205 const unsigned target_cache
=
2206 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2207 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2208 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2211 brw_push_insn_state(p
);
2212 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2213 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2214 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2216 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2218 /* set message header global offset field (reg 0, element 2) */
2219 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2221 brw_pop_insn_state(p
);
2225 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2227 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2228 brw_inst_set_compression(devinfo
, insn
, false);
2230 brw_set_dest(p
, insn
, dest
); /* UW? */
2231 if (devinfo
->gen
>= 6) {
2232 brw_set_src0(p
, insn
, mrf
);
2234 brw_set_src0(p
, insn
, brw_null_reg());
2235 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2238 brw_set_dp_read_message(p
,
2240 brw_scratch_surface_idx(p
),
2241 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2242 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2245 true, /* header_present */
2251 gen7_block_read_scratch(struct brw_codegen
*p
,
2252 struct brw_reg dest
,
2256 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2257 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2259 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2261 /* The HW requires that the header is present; this is to get the g0.5
2264 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2266 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2267 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2268 * is 32 bytes, which happens to be the size of a register.
2271 assert(offset
< (1 << 12));
2273 gen7_set_dp_scratch_message(p
, insn
,
2274 false, /* scratch read */
2276 false, /* invalidate after read */
2279 1, /* mlen: just g0 */
2280 num_regs
, /* rlen */
2281 true); /* header present */
2285 * Read float[4] vectors from the data port constant cache.
2286 * Location (in buffer) should be a multiple of 16.
2287 * Used for fetching shader constants.
2289 void brw_oword_block_read(struct brw_codegen
*p
,
2290 struct brw_reg dest
,
2293 uint32_t bind_table_index
)
2295 const struct gen_device_info
*devinfo
= p
->devinfo
;
2296 const unsigned target_cache
=
2297 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2298 BRW_DATAPORT_READ_TARGET_DATA_CACHE
);
2299 const unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, p
->current
);
2301 /* On newer hardware, offset is in units of owords. */
2302 if (devinfo
->gen
>= 6)
2305 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2307 brw_push_insn_state(p
);
2308 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2309 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2310 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2312 brw_push_insn_state(p
);
2313 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2314 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2316 /* set message header global offset field (reg 0, element 2) */
2318 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2320 2), BRW_REGISTER_TYPE_UD
),
2321 brw_imm_ud(offset
));
2322 brw_pop_insn_state(p
);
2324 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2326 /* cast dest to a uword[8] vector */
2327 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2329 brw_set_dest(p
, insn
, dest
);
2330 if (devinfo
->gen
>= 6) {
2331 brw_set_src0(p
, insn
, mrf
);
2333 brw_set_src0(p
, insn
, brw_null_reg());
2334 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2337 brw_set_dp_read_message(p
, insn
, bind_table_index
,
2338 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2339 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2342 true, /* header_present */
2343 DIV_ROUND_UP(exec_size
, 8)); /* response_length */
2345 brw_pop_insn_state(p
);
2349 void brw_fb_WRITE(struct brw_codegen
*p
,
2350 struct brw_reg payload
,
2351 struct brw_reg implied_header
,
2352 unsigned msg_control
,
2353 unsigned binding_table_index
,
2354 unsigned msg_length
,
2355 unsigned response_length
,
2357 bool last_render_target
,
2358 bool header_present
)
2360 const struct gen_device_info
*devinfo
= p
->devinfo
;
2361 const unsigned target_cache
=
2362 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2363 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2366 struct brw_reg dest
, src0
;
2368 if (brw_inst_exec_size(devinfo
, p
->current
) >= BRW_EXECUTE_16
)
2369 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2371 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2373 if (devinfo
->gen
>= 6) {
2374 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2376 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2378 brw_inst_set_compression(devinfo
, insn
, false);
2380 if (devinfo
->gen
>= 6) {
2381 /* headerless version, just submit color payload */
2384 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2386 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2387 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2388 src0
= implied_header
;
2390 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2393 brw_set_dest(p
, insn
, dest
);
2394 brw_set_src0(p
, insn
, src0
);
2395 brw_set_dp_write_message(p
,
2397 binding_table_index
,
2406 0 /* send_commit_msg */);
2410 gen9_fb_READ(struct brw_codegen
*p
,
2412 struct brw_reg payload
,
2413 unsigned binding_table_index
,
2414 unsigned msg_length
,
2415 unsigned response_length
,
2418 const struct gen_device_info
*devinfo
= p
->devinfo
;
2419 assert(devinfo
->gen
>= 9);
2420 const unsigned msg_subtype
=
2421 brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
? 0 : 1;
2422 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2424 brw_set_dest(p
, insn
, dst
);
2425 brw_set_src0(p
, insn
, payload
);
2426 brw_set_dp_read_message(p
, insn
, binding_table_index
,
2427 per_sample
<< 5 | msg_subtype
,
2428 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2429 GEN6_SFID_DATAPORT_RENDER_CACHE
,
2430 msg_length
, true /* header_present */,
2432 brw_inst_set_rt_slot_group(devinfo
, insn
,
2433 brw_inst_qtr_control(devinfo
, p
->current
) / 2);
2439 * Texture sample instruction.
2440 * Note: the msg_type plus msg_length values determine exactly what kind
2441 * of sampling operation is performed. See volume 4, page 161 of docs.
2443 void brw_SAMPLE(struct brw_codegen
*p
,
2444 struct brw_reg dest
,
2445 unsigned msg_reg_nr
,
2446 struct brw_reg src0
,
2447 unsigned binding_table_index
,
2450 unsigned response_length
,
2451 unsigned msg_length
,
2452 unsigned header_present
,
2454 unsigned return_format
)
2456 const struct gen_device_info
*devinfo
= p
->devinfo
;
2459 if (msg_reg_nr
!= -1)
2460 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2462 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2463 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2465 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2467 * "Instruction compression is not allowed for this instruction (that
2468 * is, send). The hardware behavior is undefined if this instruction is
2469 * set as compressed. However, compress control can be set to "SecHalf"
2470 * to affect the EMask generation."
2472 * No similar wording is found in later PRMs, but there are examples
2473 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2474 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2475 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2477 brw_inst_set_compression(devinfo
, insn
, false);
2479 if (devinfo
->gen
< 6)
2480 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2482 brw_set_dest(p
, insn
, dest
);
2483 brw_set_src0(p
, insn
, src0
);
2484 brw_set_sampler_message(p
, insn
,
2485 binding_table_index
,
2495 /* Adjust the message header's sampler state pointer to
2496 * select the correct group of 16 samplers.
2498 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2499 struct brw_reg header
,
2500 struct brw_reg sampler_index
)
2502 /* The "Sampler Index" field can only store values between 0 and 15.
2503 * However, we can add an offset to the "Sampler State Pointer"
2504 * field, effectively selecting a different set of 16 samplers.
2506 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2507 * offset, and each sampler state is only 16-bytes, so we can't
2508 * exclusively use the offset - we have to use both.
2511 const struct gen_device_info
*devinfo
= p
->devinfo
;
2513 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2514 const int sampler_state_size
= 16; /* 16 bytes */
2515 uint32_t sampler
= sampler_index
.ud
;
2517 if (sampler
>= 16) {
2518 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2520 get_element_ud(header
, 3),
2521 get_element_ud(brw_vec8_grf(0, 0), 3),
2522 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2525 /* Non-const sampler array indexing case */
2526 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2530 struct brw_reg temp
= get_element_ud(header
, 3);
2532 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2533 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2535 get_element_ud(header
, 3),
2536 get_element_ud(brw_vec8_grf(0, 0), 3),
2541 /* All these variables are pretty confusing - we might be better off
2542 * using bitmasks and macros for this, in the old style. Or perhaps
2543 * just having the caller instantiate the fields in dword3 itself.
2545 void brw_urb_WRITE(struct brw_codegen
*p
,
2546 struct brw_reg dest
,
2547 unsigned msg_reg_nr
,
2548 struct brw_reg src0
,
2549 enum brw_urb_write_flags flags
,
2550 unsigned msg_length
,
2551 unsigned response_length
,
2555 const struct gen_device_info
*devinfo
= p
->devinfo
;
2558 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2560 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2561 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2562 brw_push_insn_state(p
);
2563 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2564 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2565 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2566 BRW_REGISTER_TYPE_UD
),
2567 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2568 brw_imm_ud(0xff00));
2569 brw_pop_insn_state(p
);
2572 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2574 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2576 brw_set_dest(p
, insn
, dest
);
2577 brw_set_src0(p
, insn
, src0
);
2578 brw_set_src1(p
, insn
, brw_imm_d(0));
2580 if (devinfo
->gen
< 6)
2581 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2583 brw_set_urb_message(p
,
2593 brw_send_indirect_message(struct brw_codegen
*p
,
2596 struct brw_reg payload
,
2597 struct brw_reg desc
)
2599 const struct gen_device_info
*devinfo
= p
->devinfo
;
2600 struct brw_inst
*send
;
2603 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2605 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2607 /* We hold on to the setup instruction (the SEND in the direct case, the OR
2608 * in the indirect case) by its index in the instruction store. The
2609 * pointer returned by next_insn() may become invalid if emitting the SEND
2610 * in the indirect case reallocs the store.
2613 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2615 send
= next_insn(p
, BRW_OPCODE_SEND
);
2616 brw_set_src1(p
, send
, desc
);
2619 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2621 brw_push_insn_state(p
);
2622 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2623 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2624 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2626 /* Load the indirect descriptor to an address register using OR so the
2627 * caller can specify additional descriptor bits with the usual
2628 * brw_set_*_message() helper functions.
2631 brw_OR(p
, addr
, desc
, brw_imm_ud(0));
2633 brw_pop_insn_state(p
);
2635 send
= next_insn(p
, BRW_OPCODE_SEND
);
2636 brw_set_src1(p
, send
, addr
);
2639 if (dst
.width
< BRW_EXECUTE_8
)
2640 brw_inst_set_exec_size(devinfo
, send
, dst
.width
);
2642 brw_set_dest(p
, send
, dst
);
2643 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2644 brw_inst_set_sfid(devinfo
, send
, sfid
);
2646 return &p
->store
[setup
];
2649 static struct brw_inst
*
2650 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2653 struct brw_reg payload
,
2654 struct brw_reg surface
,
2655 unsigned message_len
,
2656 unsigned response_len
,
2657 bool header_present
)
2659 const struct gen_device_info
*devinfo
= p
->devinfo
;
2660 struct brw_inst
*insn
;
2662 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2663 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2665 brw_push_insn_state(p
);
2666 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2667 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2668 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2670 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2671 * some surface array is accessed out of bounds.
2673 insn
= brw_AND(p
, addr
,
2674 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2675 BRW_GET_SWZ(surface
.swizzle
, 0)),
2678 brw_pop_insn_state(p
);
2683 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
);
2684 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2685 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2686 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2692 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2693 brw_inst
*insn
, int while_offset
, int start_offset
)
2695 int scale
= 16 / brw_jump_scale(devinfo
);
2696 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2697 : brw_inst_jip(devinfo
, insn
);
2699 return while_offset
+ jip
* scale
<= start_offset
;
2704 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2707 void *store
= p
->store
;
2708 const struct gen_device_info
*devinfo
= p
->devinfo
;
2712 for (offset
= next_offset(devinfo
, store
, start_offset
);
2713 offset
< p
->next_insn_offset
;
2714 offset
= next_offset(devinfo
, store
, offset
)) {
2715 brw_inst
*insn
= store
+ offset
;
2717 switch (brw_inst_opcode(devinfo
, insn
)) {
2721 case BRW_OPCODE_ENDIF
:
2726 case BRW_OPCODE_WHILE
:
2727 /* If the while doesn't jump before our instruction, it's the end
2728 * of a sibling do...while loop. Ignore it.
2730 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2733 case BRW_OPCODE_ELSE
:
2734 case BRW_OPCODE_HALT
:
2743 /* There is no DO instruction on gen6, so to find the end of the loop
2744 * we have to see if the loop is jumping back before our start
2748 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2750 const struct gen_device_info
*devinfo
= p
->devinfo
;
2752 void *store
= p
->store
;
2754 assert(devinfo
->gen
>= 6);
2756 /* Always start after the instruction (such as a WHILE) we're trying to fix
2759 for (offset
= next_offset(devinfo
, store
, start_offset
);
2760 offset
< p
->next_insn_offset
;
2761 offset
= next_offset(devinfo
, store
, offset
)) {
2762 brw_inst
*insn
= store
+ offset
;
2764 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2765 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2769 assert(!"not reached");
2770 return start_offset
;
2773 /* After program generation, go back and update the UIP and JIP of
2774 * BREAK, CONT, and HALT instructions to their correct locations.
2777 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2779 const struct gen_device_info
*devinfo
= p
->devinfo
;
2781 int br
= brw_jump_scale(devinfo
);
2782 int scale
= 16 / br
;
2783 void *store
= p
->store
;
2785 if (devinfo
->gen
< 6)
2788 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2789 brw_inst
*insn
= store
+ offset
;
2790 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2792 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2793 switch (brw_inst_opcode(devinfo
, insn
)) {
2794 case BRW_OPCODE_BREAK
:
2795 assert(block_end_offset
!= 0);
2796 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2797 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2798 brw_inst_set_uip(devinfo
, insn
,
2799 (brw_find_loop_end(p
, offset
) - offset
+
2800 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2802 case BRW_OPCODE_CONTINUE
:
2803 assert(block_end_offset
!= 0);
2804 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2805 brw_inst_set_uip(devinfo
, insn
,
2806 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2808 assert(brw_inst_uip(devinfo
, insn
) != 0);
2809 assert(brw_inst_jip(devinfo
, insn
) != 0);
2812 case BRW_OPCODE_ENDIF
: {
2813 int32_t jump
= (block_end_offset
== 0) ?
2814 1 * br
: (block_end_offset
- offset
) / scale
;
2815 if (devinfo
->gen
>= 7)
2816 brw_inst_set_jip(devinfo
, insn
, jump
);
2818 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2822 case BRW_OPCODE_HALT
:
2823 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2825 * "In case of the halt instruction not inside any conditional
2826 * code block, the value of <JIP> and <UIP> should be the
2827 * same. In case of the halt instruction inside conditional code
2828 * block, the <UIP> should be the end of the program, and the
2829 * <JIP> should be end of the most inner conditional code block."
2831 * The uip will have already been set by whoever set up the
2834 if (block_end_offset
== 0) {
2835 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2837 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2839 assert(brw_inst_uip(devinfo
, insn
) != 0);
2840 assert(brw_inst_jip(devinfo
, insn
) != 0);
2846 void brw_ff_sync(struct brw_codegen
*p
,
2847 struct brw_reg dest
,
2848 unsigned msg_reg_nr
,
2849 struct brw_reg src0
,
2851 unsigned response_length
,
2854 const struct gen_device_info
*devinfo
= p
->devinfo
;
2857 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2859 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2860 brw_set_dest(p
, insn
, dest
);
2861 brw_set_src0(p
, insn
, src0
);
2862 brw_set_src1(p
, insn
, brw_imm_d(0));
2864 if (devinfo
->gen
< 6)
2865 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2867 brw_set_ff_sync_message(p
,
2875 * Emit the SEND instruction necessary to generate stream output data on Gen6
2876 * (for transform feedback).
2878 * If send_commit_msg is true, this is the last piece of stream output data
2879 * from this thread, so send the data as a committed write. According to the
2880 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2882 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2883 * writes are complete by sending the final write as a committed write."
2886 brw_svb_write(struct brw_codegen
*p
,
2887 struct brw_reg dest
,
2888 unsigned msg_reg_nr
,
2889 struct brw_reg src0
,
2890 unsigned binding_table_index
,
2891 bool send_commit_msg
)
2893 const struct gen_device_info
*devinfo
= p
->devinfo
;
2894 const unsigned target_cache
=
2895 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2896 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2897 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2900 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2902 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2903 brw_set_dest(p
, insn
, dest
);
2904 brw_set_src0(p
, insn
, src0
);
2905 brw_set_src1(p
, insn
, brw_imm_d(0));
2906 brw_set_dp_write_message(p
, insn
,
2907 binding_table_index
,
2908 0, /* msg_control: ignored */
2909 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2912 true, /* header_present */
2913 0, /* last_render_target: ignored */
2914 send_commit_msg
, /* response_length */
2915 0, /* end_of_thread */
2916 send_commit_msg
); /* send_commit_msg */
2920 brw_surface_payload_size(struct brw_codegen
*p
,
2921 unsigned num_channels
,
2926 brw_inst_access_mode(p
->devinfo
, p
->current
) == BRW_ALIGN_16
)
2928 else if (has_simd16
&&
2929 brw_inst_exec_size(p
->devinfo
, p
->current
) == BRW_EXECUTE_16
)
2930 return 2 * num_channels
;
2932 return num_channels
;
2936 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2939 bool response_expected
)
2941 const struct gen_device_info
*devinfo
= p
->devinfo
;
2942 unsigned msg_control
=
2943 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2944 (response_expected
? 1 << 5 : 0); /* Return data expected */
2946 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2947 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2948 if (brw_inst_exec_size(devinfo
, p
->current
) != BRW_EXECUTE_16
)
2949 msg_control
|= 1 << 4; /* SIMD8 mode */
2951 brw_inst_set_dp_msg_type(devinfo
, insn
,
2952 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2954 brw_inst_set_dp_msg_type(devinfo
, insn
,
2955 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2958 brw_inst_set_dp_msg_type(devinfo
, insn
,
2959 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2961 if (brw_inst_exec_size(devinfo
, p
->current
) != BRW_EXECUTE_16
)
2962 msg_control
|= 1 << 4; /* SIMD8 mode */
2965 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2969 brw_untyped_atomic(struct brw_codegen
*p
,
2971 struct brw_reg payload
,
2972 struct brw_reg surface
,
2974 unsigned msg_length
,
2975 bool response_expected
)
2977 const struct gen_device_info
*devinfo
= p
->devinfo
;
2978 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2979 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2980 GEN7_SFID_DATAPORT_DATA_CACHE
);
2981 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2982 /* Mask out unused components -- This is especially important in Align16
2983 * mode on generations that don't have native support for SIMD4x2 atomics,
2984 * because unused but enabled components will cause the dataport to perform
2985 * additional atomic operations on the addresses that happen to be in the
2986 * uninitialized Y, Z and W coordinates of the payload.
2988 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2989 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2990 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2991 brw_surface_payload_size(p
, response_expected
,
2992 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2995 brw_set_dp_untyped_atomic_message(
2996 p
, insn
, atomic_op
, response_expected
);
3000 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
3001 struct brw_inst
*insn
,
3002 unsigned num_channels
)
3004 const struct gen_device_info
*devinfo
= p
->devinfo
;
3005 /* Set mask of 32-bit channels to drop. */
3006 unsigned msg_control
= 0xf & (0xf << num_channels
);
3008 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3009 if (brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
)
3010 msg_control
|= 1 << 4; /* SIMD16 mode */
3012 msg_control
|= 2 << 4; /* SIMD8 mode */
3015 brw_inst_set_dp_msg_type(devinfo
, insn
,
3016 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3017 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
3018 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
3019 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3023 brw_untyped_surface_read(struct brw_codegen
*p
,
3025 struct brw_reg payload
,
3026 struct brw_reg surface
,
3027 unsigned msg_length
,
3028 unsigned num_channels
)
3030 const struct gen_device_info
*devinfo
= p
->devinfo
;
3031 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3032 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3033 GEN7_SFID_DATAPORT_DATA_CACHE
);
3034 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3035 p
, sfid
, dst
, payload
, surface
, msg_length
,
3036 brw_surface_payload_size(p
, num_channels
, true, true),
3039 brw_set_dp_untyped_surface_read_message(
3040 p
, insn
, num_channels
);
3044 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
3045 struct brw_inst
*insn
,
3046 unsigned num_channels
)
3048 const struct gen_device_info
*devinfo
= p
->devinfo
;
3049 /* Set mask of 32-bit channels to drop. */
3050 unsigned msg_control
= 0xf & (0xf << num_channels
);
3052 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3053 if (brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
)
3054 msg_control
|= 1 << 4; /* SIMD16 mode */
3056 msg_control
|= 2 << 4; /* SIMD8 mode */
3058 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
3059 msg_control
|= 0 << 4; /* SIMD4x2 mode */
3061 msg_control
|= 2 << 4; /* SIMD8 mode */
3064 brw_inst_set_dp_msg_type(devinfo
, insn
,
3065 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3066 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
3067 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
3068 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3072 brw_untyped_surface_write(struct brw_codegen
*p
,
3073 struct brw_reg payload
,
3074 struct brw_reg surface
,
3075 unsigned msg_length
,
3076 unsigned num_channels
)
3078 const struct gen_device_info
*devinfo
= p
->devinfo
;
3079 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3080 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3081 GEN7_SFID_DATAPORT_DATA_CACHE
);
3082 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3083 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3084 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3085 WRITEMASK_X
: WRITEMASK_XYZW
;
3086 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3087 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3088 payload
, surface
, msg_length
, 0, align1
);
3090 brw_set_dp_untyped_surface_write_message(
3091 p
, insn
, num_channels
);
3095 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
3096 struct brw_inst
*insn
,
3098 bool response_expected
)
3100 const struct gen_device_info
*devinfo
= p
->devinfo
;
3101 unsigned msg_control
=
3102 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3103 (response_expected
? 1 << 5 : 0); /* Return data expected */
3105 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3106 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3107 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3108 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3110 brw_inst_set_dp_msg_type(devinfo
, insn
,
3111 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
3113 brw_inst_set_dp_msg_type(devinfo
, insn
,
3114 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
3118 brw_inst_set_dp_msg_type(devinfo
, insn
,
3119 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
3121 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3122 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3125 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3129 brw_typed_atomic(struct brw_codegen
*p
,
3131 struct brw_reg payload
,
3132 struct brw_reg surface
,
3134 unsigned msg_length
,
3135 bool response_expected
) {
3136 const struct gen_device_info
*devinfo
= p
->devinfo
;
3137 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3138 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3139 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3140 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3141 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3142 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3143 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3144 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
3145 brw_surface_payload_size(p
, response_expected
,
3146 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3149 brw_set_dp_typed_atomic_message(
3150 p
, insn
, atomic_op
, response_expected
);
3154 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3155 struct brw_inst
*insn
,
3156 unsigned num_channels
)
3158 const struct gen_device_info
*devinfo
= p
->devinfo
;
3159 /* Set mask of unused channels. */
3160 unsigned msg_control
= 0xf & (0xf << num_channels
);
3162 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3163 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3164 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3165 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3167 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3170 brw_inst_set_dp_msg_type(devinfo
, insn
,
3171 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3173 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3174 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3175 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3178 brw_inst_set_dp_msg_type(devinfo
, insn
,
3179 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3182 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3186 brw_typed_surface_read(struct brw_codegen
*p
,
3188 struct brw_reg payload
,
3189 struct brw_reg surface
,
3190 unsigned msg_length
,
3191 unsigned num_channels
)
3193 const struct gen_device_info
*devinfo
= p
->devinfo
;
3194 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3195 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3196 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3197 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3198 p
, sfid
, dst
, payload
, surface
, msg_length
,
3199 brw_surface_payload_size(p
, num_channels
,
3200 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3203 brw_set_dp_typed_surface_read_message(
3204 p
, insn
, num_channels
);
3208 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3209 struct brw_inst
*insn
,
3210 unsigned num_channels
)
3212 const struct gen_device_info
*devinfo
= p
->devinfo
;
3213 /* Set mask of unused channels. */
3214 unsigned msg_control
= 0xf & (0xf << num_channels
);
3216 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3217 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3218 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3219 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3221 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3224 brw_inst_set_dp_msg_type(devinfo
, insn
,
3225 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3228 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3229 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3230 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3233 brw_inst_set_dp_msg_type(devinfo
, insn
,
3234 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3237 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3241 brw_typed_surface_write(struct brw_codegen
*p
,
3242 struct brw_reg payload
,
3243 struct brw_reg surface
,
3244 unsigned msg_length
,
3245 unsigned num_channels
)
3247 const struct gen_device_info
*devinfo
= p
->devinfo
;
3248 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3249 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3250 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3251 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3252 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3253 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3254 WRITEMASK_X
: WRITEMASK_XYZW
);
3255 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3256 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3257 payload
, surface
, msg_length
, 0, true);
3259 brw_set_dp_typed_surface_write_message(
3260 p
, insn
, num_channels
);
3264 brw_set_memory_fence_message(struct brw_codegen
*p
,
3265 struct brw_inst
*insn
,
3266 enum brw_message_target sfid
,
3269 const struct gen_device_info
*devinfo
= p
->devinfo
;
3271 brw_set_message_descriptor(p
, insn
, sfid
,
3272 1 /* message length */,
3273 (commit_enable
? 1 : 0) /* response length */,
3274 true /* header present */,
3278 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3279 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3281 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3282 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3285 unreachable("Not reached");
3289 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3293 brw_memory_fence(struct brw_codegen
*p
,
3296 const struct gen_device_info
*devinfo
= p
->devinfo
;
3297 const bool commit_enable
= devinfo
->gen
== 7 && !devinfo
->is_haswell
;
3298 struct brw_inst
*insn
;
3300 brw_push_insn_state(p
);
3301 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3302 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3305 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3306 * message doesn't write anything back.
3308 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3309 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3310 brw_set_dest(p
, insn
, dst
);
3311 brw_set_src0(p
, insn
, dst
);
3312 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3315 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3316 /* IVB does typed surface access through the render cache, so we need to
3317 * flush it too. Use a different register so both flushes can be
3318 * pipelined by the hardware.
3320 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3321 brw_set_dest(p
, insn
, offset(dst
, 1));
3322 brw_set_src0(p
, insn
, offset(dst
, 1));
3323 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3326 /* Now write the response of the second message into the response of the
3327 * first to trigger a pipeline stall -- This way future render and data
3328 * cache messages will be properly ordered with respect to past data and
3329 * render cache messages.
3331 brw_MOV(p
, dst
, offset(dst
, 1));
3334 brw_pop_insn_state(p
);
3338 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3339 struct brw_reg dest
,
3343 struct brw_reg data
,
3344 unsigned msg_length
,
3345 unsigned response_length
)
3347 const struct gen_device_info
*devinfo
= p
->devinfo
;
3348 struct brw_inst
*insn
;
3349 const uint16_t exec_size
= brw_inst_exec_size(devinfo
, p
->current
);
3351 /* brw_send_indirect_message will automatically use a direct send message
3352 * if data is actually immediate.
3354 insn
= brw_send_indirect_message(p
,
3355 GEN7_SFID_PIXEL_INTERPOLATOR
,
3359 brw_inst_set_mlen(devinfo
, insn
, msg_length
);
3360 brw_inst_set_rlen(devinfo
, insn
, response_length
);
3362 brw_inst_set_pi_simd_mode(devinfo
, insn
, exec_size
== BRW_EXECUTE_16
);
3363 brw_inst_set_pi_slot_group(devinfo
, insn
, 0); /* zero unless 32/64px dispatch */
3364 brw_inst_set_pi_nopersp(devinfo
, insn
, noperspective
);
3365 brw_inst_set_pi_message_type(devinfo
, insn
, mode
);
3369 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3370 struct brw_reg mask
)
3372 const struct gen_device_info
*devinfo
= p
->devinfo
;
3373 const unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, p
->current
);
3374 const unsigned qtr_control
= brw_inst_qtr_control(devinfo
, p
->current
);
3377 assert(devinfo
->gen
>= 7);
3378 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3380 brw_push_insn_state(p
);
3382 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3383 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3385 if (devinfo
->gen
>= 8) {
3386 /* Getting the first active channel index is easy on Gen8: Just find
3387 * the first bit set in the execution mask. The register exists on
3388 * HSW already but it reads back as all ones when the current
3389 * instruction has execution masking disabled, so it's kind of
3392 struct brw_reg exec_mask
=
3393 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3395 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3396 /* Unfortunately, ce0 does not take into account the thread
3397 * dispatch mask, which may be a problem in cases where it's not
3398 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3399 * some n). Combine ce0 with the given dispatch (or vector) mask
3400 * to mask off those channels which were never dispatched by the
3403 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3404 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3405 exec_mask
= vec1(dst
);
3408 /* Quarter control has the effect of magically shifting the value of
3409 * ce0 so you'll get the first active channel relative to the
3410 * specified quarter control as result.
3412 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3414 const struct brw_reg flag
= brw_flag_reg(1, 0);
3416 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3418 /* Run enough instructions returning zero with execution masking and
3419 * a conditional modifier enabled in order to get the full execution
3420 * mask in f1.0. We could use a single 32-wide move here if it
3421 * weren't because of the hardware bug that causes channel enables to
3422 * be applied incorrectly to the second half of 32-wide instructions
3425 const unsigned lower_size
= MIN2(16, exec_size
);
3426 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3427 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3429 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3430 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3431 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3432 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3433 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3436 /* Find the first bit set in the exec_size-wide portion of the flag
3437 * register that was updated by the last sequence of MOV
3440 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3441 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3444 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3446 if (devinfo
->gen
>= 8 &&
3447 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3448 /* In SIMD4x2 mode the first active channel index is just the
3449 * negation of the first bit of the mask register. Note that ce0
3450 * doesn't take into account the dispatch mask, so the Gen7 path
3451 * should be used instead unless you have the guarantee that the
3452 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3455 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3456 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3460 /* Overwrite the destination without and with execution masking to
3461 * find out which of the channels is active.
3463 brw_push_insn_state(p
);
3464 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3465 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3468 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3470 brw_pop_insn_state(p
);
3471 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3475 brw_pop_insn_state(p
);
3479 brw_broadcast(struct brw_codegen
*p
,
3484 const struct gen_device_info
*devinfo
= p
->devinfo
;
3485 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3488 brw_push_insn_state(p
);
3489 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3490 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3492 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3493 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3495 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3496 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3497 /* Trivial, the source is already uniform or the index is a constant.
3498 * We will typically not get here if the optimizer is doing its job, but
3499 * asserting would be mean.
3501 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3503 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3504 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3507 const struct brw_reg addr
=
3508 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3509 const unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3510 /* Limit in bytes of the signed indirect addressing immediate. */
3511 const unsigned limit
= 512;
3513 brw_push_insn_state(p
);
3514 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3515 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3517 /* Take into account the component size and horizontal stride. */
3518 assert(src
.vstride
== src
.hstride
+ src
.width
);
3519 brw_SHL(p
, addr
, vec1(idx
),
3520 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3523 /* We can only address up to limit bytes using the indirect
3524 * addressing immediate, account for the difference if the source
3525 * register is above this limit.
3527 if (offset
>= limit
)
3528 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3530 brw_pop_insn_state(p
);
3532 /* Use indirect addressing to fetch the specified component. */
3534 retype(brw_vec1_indirect(addr
.subnr
, offset
% limit
),
3537 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3538 * to all bits of a flag register,
3542 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3543 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3544 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3545 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3547 /* and use predicated SEL to pick the right channel. */
3548 inst
= brw_SEL(p
, dst
,
3549 stride(suboffset(src
, 4), 4, 4, 1),
3550 stride(src
, 4, 4, 1));
3551 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3552 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3556 brw_pop_insn_state(p
);
3560 * This instruction is generated as a single-channel align1 instruction by
3561 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3563 * We can't use the typed atomic op in the FS because that has the execution
3564 * mask ANDed with the pixel mask, but we just want to write the one dword for
3567 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3568 * one u32. So we use the same untyped atomic write message as the pixel
3571 * The untyped atomic operation requires a BUFFER surface type with RAW
3572 * format, and is only accessible through the legacy DATA_CACHE dataport
3575 void brw_shader_time_add(struct brw_codegen
*p
,
3576 struct brw_reg payload
,
3577 uint32_t surf_index
)
3579 const unsigned sfid
= (p
->devinfo
->gen
>= 8 || p
->devinfo
->is_haswell
?
3580 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3581 GEN7_SFID_DATAPORT_DATA_CACHE
);
3582 assert(p
->devinfo
->gen
>= 7);
3584 brw_push_insn_state(p
);
3585 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3586 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3587 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3588 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3590 /* We use brw_vec1_reg and unmasked because we want to increment the given
3593 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3595 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3597 brw_set_src1(p
, send
, brw_imm_ud(0));
3598 brw_set_message_descriptor(p
, send
, sfid
, 2, 0, false, false);
3599 brw_inst_set_binding_table_index(p
->devinfo
, send
, surf_index
);
3600 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3602 brw_pop_insn_state(p
);
3607 * Emit the SEND message for a barrier
3610 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3612 const struct gen_device_info
*devinfo
= p
->devinfo
;
3613 struct brw_inst
*inst
;
3615 assert(devinfo
->gen
>= 7);
3617 brw_push_insn_state(p
);
3618 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3619 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3620 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3621 brw_set_src0(p
, inst
, src
);
3622 brw_set_src1(p
, inst
, brw_null_reg());
3624 brw_set_message_descriptor(p
, inst
, BRW_SFID_MESSAGE_GATEWAY
,
3626 0 /* response_length */,
3627 false /* header_present */,
3628 false /* end_of_thread */);
3630 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3631 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3632 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3634 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3635 brw_pop_insn_state(p
);
3640 * Emit the wait instruction for a barrier
3643 brw_WAIT(struct brw_codegen
*p
)
3645 const struct gen_device_info
*devinfo
= p
->devinfo
;
3646 struct brw_inst
*insn
;
3648 struct brw_reg src
= brw_notification_reg();
3650 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3651 brw_set_dest(p
, insn
, src
);
3652 brw_set_src0(p
, insn
, src
);
3653 brw_set_src1(p
, insn
, brw_null_reg());
3655 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3656 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);