2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 assert(devinfo
->gen
< 12);
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct gen_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
91 const struct gen_device_info
*devinfo
= p
->devinfo
;
93 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
94 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
95 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
96 assert(dest
.nr
< 128);
98 /* The hardware has a restriction where a destination of size Byte with
99 * a stride of 1 is only allowed for a packed byte MOV. For any other
100 * instruction, the stride must be at least 2, even when the destination
101 * is the NULL register.
103 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
104 dest
.nr
== BRW_ARF_NULL
&&
105 type_sz(dest
.type
) == 1 &&
106 dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
) {
107 dest
.hstride
= BRW_HORIZONTAL_STRIDE_2
;
110 gen7_convert_mrf_to_grf(p
, &dest
);
112 if (devinfo
->gen
>= 12 &&
113 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
114 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
115 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
116 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
117 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
118 assert(dest
.subnr
== 0);
119 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
120 (dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
121 dest
.vstride
== dest
.width
+ 1));
122 assert(!dest
.negate
&& !dest
.abs
);
123 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
124 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
126 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
127 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
128 assert(devinfo
->gen
< 12);
129 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
130 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
131 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
132 assert(dest
.subnr
% 16 == 0);
133 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
134 dest
.vstride
== dest
.width
+ 1);
135 assert(!dest
.negate
&& !dest
.abs
);
136 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
137 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
138 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
140 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
141 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
143 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
144 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
146 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
147 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
148 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
149 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
150 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
152 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
153 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
154 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
155 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
156 assert(dest
.writemask
!= 0);
158 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
159 * Although Dst.HorzStride is a don't care for Align16, HW needs
160 * this to be programmed as "01".
162 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
165 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
167 /* These are different sizes in align1 vs align16:
169 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
170 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
171 dest
.indirect_offset
);
172 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
173 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
174 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
176 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
177 dest
.indirect_offset
);
178 /* even ignored in da16, still need to set as '01' */
179 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
184 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
185 * or 16 (SIMD16), as that's normally correct. However, when dealing with
186 * small registers, it can be useful for us to automatically reduce it to
187 * match the register size.
189 if (p
->automatic_exec_sizes
) {
191 * In platforms that support fp64 we can emit instructions with a width
192 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
193 * these cases we need to make sure that these instructions have their
194 * exec sizes set properly when they are emitted and we can't rely on
195 * this code to fix it.
198 if (devinfo
->gen
>= 6)
199 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
201 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
204 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
209 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
211 const struct gen_device_info
*devinfo
= p
->devinfo
;
213 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
214 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
215 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
216 assert(reg
.nr
< 128);
218 gen7_convert_mrf_to_grf(p
, ®
);
220 if (devinfo
->gen
>= 6 &&
221 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
222 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
223 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
224 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
225 /* Any source modifiers or regions will be ignored, since this just
226 * identifies the MRF/GRF to start reading the message contents from.
227 * Check for some likely failures.
231 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
234 if (devinfo
->gen
>= 12 &&
235 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
236 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
237 assert(reg
.file
!= BRW_IMMEDIATE_VALUE
);
238 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
239 assert(reg
.subnr
== 0);
240 assert(has_scalar_region(reg
) ||
241 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
242 reg
.vstride
== reg
.width
+ 1));
243 assert(!reg
.negate
&& !reg
.abs
);
244 brw_inst_set_send_src0_reg_file(devinfo
, inst
, reg
.file
);
245 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
247 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
248 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
249 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
250 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
251 assert(reg
.subnr
% 16 == 0);
252 assert(has_scalar_region(reg
) ||
253 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
254 reg
.vstride
== reg
.width
+ 1));
255 assert(!reg
.negate
&& !reg
.abs
);
256 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
257 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
259 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
260 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
261 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
262 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
264 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
265 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
266 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
267 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
268 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
269 reg
.type
== BRW_REGISTER_TYPE_Q
)
270 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
272 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
274 if (devinfo
->gen
< 12 && type_sz(reg
.type
) < 8) {
275 brw_inst_set_src1_reg_file(devinfo
, inst
,
276 BRW_ARCHITECTURE_REGISTER_FILE
);
277 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
278 brw_inst_src0_reg_hw_type(devinfo
, inst
));
281 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
282 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
283 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
284 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
286 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
289 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
291 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
292 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
294 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
298 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
299 if (reg
.width
== BRW_WIDTH_1
&&
300 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
301 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
302 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
303 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
305 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
306 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
307 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
310 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
311 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
312 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
313 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
314 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
315 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
316 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
317 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
319 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
320 /* This is an oddity of the fact we're using the same
321 * descriptions for registers in align_16 as align_1:
323 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
324 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
325 reg
.type
== BRW_REGISTER_TYPE_DF
&&
326 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
329 * "For Align16 access mode, only encodings of 0000 and 0011
330 * are allowed. Other codes are reserved."
332 * Presumably the DevSNB behavior applies to IVB as well.
334 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
336 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
345 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
347 const struct gen_device_info
*devinfo
= p
->devinfo
;
349 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
350 assert(reg
.nr
< 128);
352 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
353 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
||
354 (devinfo
->gen
>= 12 &&
355 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
356 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
))) {
357 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
358 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
359 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
360 assert(reg
.subnr
== 0);
361 assert(has_scalar_region(reg
) ||
362 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
363 reg
.vstride
== reg
.width
+ 1));
364 assert(!reg
.negate
&& !reg
.abs
);
365 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
366 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
368 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
370 * "Accumulator registers may be accessed explicitly as src0
373 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
374 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
376 gen7_convert_mrf_to_grf(p
, ®
);
377 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
379 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
380 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
381 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
383 /* Only src1 can be immediate in two-argument instructions.
385 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
387 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
388 /* two-argument instructions can only use 32-bit immediates */
389 assert(type_sz(reg
.type
) < 8);
390 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
392 /* This is a hardware restriction, which may or may not be lifted
395 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
396 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
398 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
399 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
400 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
402 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
405 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
406 if (reg
.width
== BRW_WIDTH_1
&&
407 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
408 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
409 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
410 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
412 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
413 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
414 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
417 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
418 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
419 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
420 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
421 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
422 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
423 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
424 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
426 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
427 /* This is an oddity of the fact we're using the same
428 * descriptions for registers in align_16 as align_1:
430 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
431 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
432 reg
.type
== BRW_REGISTER_TYPE_DF
&&
433 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
436 * "For Align16 access mode, only encodings of 0000 and 0011
437 * are allowed. Other codes are reserved."
439 * Presumably the DevSNB behavior applies to IVB as well.
441 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
443 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
451 * Specify the descriptor and extended descriptor immediate for a SEND(C)
452 * message instruction.
455 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
456 unsigned desc
, unsigned ex_desc
)
458 const struct gen_device_info
*devinfo
= p
->devinfo
;
459 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
460 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
461 if (devinfo
->gen
< 12)
462 brw_inst_set_src1_file_type(devinfo
, inst
,
463 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
464 brw_inst_set_send_desc(devinfo
, inst
, desc
);
465 if (devinfo
->gen
>= 9)
466 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
469 static void brw_set_math_message( struct brw_codegen
*p
,
472 unsigned integer_type
,
476 const struct gen_device_info
*devinfo
= p
->devinfo
;
478 unsigned response_length
;
480 /* Infer message length from the function */
482 case BRW_MATH_FUNCTION_POW
:
483 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
484 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
485 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
493 /* Infer response length from the function */
495 case BRW_MATH_FUNCTION_SINCOS
:
496 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
504 brw_set_desc(p
, inst
, brw_message_desc(
505 devinfo
, msg_length
, response_length
, false));
507 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
508 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
509 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
510 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
511 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
512 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
513 brw_inst_set_saturate(devinfo
, inst
, 0);
517 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
520 unsigned response_length
,
523 const struct gen_device_info
*devinfo
= p
->devinfo
;
525 brw_set_desc(p
, insn
, brw_message_desc(
526 devinfo
, 1, response_length
, true));
528 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
529 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
530 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
531 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
532 /* The following fields are not used by FF_SYNC: */
533 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
534 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
535 brw_inst_set_urb_used(devinfo
, insn
, 0);
536 brw_inst_set_urb_complete(devinfo
, insn
, 0);
539 static void brw_set_urb_message( struct brw_codegen
*p
,
541 enum brw_urb_write_flags flags
,
543 unsigned response_length
,
545 unsigned swizzle_control
)
547 const struct gen_device_info
*devinfo
= p
->devinfo
;
549 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
550 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
551 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
553 brw_set_desc(p
, insn
, brw_message_desc(
554 devinfo
, msg_length
, response_length
, true));
556 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
557 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
559 if (flags
& BRW_URB_WRITE_OWORD
) {
560 assert(msg_length
== 2); /* header + one OWORD of data */
561 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
563 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
566 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
567 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
569 if (devinfo
->gen
< 8) {
570 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
573 if (devinfo
->gen
< 7) {
574 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
575 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
577 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
578 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
583 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
587 bool invalidate_after_read
,
589 unsigned addr_offset
,
594 const struct gen_device_info
*devinfo
= p
->devinfo
;
595 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
596 (devinfo
->gen
>= 8 && num_regs
== 8));
597 const unsigned block_size
= (devinfo
->gen
>= 8 ? util_logbase2(num_regs
) :
600 brw_set_desc(p
, inst
, brw_message_desc(
601 devinfo
, mlen
, rlen
, header_present
));
603 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
604 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
605 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
606 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
607 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
608 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
609 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
613 brw_inst_set_state(const struct gen_device_info
*devinfo
,
615 const struct brw_insn_state
*state
)
617 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
618 brw_inst_set_group(devinfo
, insn
, state
->group
);
619 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
620 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
621 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
622 if (devinfo
->gen
>= 12)
623 brw_inst_set_swsb(devinfo
, insn
, tgl_swsb_encode(state
->swsb
));
624 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
625 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
626 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
628 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
629 state
->access_mode
== BRW_ALIGN_16
) {
630 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
631 if (devinfo
->gen
>= 7)
632 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
634 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
635 if (devinfo
->gen
>= 7)
636 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
639 if (devinfo
->gen
>= 6)
640 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
643 #define next_insn brw_next_insn
645 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
647 const struct gen_device_info
*devinfo
= p
->devinfo
;
650 if (p
->nr_insn
+ 1 > p
->store_size
) {
652 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
655 p
->next_insn_offset
+= 16;
656 insn
= &p
->store
[p
->nr_insn
++];
658 memset(insn
, 0, sizeof(*insn
));
659 brw_inst_set_opcode(devinfo
, insn
, opcode
);
661 /* Apply the default instruction state */
662 brw_inst_set_state(devinfo
, insn
, p
->current
);
668 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
669 struct brw_reg dest
, struct brw_reg src
)
671 brw_inst
*insn
= next_insn(p
, opcode
);
672 brw_set_dest(p
, insn
, dest
);
673 brw_set_src0(p
, insn
, src
);
678 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
679 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
681 /* 64-bit immediates are only supported on 1-src instructions */
682 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
683 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
685 brw_inst
*insn
= next_insn(p
, opcode
);
686 brw_set_dest(p
, insn
, dest
);
687 brw_set_src0(p
, insn
, src0
);
688 brw_set_src1(p
, insn
, src1
);
693 get_3src_subreg_nr(struct brw_reg reg
)
695 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
696 * use 32-bit units (components 0..7). Since they only support F/D/UD
697 * types, this doesn't lose any flexibility, but uses fewer bits.
699 return reg
.subnr
/ 4;
702 static enum gen10_align1_3src_vertical_stride
703 to_3src_align1_vstride(const struct gen_device_info
*devinfo
,
704 enum brw_vertical_stride vstride
)
707 case BRW_VERTICAL_STRIDE_0
:
708 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
709 case BRW_VERTICAL_STRIDE_1
:
710 assert(devinfo
->gen
>= 12);
711 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1
;
712 case BRW_VERTICAL_STRIDE_2
:
713 assert(devinfo
->gen
< 12);
714 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
715 case BRW_VERTICAL_STRIDE_4
:
716 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
717 case BRW_VERTICAL_STRIDE_8
:
718 case BRW_VERTICAL_STRIDE_16
:
719 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
721 unreachable("invalid vstride");
726 static enum gen10_align1_3src_src_horizontal_stride
727 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
730 case BRW_HORIZONTAL_STRIDE_0
:
731 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
732 case BRW_HORIZONTAL_STRIDE_1
:
733 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
734 case BRW_HORIZONTAL_STRIDE_2
:
735 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
736 case BRW_HORIZONTAL_STRIDE_4
:
737 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
739 unreachable("invalid hstride");
744 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
745 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
747 const struct gen_device_info
*devinfo
= p
->devinfo
;
748 brw_inst
*inst
= next_insn(p
, opcode
);
750 gen7_convert_mrf_to_grf(p
, &dest
);
752 assert(dest
.nr
< 128);
754 if (devinfo
->gen
>= 10)
755 assert(!(src0
.file
== BRW_IMMEDIATE_VALUE
&&
756 src2
.file
== BRW_IMMEDIATE_VALUE
));
758 assert(src0
.file
== BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
759 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
&& src1
.nr
< 128);
760 assert(src2
.file
== BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
761 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
762 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
763 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
764 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
766 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
767 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
768 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
770 if (devinfo
->gen
>= 12) {
771 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
, dest
.file
);
772 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
774 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
775 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
776 BRW_ALIGN1_3SRC_ACCUMULATOR
);
777 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
779 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
780 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
781 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
784 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
786 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
788 if (brw_reg_type_is_floating_point(dest
.type
)) {
789 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
790 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
792 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
793 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
796 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
797 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
798 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
799 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
801 if (src0
.file
== BRW_IMMEDIATE_VALUE
) {
802 brw_inst_set_3src_a1_src0_imm(devinfo
, inst
, src0
.ud
);
804 brw_inst_set_3src_a1_src0_vstride(
805 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src0
.vstride
));
806 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
807 to_3src_align1_hstride(src0
.hstride
));
808 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
809 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
810 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
812 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
814 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
815 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
817 brw_inst_set_3src_a1_src1_vstride(
818 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src1
.vstride
));
819 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
820 to_3src_align1_hstride(src1
.hstride
));
822 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
823 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
824 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
826 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
828 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
829 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
831 if (src2
.file
== BRW_IMMEDIATE_VALUE
) {
832 brw_inst_set_3src_a1_src2_imm(devinfo
, inst
, src2
.ud
);
834 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
835 to_3src_align1_hstride(src2
.hstride
));
836 /* no vstride on src2 */
837 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
838 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
839 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
840 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
843 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
844 src0
.file
== BRW_IMMEDIATE_VALUE
||
845 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
846 src0
.type
== BRW_REGISTER_TYPE_NF
));
847 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
848 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
849 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
850 src2
.file
== BRW_IMMEDIATE_VALUE
);
852 if (devinfo
->gen
>= 12) {
853 if (src0
.file
== BRW_IMMEDIATE_VALUE
) {
854 brw_inst_set_3src_a1_src0_is_imm(devinfo
, inst
, 1);
856 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
, src0
.file
);
859 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
, src1
.file
);
861 if (src2
.file
== BRW_IMMEDIATE_VALUE
) {
862 brw_inst_set_3src_a1_src2_is_imm(devinfo
, inst
, 1);
864 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
, src2
.file
);
867 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
868 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
869 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
870 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
871 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
872 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
873 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
874 BRW_ALIGN1_3SRC_ACCUMULATOR
);
875 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
876 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
877 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
878 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
882 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
883 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
884 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
885 dest
.type
== BRW_REGISTER_TYPE_DF
||
886 dest
.type
== BRW_REGISTER_TYPE_D
||
887 dest
.type
== BRW_REGISTER_TYPE_UD
||
888 (dest
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 8));
889 if (devinfo
->gen
== 6) {
890 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
891 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
893 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
894 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 4);
895 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
897 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
898 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
899 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
900 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
901 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
902 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
903 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
904 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
906 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
907 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
908 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
909 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
910 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
911 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
912 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
913 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
915 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
916 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
917 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
918 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
919 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
920 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
921 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
922 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
924 if (devinfo
->gen
>= 7) {
925 /* Set both the source and destination types based on dest.type,
926 * ignoring the source register types. The MAD and LRP emitters ensure
927 * that all four types are float. The BFE and BFI2 emitters, however,
928 * may send us mixed D and UD types and want us to ignore that and use
929 * the destination type.
931 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
932 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
934 /* From the Bspec, 3D Media GPGPU, Instruction fields, srcType:
936 * "Three source instructions can use operands with mixed-mode
937 * precision. When SrcType field is set to :f or :hf it defines
938 * precision for source 0 only, and fields Src1Type and Src2Type
939 * define precision for other source operands:
941 * 0b = :f. Single precision Float (32-bit).
942 * 1b = :hf. Half precision Float (16-bit)."
944 if (src1
.type
== BRW_REGISTER_TYPE_HF
)
945 brw_inst_set_3src_a16_src1_type(devinfo
, inst
, 1);
947 if (src2
.type
== BRW_REGISTER_TYPE_HF
)
948 brw_inst_set_3src_a16_src2_type(devinfo
, inst
, 1);
956 /***********************************************************************
957 * Convenience routines.
960 brw_inst *brw_##OP(struct brw_codegen *p, \
961 struct brw_reg dest, \
962 struct brw_reg src0) \
964 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
968 brw_inst *brw_##OP(struct brw_codegen *p, \
969 struct brw_reg dest, \
970 struct brw_reg src0, \
971 struct brw_reg src1) \
973 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
977 brw_inst *brw_##OP(struct brw_codegen *p, \
978 struct brw_reg dest, \
979 struct brw_reg src0, \
980 struct brw_reg src1, \
981 struct brw_reg src2) \
983 if (p->current->access_mode == BRW_ALIGN_16) { \
984 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
985 src0.swizzle = BRW_SWIZZLE_XXXX; \
986 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
987 src1.swizzle = BRW_SWIZZLE_XXXX; \
988 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
989 src2.swizzle = BRW_SWIZZLE_XXXX; \
991 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
995 brw_inst *brw_##OP(struct brw_codegen *p, \
996 struct brw_reg dest, \
997 struct brw_reg src0, \
998 struct brw_reg src1, \
999 struct brw_reg src2) \
1001 assert(dest.type == BRW_REGISTER_TYPE_F || \
1002 dest.type == BRW_REGISTER_TYPE_DF); \
1003 if (dest.type == BRW_REGISTER_TYPE_F) { \
1004 assert(src0.type == BRW_REGISTER_TYPE_F); \
1005 assert(src1.type == BRW_REGISTER_TYPE_F); \
1006 assert(src2.type == BRW_REGISTER_TYPE_F); \
1007 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
1008 assert(src0.type == BRW_REGISTER_TYPE_DF); \
1009 assert(src1.type == BRW_REGISTER_TYPE_DF); \
1010 assert(src2.type == BRW_REGISTER_TYPE_DF); \
1013 if (p->current->access_mode == BRW_ALIGN_16) { \
1014 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
1015 src0.swizzle = BRW_SWIZZLE_XXXX; \
1016 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
1017 src1.swizzle = BRW_SWIZZLE_XXXX; \
1018 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
1019 src2.swizzle = BRW_SWIZZLE_XXXX; \
1021 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1061 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
1063 const struct gen_device_info
*devinfo
= p
->devinfo
;
1065 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
1066 * To avoid the problems that causes, we use an <X,2,0> source region to
1067 * read each element twice.
1069 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1070 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
1071 dest
.type
== BRW_REGISTER_TYPE_DF
&&
1072 (src0
.type
== BRW_REGISTER_TYPE_F
||
1073 src0
.type
== BRW_REGISTER_TYPE_D
||
1074 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
1075 !has_scalar_region(src0
)) {
1076 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
1077 src0
.vstride
= src0
.hstride
;
1078 src0
.width
= BRW_WIDTH_2
;
1079 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1082 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1086 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1087 struct brw_reg src0
, struct brw_reg src1
)
1090 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1091 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1092 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1093 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1094 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1097 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1098 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1099 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1100 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1101 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1104 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1108 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1109 struct brw_reg src0
, struct brw_reg src1
)
1111 assert(dest
.type
== src0
.type
);
1112 assert(src0
.type
== src1
.type
);
1113 switch (src0
.type
) {
1114 case BRW_REGISTER_TYPE_B
:
1115 case BRW_REGISTER_TYPE_UB
:
1116 case BRW_REGISTER_TYPE_W
:
1117 case BRW_REGISTER_TYPE_UW
:
1118 case BRW_REGISTER_TYPE_D
:
1119 case BRW_REGISTER_TYPE_UD
:
1122 unreachable("Bad type for brw_AVG");
1125 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1129 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1130 struct brw_reg src0
, struct brw_reg src1
)
1133 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1134 src0
.type
== BRW_REGISTER_TYPE_UD
||
1135 src1
.type
== BRW_REGISTER_TYPE_D
||
1136 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1137 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1140 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1141 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1142 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1143 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1144 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1147 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1148 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1149 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1150 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1151 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1154 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1155 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1156 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1157 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1159 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1163 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1164 struct brw_reg src0
, struct brw_reg src1
)
1166 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1167 src0
.width
= BRW_WIDTH_1
;
1168 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1169 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1173 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1174 struct brw_reg src0
, struct brw_reg src1
)
1176 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1177 src0
.width
= BRW_WIDTH_1
;
1178 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1179 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1180 src1
.width
= BRW_WIDTH_8
;
1181 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1182 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1186 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1188 const struct gen_device_info
*devinfo
= p
->devinfo
;
1189 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1190 /* The F32TO16 instruction doesn't support 32-bit destination types in
1191 * Align1 mode, and neither does the Gen8 implementation in terms of a
1192 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1193 * an undocumented feature.
1195 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1196 (!align16
|| devinfo
->gen
>= 8));
1200 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1202 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1203 dst
.type
== BRW_REGISTER_TYPE_W
||
1204 dst
.type
== BRW_REGISTER_TYPE_UW
||
1205 dst
.type
== BRW_REGISTER_TYPE_HF
);
1208 brw_push_insn_state(p
);
1210 if (needs_zero_fill
) {
1211 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1212 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1215 if (devinfo
->gen
>= 8) {
1216 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1218 assert(devinfo
->gen
== 7);
1219 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1222 if (needs_zero_fill
) {
1223 if (devinfo
->gen
< 12)
1224 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1225 brw_set_default_swsb(p
, tgl_swsb_null());
1226 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1227 if (devinfo
->gen
< 12)
1228 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1231 brw_pop_insn_state(p
);
1236 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1238 const struct gen_device_info
*devinfo
= p
->devinfo
;
1239 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1242 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1244 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1246 * Because this instruction does not have a 16-bit floating-point
1247 * type, the source data type must be Word (W). The destination type
1248 * must be F (Float).
1250 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1251 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1253 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1254 src
.type
== BRW_REGISTER_TYPE_UW
||
1255 src
.type
== BRW_REGISTER_TYPE_HF
);
1258 if (devinfo
->gen
>= 8) {
1259 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1261 assert(devinfo
->gen
== 7);
1262 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1267 void brw_NOP(struct brw_codegen
*p
)
1269 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1270 memset(insn
, 0, sizeof(*insn
));
1271 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1274 void brw_SYNC(struct brw_codegen
*p
, enum tgl_sync_function func
)
1276 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SYNC
);
1277 brw_inst_set_cond_modifier(p
->devinfo
, insn
, func
);
1280 /***********************************************************************
1281 * Comparisons, if/else/endif
1285 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1286 unsigned predicate_control
)
1288 const struct gen_device_info
*devinfo
= p
->devinfo
;
1289 struct brw_reg ip
= brw_ip_reg();
1290 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1292 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1293 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1294 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1295 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1301 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1303 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1305 p
->if_stack_depth
++;
1306 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1307 p
->if_stack_array_size
*= 2;
1308 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1309 p
->if_stack_array_size
);
1314 pop_if_stack(struct brw_codegen
*p
)
1316 p
->if_stack_depth
--;
1317 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1321 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1323 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1324 p
->loop_stack_array_size
*= 2;
1325 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1326 p
->loop_stack_array_size
);
1327 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1328 p
->loop_stack_array_size
);
1331 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1332 p
->loop_stack_depth
++;
1333 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1337 get_inner_do_insn(struct brw_codegen
*p
)
1339 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1342 /* EU takes the value from the flag register and pushes it onto some
1343 * sort of a stack (presumably merging with any flag value already on
1344 * the stack). Within an if block, the flags at the top of the stack
1345 * control execution on each channel of the unit, eg. on each of the
1346 * 16 pixel values in our wm programs.
1348 * When the matching 'else' instruction is reached (presumably by
1349 * countdown of the instruction count patched in by our ELSE/ENDIF
1350 * functions), the relevant flags are inverted.
1352 * When the matching 'endif' instruction is reached, the flags are
1353 * popped off. If the stack is now empty, normal execution resumes.
1356 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1358 const struct gen_device_info
*devinfo
= p
->devinfo
;
1361 insn
= next_insn(p
, BRW_OPCODE_IF
);
1363 /* Override the defaults for this instruction:
1365 if (devinfo
->gen
< 6) {
1366 brw_set_dest(p
, insn
, brw_ip_reg());
1367 brw_set_src0(p
, insn
, brw_ip_reg());
1368 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1369 } else if (devinfo
->gen
== 6) {
1370 brw_set_dest(p
, insn
, brw_imm_w(0));
1371 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1372 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1373 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1374 } else if (devinfo
->gen
== 7) {
1375 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1376 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1377 brw_set_src1(p
, insn
, brw_imm_w(0));
1378 brw_inst_set_jip(devinfo
, insn
, 0);
1379 brw_inst_set_uip(devinfo
, insn
, 0);
1381 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1382 if (devinfo
->gen
< 12)
1383 brw_set_src0(p
, insn
, brw_imm_d(0));
1384 brw_inst_set_jip(devinfo
, insn
, 0);
1385 brw_inst_set_uip(devinfo
, insn
, 0);
1388 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1389 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1390 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1391 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1392 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1393 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1395 push_if_stack(p
, insn
);
1396 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1400 /* This function is only used for gen6-style IF instructions with an
1401 * embedded comparison (conditional modifier). It is not used on gen7.
1404 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1405 struct brw_reg src0
, struct brw_reg src1
)
1407 const struct gen_device_info
*devinfo
= p
->devinfo
;
1410 insn
= next_insn(p
, BRW_OPCODE_IF
);
1412 brw_set_dest(p
, insn
, brw_imm_w(0));
1413 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1414 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1415 brw_set_src0(p
, insn
, src0
);
1416 brw_set_src1(p
, insn
, src1
);
1418 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1419 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1420 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1422 push_if_stack(p
, insn
);
1427 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1430 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1431 brw_inst
*if_inst
, brw_inst
*else_inst
)
1433 const struct gen_device_info
*devinfo
= p
->devinfo
;
1435 /* The next instruction (where the ENDIF would be, if it existed) */
1436 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1438 assert(p
->single_program_flow
);
1439 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1440 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1441 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1443 /* Convert IF to an ADD instruction that moves the instruction pointer
1444 * to the first instruction of the ELSE block. If there is no ELSE
1445 * block, point to where ENDIF would be. Reverse the predicate.
1447 * There's no need to execute an ENDIF since we don't need to do any
1448 * stack operations, and if we're currently executing, we just want to
1449 * continue normally.
1451 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1452 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1454 if (else_inst
!= NULL
) {
1455 /* Convert ELSE to an ADD instruction that points where the ENDIF
1458 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1460 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1461 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1463 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1468 * Patch IF and ELSE instructions with appropriate jump targets.
1471 patch_IF_ELSE(struct brw_codegen
*p
,
1472 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1474 const struct gen_device_info
*devinfo
= p
->devinfo
;
1476 /* We shouldn't be patching IF and ELSE instructions in single program flow
1477 * mode when gen < 6, because in single program flow mode on those
1478 * platforms, we convert flow control instructions to conditional ADDs that
1479 * operate on IP (see brw_ENDIF).
1481 * However, on Gen6, writing to IP doesn't work in single program flow mode
1482 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1483 * not be updated by non-flow control instructions."). And on later
1484 * platforms, there is no significant benefit to converting control flow
1485 * instructions to conditional ADDs. So we do patch IF and ELSE
1486 * instructions in single program flow mode on those platforms.
1488 if (devinfo
->gen
< 6)
1489 assert(!p
->single_program_flow
);
1491 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1492 assert(endif_inst
!= NULL
);
1493 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1495 unsigned br
= brw_jump_scale(devinfo
);
1497 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1498 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1500 if (else_inst
== NULL
) {
1501 /* Patch IF -> ENDIF */
1502 if (devinfo
->gen
< 6) {
1503 /* Turn it into an IFF, which means no mask stack operations for
1504 * all-false and jumping past the ENDIF.
1506 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1507 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1508 br
* (endif_inst
- if_inst
+ 1));
1509 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1510 } else if (devinfo
->gen
== 6) {
1511 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1512 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1514 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1515 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1518 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1520 /* Patch IF -> ELSE */
1521 if (devinfo
->gen
< 6) {
1522 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1523 br
* (else_inst
- if_inst
));
1524 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1525 } else if (devinfo
->gen
== 6) {
1526 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1527 br
* (else_inst
- if_inst
+ 1));
1530 /* Patch ELSE -> ENDIF */
1531 if (devinfo
->gen
< 6) {
1532 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1535 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1536 br
* (endif_inst
- else_inst
+ 1));
1537 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1538 } else if (devinfo
->gen
== 6) {
1539 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1540 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1541 br
* (endif_inst
- else_inst
));
1543 /* The IF instruction's JIP should point just past the ELSE */
1544 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1545 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1546 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1547 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1548 if (devinfo
->gen
>= 8) {
1549 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1550 * should point to ENDIF.
1552 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1559 brw_ELSE(struct brw_codegen
*p
)
1561 const struct gen_device_info
*devinfo
= p
->devinfo
;
1564 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1566 if (devinfo
->gen
< 6) {
1567 brw_set_dest(p
, insn
, brw_ip_reg());
1568 brw_set_src0(p
, insn
, brw_ip_reg());
1569 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1570 } else if (devinfo
->gen
== 6) {
1571 brw_set_dest(p
, insn
, brw_imm_w(0));
1572 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1573 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1574 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1575 } else if (devinfo
->gen
== 7) {
1576 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1577 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1578 brw_set_src1(p
, insn
, brw_imm_w(0));
1579 brw_inst_set_jip(devinfo
, insn
, 0);
1580 brw_inst_set_uip(devinfo
, insn
, 0);
1582 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1583 if (devinfo
->gen
< 12)
1584 brw_set_src0(p
, insn
, brw_imm_d(0));
1585 brw_inst_set_jip(devinfo
, insn
, 0);
1586 brw_inst_set_uip(devinfo
, insn
, 0);
1589 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1590 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1591 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1592 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1594 push_if_stack(p
, insn
);
1598 brw_ENDIF(struct brw_codegen
*p
)
1600 const struct gen_device_info
*devinfo
= p
->devinfo
;
1601 brw_inst
*insn
= NULL
;
1602 brw_inst
*else_inst
= NULL
;
1603 brw_inst
*if_inst
= NULL
;
1605 bool emit_endif
= true;
1607 /* In single program flow mode, we can express IF and ELSE instructions
1608 * equivalently as ADD instructions that operate on IP. On platforms prior
1609 * to Gen6, flow control instructions cause an implied thread switch, so
1610 * this is a significant savings.
1612 * However, on Gen6, writing to IP doesn't work in single program flow mode
1613 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1614 * not be updated by non-flow control instructions."). And on later
1615 * platforms, there is no significant benefit to converting control flow
1616 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1619 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1623 * A single next_insn() may change the base address of instruction store
1624 * memory(p->store), so call it first before referencing the instruction
1625 * store pointer from an index
1628 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1630 /* Pop the IF and (optional) ELSE instructions from the stack */
1631 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1632 tmp
= pop_if_stack(p
);
1633 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1635 tmp
= pop_if_stack(p
);
1640 /* ENDIF is useless; don't bother emitting it. */
1641 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1645 if (devinfo
->gen
< 6) {
1646 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1647 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1648 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1649 } else if (devinfo
->gen
== 6) {
1650 brw_set_dest(p
, insn
, brw_imm_w(0));
1651 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1652 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1653 } else if (devinfo
->gen
== 7) {
1654 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1655 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1656 brw_set_src1(p
, insn
, brw_imm_w(0));
1658 brw_set_src0(p
, insn
, brw_imm_d(0));
1661 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1662 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1663 if (devinfo
->gen
< 6)
1664 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1666 /* Also pop item off the stack in the endif instruction: */
1667 if (devinfo
->gen
< 6) {
1668 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1669 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1670 } else if (devinfo
->gen
== 6) {
1671 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1673 brw_inst_set_jip(devinfo
, insn
, 2);
1675 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1679 brw_BREAK(struct brw_codegen
*p
)
1681 const struct gen_device_info
*devinfo
= p
->devinfo
;
1684 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1685 if (devinfo
->gen
>= 8) {
1686 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1687 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1688 } else if (devinfo
->gen
>= 6) {
1689 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1690 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1691 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1693 brw_set_dest(p
, insn
, brw_ip_reg());
1694 brw_set_src0(p
, insn
, brw_ip_reg());
1695 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1696 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1697 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1699 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1700 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1706 brw_CONT(struct brw_codegen
*p
)
1708 const struct gen_device_info
*devinfo
= p
->devinfo
;
1711 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1712 brw_set_dest(p
, insn
, brw_ip_reg());
1713 if (devinfo
->gen
>= 8) {
1714 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1716 brw_set_src0(p
, insn
, brw_ip_reg());
1717 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1720 if (devinfo
->gen
< 6) {
1721 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1722 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1724 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1725 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1730 brw_HALT(struct brw_codegen
*p
)
1732 const struct gen_device_info
*devinfo
= p
->devinfo
;
1735 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1736 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1737 if (devinfo
->gen
< 6) {
1738 /* From the Gen4 PRM:
1740 * "IP register must be put (for example, by the assembler) at <dst>
1741 * and <src0> locations.
1743 brw_set_dest(p
, insn
, brw_ip_reg());
1744 brw_set_src0(p
, insn
, brw_ip_reg());
1745 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* exitcode updated later. */
1746 } else if (devinfo
->gen
< 8) {
1747 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1748 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1749 } else if (devinfo
->gen
< 12) {
1750 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1753 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1754 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1760 * The DO/WHILE is just an unterminated loop -- break or continue are
1761 * used for control within the loop. We have a few ways they can be
1764 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1765 * jip and no DO instruction.
1767 * For non-uniform control flow pre-gen6, there's a DO instruction to
1768 * push the mask, and a WHILE to jump back, and BREAK to get out and
1771 * For gen6, there's no more mask stack, so no need for DO. WHILE
1772 * just points back to the first instruction of the loop.
1775 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1777 const struct gen_device_info
*devinfo
= p
->devinfo
;
1779 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1780 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1781 return &p
->store
[p
->nr_insn
];
1783 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1785 push_loop_stack(p
, insn
);
1787 /* Override the defaults for this instruction:
1789 brw_set_dest(p
, insn
, brw_null_reg());
1790 brw_set_src0(p
, insn
, brw_null_reg());
1791 brw_set_src1(p
, insn
, brw_null_reg());
1793 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1794 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1795 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1802 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1805 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1806 * nesting, since it can always just point to the end of the block/current loop.
1809 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1811 const struct gen_device_info
*devinfo
= p
->devinfo
;
1812 brw_inst
*do_inst
= get_inner_do_insn(p
);
1814 unsigned br
= brw_jump_scale(devinfo
);
1816 assert(devinfo
->gen
< 6);
1818 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1819 /* If the jump count is != 0, that means that this instruction has already
1820 * been patched because it's part of a loop inside of the one we're
1823 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1824 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1825 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1826 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1827 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1828 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1834 brw_WHILE(struct brw_codegen
*p
)
1836 const struct gen_device_info
*devinfo
= p
->devinfo
;
1837 brw_inst
*insn
, *do_insn
;
1838 unsigned br
= brw_jump_scale(devinfo
);
1840 if (devinfo
->gen
>= 6) {
1841 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1842 do_insn
= get_inner_do_insn(p
);
1844 if (devinfo
->gen
>= 8) {
1845 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1846 if (devinfo
->gen
< 12)
1847 brw_set_src0(p
, insn
, brw_imm_d(0));
1848 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1849 } else if (devinfo
->gen
== 7) {
1850 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1851 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1852 brw_set_src1(p
, insn
, brw_imm_w(0));
1853 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1855 brw_set_dest(p
, insn
, brw_imm_w(0));
1856 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1857 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1858 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1861 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1864 if (p
->single_program_flow
) {
1865 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1866 do_insn
= get_inner_do_insn(p
);
1868 brw_set_dest(p
, insn
, brw_ip_reg());
1869 brw_set_src0(p
, insn
, brw_ip_reg());
1870 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1871 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1873 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1874 do_insn
= get_inner_do_insn(p
);
1876 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1878 brw_set_dest(p
, insn
, brw_ip_reg());
1879 brw_set_src0(p
, insn
, brw_ip_reg());
1880 brw_set_src1(p
, insn
, brw_imm_d(0));
1882 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1883 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1884 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1886 brw_patch_break_cont(p
, insn
);
1889 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1891 p
->loop_stack_depth
--;
1898 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1900 const struct gen_device_info
*devinfo
= p
->devinfo
;
1901 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1904 if (devinfo
->gen
>= 5)
1907 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1908 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1910 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1911 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1914 /* To integrate with the above, it makes sense that the comparison
1915 * instruction should populate the flag register. It might be simpler
1916 * just to use the flag reg for most WM tasks?
1918 void brw_CMP(struct brw_codegen
*p
,
1919 struct brw_reg dest
,
1920 unsigned conditional
,
1921 struct brw_reg src0
,
1922 struct brw_reg src1
)
1924 const struct gen_device_info
*devinfo
= p
->devinfo
;
1925 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1927 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1928 brw_set_dest(p
, insn
, dest
);
1929 brw_set_src0(p
, insn
, src0
);
1930 brw_set_src1(p
, insn
, src1
);
1932 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1934 * "Any CMP instruction with a null destination must use a {switch}."
1936 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1937 * mentioned on their work-arounds pages.
1939 if (devinfo
->gen
== 7) {
1940 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1941 dest
.nr
== BRW_ARF_NULL
) {
1942 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1947 /***********************************************************************
1948 * Helpers for the various SEND message types:
1951 /** Extended math function, float[8].
1953 void gen4_math(struct brw_codegen
*p
,
1954 struct brw_reg dest
,
1956 unsigned msg_reg_nr
,
1958 unsigned precision
)
1960 const struct gen_device_info
*devinfo
= p
->devinfo
;
1961 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1963 if (has_scalar_region(src
)) {
1964 data_type
= BRW_MATH_DATA_SCALAR
;
1966 data_type
= BRW_MATH_DATA_VECTOR
;
1969 assert(devinfo
->gen
< 6);
1971 /* Example code doesn't set predicate_control for send
1974 brw_inst_set_pred_control(devinfo
, insn
, 0);
1975 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1977 brw_set_dest(p
, insn
, dest
);
1978 brw_set_src0(p
, insn
, src
);
1979 brw_set_math_message(p
,
1982 src
.type
== BRW_REGISTER_TYPE_D
,
1987 void gen6_math(struct brw_codegen
*p
,
1988 struct brw_reg dest
,
1990 struct brw_reg src0
,
1991 struct brw_reg src1
)
1993 const struct gen_device_info
*devinfo
= p
->devinfo
;
1994 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1996 assert(devinfo
->gen
>= 6);
1998 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1999 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
2001 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2002 if (devinfo
->gen
== 6) {
2003 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2004 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2007 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2008 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2009 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2010 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2011 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2012 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2013 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2015 assert(src0
.type
== BRW_REGISTER_TYPE_F
||
2016 (src0
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2017 assert(src1
.type
== BRW_REGISTER_TYPE_F
||
2018 (src1
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2021 /* Source modifiers are ignored for extended math instructions on Gen6. */
2022 if (devinfo
->gen
== 6) {
2023 assert(!src0
.negate
);
2025 assert(!src1
.negate
);
2029 brw_inst_set_math_function(devinfo
, insn
, function
);
2031 brw_set_dest(p
, insn
, dest
);
2032 brw_set_src0(p
, insn
, src0
);
2033 brw_set_src1(p
, insn
, src1
);
2037 * Return the right surface index to access the thread scratch space using
2038 * stateless dataport messages.
2041 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2043 /* The scratch space is thread-local so IA coherency is unnecessary. */
2044 if (p
->devinfo
->gen
>= 8)
2045 return GEN8_BTI_STATELESS_NON_COHERENT
;
2047 return BRW_BTI_STATELESS
;
2051 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2052 * using a constant offset per channel.
2054 * The offset must be aligned to oword size (16 bytes). Used for
2055 * register spilling.
2057 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2062 const struct gen_device_info
*devinfo
= p
->devinfo
;
2063 const unsigned target_cache
=
2064 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2065 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2066 BRW_SFID_DATAPORT_WRITE
);
2067 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2070 if (devinfo
->gen
>= 6)
2073 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2075 const unsigned mlen
= 1 + num_regs
;
2077 /* Set up the message header. This is g0, with g0.2 filled with
2078 * the offset. We don't want to leave our offset around in g0 or
2079 * it'll screw up texture samples, so set it up inside the message
2083 brw_push_insn_state(p
);
2084 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2085 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2086 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2087 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2089 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2091 /* set message header global offset field (reg 0, element 2) */
2092 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2093 brw_set_default_swsb(p
, tgl_swsb_null());
2095 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2097 2), BRW_REGISTER_TYPE_UD
),
2098 brw_imm_ud(offset
));
2100 brw_pop_insn_state(p
);
2101 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2105 struct brw_reg dest
;
2106 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2107 int send_commit_msg
;
2108 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2109 BRW_REGISTER_TYPE_UW
);
2111 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2112 brw_inst_set_compression(devinfo
, insn
, false);
2114 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2115 src_header
= vec16(src_header
);
2117 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2118 if (devinfo
->gen
< 6)
2119 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2121 /* Until gen6, writes followed by reads from the same location
2122 * are not guaranteed to be ordered unless write_commit is set.
2123 * If set, then a no-op write is issued to the destination
2124 * register to set a dependency, and a read from the destination
2125 * can be used to ensure the ordering.
2127 * For gen6, only writes between different threads need ordering
2128 * protection. Our use of DP writes is all about register
2129 * spilling within a thread.
2131 if (devinfo
->gen
>= 6) {
2132 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2133 send_commit_msg
= 0;
2136 send_commit_msg
= 1;
2139 brw_set_dest(p
, insn
, dest
);
2140 if (devinfo
->gen
>= 6) {
2141 brw_set_src0(p
, insn
, mrf
);
2143 brw_set_src0(p
, insn
, brw_null_reg());
2146 if (devinfo
->gen
>= 6)
2147 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2149 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2151 brw_set_desc(p
, insn
,
2152 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2153 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2154 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2155 msg_type
, 0, /* not a render target */
2162 * Read a block of owords (half a GRF each) from the scratch buffer
2163 * using a constant index per channel.
2165 * Offset must be aligned to oword size (16 bytes). Used for register
2169 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2170 struct brw_reg dest
,
2175 const struct gen_device_info
*devinfo
= p
->devinfo
;
2176 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2178 if (devinfo
->gen
>= 6)
2181 if (p
->devinfo
->gen
>= 7) {
2182 /* On gen 7 and above, we no longer have message registers and we can
2183 * send from any register we want. By using the destination register
2184 * for the message, we guarantee that the implied message write won't
2185 * accidentally overwrite anything. This has been a problem because
2186 * the MRF registers and source for the final FB write are both fixed
2189 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2191 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2193 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2195 const unsigned rlen
= num_regs
;
2196 const unsigned target_cache
=
2197 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2198 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2199 BRW_SFID_DATAPORT_READ
);
2202 brw_push_insn_state(p
);
2203 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2204 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2205 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2206 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2208 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2210 /* set message header global offset field (reg 0, element 2) */
2211 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2212 brw_set_default_swsb(p
, tgl_swsb_null());
2213 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2215 brw_pop_insn_state(p
);
2216 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2220 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2222 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2223 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2224 brw_inst_set_compression(devinfo
, insn
, false);
2226 brw_set_dest(p
, insn
, dest
); /* UW? */
2227 if (devinfo
->gen
>= 6) {
2228 brw_set_src0(p
, insn
, mrf
);
2230 brw_set_src0(p
, insn
, brw_null_reg());
2231 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2234 brw_set_desc(p
, insn
,
2235 brw_message_desc(devinfo
, 1, rlen
, true) |
2236 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2237 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2238 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2239 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2244 gen7_block_read_scratch(struct brw_codegen
*p
,
2245 struct brw_reg dest
,
2249 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2250 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2252 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2254 /* The HW requires that the header is present; this is to get the g0.5
2257 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2259 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2260 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2261 * is 32 bytes, which happens to be the size of a register.
2264 assert(offset
< (1 << 12));
2266 gen7_set_dp_scratch_message(p
, insn
,
2267 false, /* scratch read */
2269 false, /* invalidate after read */
2272 1, /* mlen: just g0 */
2273 num_regs
, /* rlen */
2274 true); /* header present */
2278 * Read float[4] vectors from the data port constant cache.
2279 * Location (in buffer) should be a multiple of 16.
2280 * Used for fetching shader constants.
2282 void brw_oword_block_read(struct brw_codegen
*p
,
2283 struct brw_reg dest
,
2286 uint32_t bind_table_index
)
2288 const struct gen_device_info
*devinfo
= p
->devinfo
;
2289 const unsigned target_cache
=
2290 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2291 BRW_SFID_DATAPORT_READ
);
2292 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2293 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2295 /* On newer hardware, offset is in units of owords. */
2296 if (devinfo
->gen
>= 6)
2299 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2301 brw_push_insn_state(p
);
2302 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2303 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2304 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2306 brw_push_insn_state(p
);
2307 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2308 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2309 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2311 /* set message header global offset field (reg 0, element 2) */
2312 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2313 brw_set_default_swsb(p
, tgl_swsb_null());
2315 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2317 2), BRW_REGISTER_TYPE_UD
),
2318 brw_imm_ud(offset
));
2319 brw_pop_insn_state(p
);
2321 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2323 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2325 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2327 /* cast dest to a uword[8] vector */
2328 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2330 brw_set_dest(p
, insn
, dest
);
2331 if (devinfo
->gen
>= 6) {
2332 brw_set_src0(p
, insn
, mrf
);
2334 brw_set_src0(p
, insn
, brw_null_reg());
2335 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2338 brw_set_desc(p
, insn
,
2339 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2340 brw_dp_read_desc(devinfo
, bind_table_index
,
2341 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2342 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2343 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2345 brw_pop_insn_state(p
);
2349 brw_fb_WRITE(struct brw_codegen
*p
,
2350 struct brw_reg payload
,
2351 struct brw_reg implied_header
,
2352 unsigned msg_control
,
2353 unsigned binding_table_index
,
2354 unsigned msg_length
,
2355 unsigned response_length
,
2357 bool last_render_target
,
2358 bool header_present
)
2360 const struct gen_device_info
*devinfo
= p
->devinfo
;
2361 const unsigned target_cache
=
2362 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2363 BRW_SFID_DATAPORT_WRITE
);
2366 struct brw_reg dest
, src0
;
2368 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2369 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2371 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2373 if (devinfo
->gen
>= 6) {
2374 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2376 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2378 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2379 brw_inst_set_compression(devinfo
, insn
, false);
2381 if (devinfo
->gen
>= 6) {
2382 /* headerless version, just submit color payload */
2385 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2387 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2388 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2389 src0
= implied_header
;
2391 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2394 brw_set_dest(p
, insn
, dest
);
2395 brw_set_src0(p
, insn
, src0
);
2396 brw_set_desc(p
, insn
,
2397 brw_message_desc(devinfo
, msg_length
, response_length
,
2399 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2400 msg_type
, last_render_target
,
2401 0 /* send_commit_msg */));
2402 brw_inst_set_eot(devinfo
, insn
, eot
);
2408 gen9_fb_READ(struct brw_codegen
*p
,
2410 struct brw_reg payload
,
2411 unsigned binding_table_index
,
2412 unsigned msg_length
,
2413 unsigned response_length
,
2416 const struct gen_device_info
*devinfo
= p
->devinfo
;
2417 assert(devinfo
->gen
>= 9);
2418 const unsigned msg_subtype
=
2419 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2420 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2422 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2423 brw_set_dest(p
, insn
, dst
);
2424 brw_set_src0(p
, insn
, payload
);
2427 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2428 brw_dp_read_desc(devinfo
, binding_table_index
,
2429 per_sample
<< 5 | msg_subtype
,
2430 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2431 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2432 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2438 * Texture sample instruction.
2439 * Note: the msg_type plus msg_length values determine exactly what kind
2440 * of sampling operation is performed. See volume 4, page 161 of docs.
2442 void brw_SAMPLE(struct brw_codegen
*p
,
2443 struct brw_reg dest
,
2444 unsigned msg_reg_nr
,
2445 struct brw_reg src0
,
2446 unsigned binding_table_index
,
2449 unsigned response_length
,
2450 unsigned msg_length
,
2451 unsigned header_present
,
2453 unsigned return_format
)
2455 const struct gen_device_info
*devinfo
= p
->devinfo
;
2458 if (msg_reg_nr
!= -1)
2459 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2461 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2462 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2463 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2465 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2467 * "Instruction compression is not allowed for this instruction (that
2468 * is, send). The hardware behavior is undefined if this instruction is
2469 * set as compressed. However, compress control can be set to "SecHalf"
2470 * to affect the EMask generation."
2472 * No similar wording is found in later PRMs, but there are examples
2473 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2474 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2475 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2477 brw_inst_set_compression(devinfo
, insn
, false);
2479 if (devinfo
->gen
< 6)
2480 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2482 brw_set_dest(p
, insn
, dest
);
2483 brw_set_src0(p
, insn
, src0
);
2484 brw_set_desc(p
, insn
,
2485 brw_message_desc(devinfo
, msg_length
, response_length
,
2487 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2488 msg_type
, simd_mode
, return_format
));
2491 /* Adjust the message header's sampler state pointer to
2492 * select the correct group of 16 samplers.
2494 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2495 struct brw_reg header
,
2496 struct brw_reg sampler_index
)
2498 /* The "Sampler Index" field can only store values between 0 and 15.
2499 * However, we can add an offset to the "Sampler State Pointer"
2500 * field, effectively selecting a different set of 16 samplers.
2502 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2503 * offset, and each sampler state is only 16-bytes, so we can't
2504 * exclusively use the offset - we have to use both.
2507 const struct gen_device_info
*devinfo
= p
->devinfo
;
2509 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2510 const int sampler_state_size
= 16; /* 16 bytes */
2511 uint32_t sampler
= sampler_index
.ud
;
2513 if (sampler
>= 16) {
2514 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2516 get_element_ud(header
, 3),
2517 get_element_ud(brw_vec8_grf(0, 0), 3),
2518 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2521 /* Non-const sampler array indexing case */
2522 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2526 struct brw_reg temp
= get_element_ud(header
, 3);
2528 brw_push_insn_state(p
);
2529 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2530 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
2531 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2533 get_element_ud(header
, 3),
2534 get_element_ud(brw_vec8_grf(0, 0), 3),
2536 brw_pop_insn_state(p
);
2540 /* All these variables are pretty confusing - we might be better off
2541 * using bitmasks and macros for this, in the old style. Or perhaps
2542 * just having the caller instantiate the fields in dword3 itself.
2544 void brw_urb_WRITE(struct brw_codegen
*p
,
2545 struct brw_reg dest
,
2546 unsigned msg_reg_nr
,
2547 struct brw_reg src0
,
2548 enum brw_urb_write_flags flags
,
2549 unsigned msg_length
,
2550 unsigned response_length
,
2554 const struct gen_device_info
*devinfo
= p
->devinfo
;
2557 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2559 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2560 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2561 brw_push_insn_state(p
);
2562 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2563 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2564 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2565 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2566 BRW_REGISTER_TYPE_UD
),
2567 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2568 brw_imm_ud(0xff00));
2569 brw_pop_insn_state(p
);
2572 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2574 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2576 brw_set_dest(p
, insn
, dest
);
2577 brw_set_src0(p
, insn
, src0
);
2578 brw_set_src1(p
, insn
, brw_imm_d(0));
2580 if (devinfo
->gen
< 6)
2581 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2583 brw_set_urb_message(p
,
2593 brw_send_indirect_message(struct brw_codegen
*p
,
2596 struct brw_reg payload
,
2597 struct brw_reg desc
,
2601 const struct gen_device_info
*devinfo
= p
->devinfo
;
2602 struct brw_inst
*send
;
2604 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2606 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2608 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2609 send
= next_insn(p
, BRW_OPCODE_SEND
);
2610 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2611 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2613 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2614 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2616 brw_push_insn_state(p
);
2617 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2618 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2619 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2620 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2621 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2623 /* Load the indirect descriptor to an address register using OR so the
2624 * caller can specify additional descriptor bits with the desc_imm
2627 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2629 brw_pop_insn_state(p
);
2631 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2632 send
= next_insn(p
, BRW_OPCODE_SEND
);
2633 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2635 if (devinfo
->gen
>= 12)
2636 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, true);
2638 brw_set_src1(p
, send
, addr
);
2641 brw_set_dest(p
, send
, dst
);
2642 brw_inst_set_sfid(devinfo
, send
, sfid
);
2643 brw_inst_set_eot(devinfo
, send
, eot
);
2647 brw_send_indirect_split_message(struct brw_codegen
*p
,
2650 struct brw_reg payload0
,
2651 struct brw_reg payload1
,
2652 struct brw_reg desc
,
2654 struct brw_reg ex_desc
,
2655 unsigned ex_desc_imm
,
2658 const struct gen_device_info
*devinfo
= p
->devinfo
;
2659 struct brw_inst
*send
;
2661 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2663 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2665 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2666 desc
.ud
|= desc_imm
;
2668 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2669 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2671 brw_push_insn_state(p
);
2672 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2673 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2674 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2675 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2676 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2678 /* Load the indirect descriptor to an address register using OR so the
2679 * caller can specify additional descriptor bits with the desc_imm
2682 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2684 brw_pop_insn_state(p
);
2687 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2690 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
&&
2691 (devinfo
->gen
>= 12 || (ex_desc
.ud
& INTEL_MASK(15, 12)) == 0)) {
2692 ex_desc
.ud
|= ex_desc_imm
;
2694 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2695 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2697 brw_push_insn_state(p
);
2698 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2699 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2700 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2701 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2702 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2704 /* Load the indirect extended descriptor to an address register using OR
2705 * so the caller can specify additional descriptor bits with the
2706 * desc_imm immediate.
2708 * Even though the instruction dispatcher always pulls the SFID and EOT
2709 * fields from the instruction itself, actual external unit which
2710 * processes the message gets the SFID and EOT from the extended
2711 * descriptor which comes from the address register. If we don't OR
2712 * those two bits in, the external unit may get confused and hang.
2714 unsigned imm_part
= ex_desc_imm
| sfid
| eot
<< 5;
2716 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2717 /* ex_desc bits 15:12 don't exist in the instruction encoding prior
2718 * to Gen12, so we may have fallen back to an indirect extended
2721 brw_MOV(p
, addr
, brw_imm_ud(ex_desc
.ud
| imm_part
));
2723 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(imm_part
));
2726 brw_pop_insn_state(p
);
2729 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2732 send
= next_insn(p
, devinfo
->gen
>= 12 ? BRW_OPCODE_SEND
: BRW_OPCODE_SENDS
);
2733 brw_set_dest(p
, send
, dst
);
2734 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2735 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2737 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2738 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2739 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2741 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2742 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2743 assert(desc
.subnr
== 0);
2744 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2747 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2748 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2749 brw_inst_set_sends_ex_desc(devinfo
, send
, ex_desc
.ud
);
2751 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2752 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2753 assert((ex_desc
.subnr
& 0x3) == 0);
2754 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2755 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2758 brw_inst_set_sfid(devinfo
, send
, sfid
);
2759 brw_inst_set_eot(devinfo
, send
, eot
);
2763 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2766 struct brw_reg payload
,
2767 struct brw_reg surface
,
2770 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2771 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2772 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2774 brw_push_insn_state(p
);
2775 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2776 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2777 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2778 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2779 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2781 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2782 * some surface array is accessed out of bounds.
2785 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2786 BRW_GET_SWZ(surface
.swizzle
, 0)),
2789 brw_pop_insn_state(p
);
2792 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2795 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2799 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2800 brw_inst
*insn
, int while_offset
, int start_offset
)
2802 int scale
= 16 / brw_jump_scale(devinfo
);
2803 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2804 : brw_inst_jip(devinfo
, insn
);
2806 return while_offset
+ jip
* scale
<= start_offset
;
2811 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2814 void *store
= p
->store
;
2815 const struct gen_device_info
*devinfo
= p
->devinfo
;
2819 for (offset
= next_offset(devinfo
, store
, start_offset
);
2820 offset
< p
->next_insn_offset
;
2821 offset
= next_offset(devinfo
, store
, offset
)) {
2822 brw_inst
*insn
= store
+ offset
;
2824 switch (brw_inst_opcode(devinfo
, insn
)) {
2828 case BRW_OPCODE_ENDIF
:
2833 case BRW_OPCODE_WHILE
:
2834 /* If the while doesn't jump before our instruction, it's the end
2835 * of a sibling do...while loop. Ignore it.
2837 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2840 case BRW_OPCODE_ELSE
:
2841 case BRW_OPCODE_HALT
:
2852 /* There is no DO instruction on gen6, so to find the end of the loop
2853 * we have to see if the loop is jumping back before our start
2857 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2859 const struct gen_device_info
*devinfo
= p
->devinfo
;
2861 void *store
= p
->store
;
2863 assert(devinfo
->gen
>= 6);
2865 /* Always start after the instruction (such as a WHILE) we're trying to fix
2868 for (offset
= next_offset(devinfo
, store
, start_offset
);
2869 offset
< p
->next_insn_offset
;
2870 offset
= next_offset(devinfo
, store
, offset
)) {
2871 brw_inst
*insn
= store
+ offset
;
2873 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2874 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2878 assert(!"not reached");
2879 return start_offset
;
2882 /* After program generation, go back and update the UIP and JIP of
2883 * BREAK, CONT, and HALT instructions to their correct locations.
2886 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2888 const struct gen_device_info
*devinfo
= p
->devinfo
;
2890 int br
= brw_jump_scale(devinfo
);
2891 int scale
= 16 / br
;
2892 void *store
= p
->store
;
2894 if (devinfo
->gen
< 6)
2897 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2898 brw_inst
*insn
= store
+ offset
;
2899 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2901 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2902 switch (brw_inst_opcode(devinfo
, insn
)) {
2903 case BRW_OPCODE_BREAK
:
2904 assert(block_end_offset
!= 0);
2905 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2906 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2907 brw_inst_set_uip(devinfo
, insn
,
2908 (brw_find_loop_end(p
, offset
) - offset
+
2909 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2911 case BRW_OPCODE_CONTINUE
:
2912 assert(block_end_offset
!= 0);
2913 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2914 brw_inst_set_uip(devinfo
, insn
,
2915 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2917 assert(brw_inst_uip(devinfo
, insn
) != 0);
2918 assert(brw_inst_jip(devinfo
, insn
) != 0);
2921 case BRW_OPCODE_ENDIF
: {
2922 int32_t jump
= (block_end_offset
== 0) ?
2923 1 * br
: (block_end_offset
- offset
) / scale
;
2924 if (devinfo
->gen
>= 7)
2925 brw_inst_set_jip(devinfo
, insn
, jump
);
2927 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2931 case BRW_OPCODE_HALT
:
2932 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2934 * "In case of the halt instruction not inside any conditional
2935 * code block, the value of <JIP> and <UIP> should be the
2936 * same. In case of the halt instruction inside conditional code
2937 * block, the <UIP> should be the end of the program, and the
2938 * <JIP> should be end of the most inner conditional code block."
2940 * The uip will have already been set by whoever set up the
2943 if (block_end_offset
== 0) {
2944 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2946 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2948 assert(brw_inst_uip(devinfo
, insn
) != 0);
2949 assert(brw_inst_jip(devinfo
, insn
) != 0);
2958 void brw_ff_sync(struct brw_codegen
*p
,
2959 struct brw_reg dest
,
2960 unsigned msg_reg_nr
,
2961 struct brw_reg src0
,
2963 unsigned response_length
,
2966 const struct gen_device_info
*devinfo
= p
->devinfo
;
2969 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2971 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2972 brw_set_dest(p
, insn
, dest
);
2973 brw_set_src0(p
, insn
, src0
);
2974 brw_set_src1(p
, insn
, brw_imm_d(0));
2976 if (devinfo
->gen
< 6)
2977 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2979 brw_set_ff_sync_message(p
,
2987 * Emit the SEND instruction necessary to generate stream output data on Gen6
2988 * (for transform feedback).
2990 * If send_commit_msg is true, this is the last piece of stream output data
2991 * from this thread, so send the data as a committed write. According to the
2992 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2994 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2995 * writes are complete by sending the final write as a committed write."
2998 brw_svb_write(struct brw_codegen
*p
,
2999 struct brw_reg dest
,
3000 unsigned msg_reg_nr
,
3001 struct brw_reg src0
,
3002 unsigned binding_table_index
,
3003 bool send_commit_msg
)
3005 const struct gen_device_info
*devinfo
= p
->devinfo
;
3006 const unsigned target_cache
=
3007 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
3008 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
3009 BRW_SFID_DATAPORT_WRITE
);
3012 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
3014 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3015 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
3016 brw_set_dest(p
, insn
, dest
);
3017 brw_set_src0(p
, insn
, src0
);
3018 brw_set_desc(p
, insn
,
3019 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
3020 brw_dp_write_desc(devinfo
, binding_table_index
,
3021 0, /* msg_control: ignored */
3022 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
3023 0, /* last_render_target: ignored */
3024 send_commit_msg
)); /* send_commit_msg */
3028 brw_surface_payload_size(struct brw_codegen
*p
,
3029 unsigned num_channels
,
3030 unsigned exec_size
/**< 0 for SIMD4x2 */)
3033 return 1; /* SIMD4x2 */
3034 else if (exec_size
<= 8)
3035 return num_channels
;
3037 return 2 * num_channels
;
3041 brw_untyped_atomic(struct brw_codegen
*p
,
3043 struct brw_reg payload
,
3044 struct brw_reg surface
,
3046 unsigned msg_length
,
3047 bool response_expected
,
3048 bool header_present
)
3050 const struct gen_device_info
*devinfo
= p
->devinfo
;
3051 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3052 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3053 GEN7_SFID_DATAPORT_DATA_CACHE
);
3054 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3055 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
3056 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3057 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3058 has_simd4x2
? 0 : 8;
3059 const unsigned response_length
=
3060 brw_surface_payload_size(p
, response_expected
, exec_size
);
3061 const unsigned desc
=
3062 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3063 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
3065 /* Mask out unused components -- This is especially important in Align16
3066 * mode on generations that don't have native support for SIMD4x2 atomics,
3067 * because unused but enabled components will cause the dataport to perform
3068 * additional atomic operations on the addresses that happen to be in the
3069 * uninitialized Y, Z and W coordinates of the payload.
3071 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3073 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
3074 payload
, surface
, desc
);
3078 brw_untyped_surface_read(struct brw_codegen
*p
,
3080 struct brw_reg payload
,
3081 struct brw_reg surface
,
3082 unsigned msg_length
,
3083 unsigned num_channels
)
3085 const struct gen_device_info
*devinfo
= p
->devinfo
;
3086 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3087 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3088 GEN7_SFID_DATAPORT_DATA_CACHE
);
3089 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3090 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
3091 const unsigned response_length
=
3092 brw_surface_payload_size(p
, num_channels
, exec_size
);
3093 const unsigned desc
=
3094 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3095 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
3097 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
3101 brw_untyped_surface_write(struct brw_codegen
*p
,
3102 struct brw_reg payload
,
3103 struct brw_reg surface
,
3104 unsigned msg_length
,
3105 unsigned num_channels
,
3106 bool header_present
)
3108 const struct gen_device_info
*devinfo
= p
->devinfo
;
3109 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3110 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3111 GEN7_SFID_DATAPORT_DATA_CACHE
);
3112 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3113 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
3114 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3115 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3116 has_simd4x2
? 0 : 8;
3117 const unsigned desc
=
3118 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3119 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
3120 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3121 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
3123 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3124 payload
, surface
, desc
);
3128 brw_set_memory_fence_message(struct brw_codegen
*p
,
3129 struct brw_inst
*insn
,
3130 enum brw_message_target sfid
,
3134 const struct gen_device_info
*devinfo
= p
->devinfo
;
3136 brw_set_desc(p
, insn
, brw_message_desc(
3137 devinfo
, 1, (commit_enable
? 1 : 0), true));
3139 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3142 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3143 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3145 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3146 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3149 unreachable("Not reached");
3153 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3155 assert(devinfo
->gen
>= 11 || bti
== 0);
3156 brw_inst_set_binding_table_index(devinfo
, insn
, bti
);
3160 brw_memory_fence(struct brw_codegen
*p
,
3163 enum opcode send_op
,
3164 enum brw_message_target sfid
,
3168 const struct gen_device_info
*devinfo
= p
->devinfo
;
3170 dst
= retype(vec1(dst
), BRW_REGISTER_TYPE_UW
);
3171 src
= retype(vec1(src
), BRW_REGISTER_TYPE_UD
);
3173 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3174 * message doesn't write anything back.
3176 struct brw_inst
*insn
= next_insn(p
, send_op
);
3177 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3178 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3179 brw_set_dest(p
, insn
, dst
);
3180 brw_set_src0(p
, insn
, src
);
3181 brw_set_memory_fence_message(p
, insn
, sfid
, commit_enable
, bti
);
3185 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3186 struct brw_reg dest
,
3190 struct brw_reg data
,
3191 unsigned msg_length
,
3192 unsigned response_length
)
3194 const struct gen_device_info
*devinfo
= p
->devinfo
;
3195 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3196 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3197 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3198 const unsigned desc
=
3199 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3200 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3203 /* brw_send_indirect_message will automatically use a direct send message
3204 * if data is actually immediate.
3206 brw_send_indirect_message(p
,
3207 GEN7_SFID_PIXEL_INTERPOLATOR
,
3216 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3217 struct brw_reg mask
)
3219 const struct gen_device_info
*devinfo
= p
->devinfo
;
3220 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3221 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3224 assert(devinfo
->gen
>= 7);
3225 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3227 brw_push_insn_state(p
);
3229 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3230 * unnecessary bits in the instruction words, get the information we need
3231 * and reset the default flag register. This allows more instructions to be
3234 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3235 brw_set_default_flag_reg(p
, 0, 0);
3237 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3238 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3240 if (devinfo
->gen
>= 8) {
3241 /* Getting the first active channel index is easy on Gen8: Just find
3242 * the first bit set in the execution mask. The register exists on
3243 * HSW already but it reads back as all ones when the current
3244 * instruction has execution masking disabled, so it's kind of
3247 struct brw_reg exec_mask
=
3248 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3250 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3251 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3252 /* Unfortunately, ce0 does not take into account the thread
3253 * dispatch mask, which may be a problem in cases where it's not
3254 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3255 * some n). Combine ce0 with the given dispatch (or vector) mask
3256 * to mask off those channels which were never dispatched by the
3259 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3260 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3261 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3262 exec_mask
= vec1(dst
);
3265 /* Quarter control has the effect of magically shifting the value of
3266 * ce0 so you'll get the first active channel relative to the
3267 * specified quarter control as result.
3269 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3271 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3273 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3274 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3276 /* Run enough instructions returning zero with execution masking and
3277 * a conditional modifier enabled in order to get the full execution
3278 * mask in f1.0. We could use a single 32-wide move here if it
3279 * weren't because of the hardware bug that causes channel enables to
3280 * be applied incorrectly to the second half of 32-wide instructions
3283 const unsigned lower_size
= MIN2(16, exec_size
);
3284 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3285 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3287 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3288 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3289 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3290 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3291 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3292 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3295 /* Find the first bit set in the exec_size-wide portion of the flag
3296 * register that was updated by the last sequence of MOV
3299 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3300 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3301 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3304 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3306 if (devinfo
->gen
>= 8 &&
3307 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3308 /* In SIMD4x2 mode the first active channel index is just the
3309 * negation of the first bit of the mask register. Note that ce0
3310 * doesn't take into account the dispatch mask, so the Gen7 path
3311 * should be used instead unless you have the guarantee that the
3312 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3315 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3316 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3320 /* Overwrite the destination without and with execution masking to
3321 * find out which of the channels is active.
3323 brw_push_insn_state(p
);
3324 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3325 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3328 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3330 brw_pop_insn_state(p
);
3331 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3335 brw_pop_insn_state(p
);
3339 brw_broadcast(struct brw_codegen
*p
,
3344 const struct gen_device_info
*devinfo
= p
->devinfo
;
3345 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3348 brw_push_insn_state(p
);
3349 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3350 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3352 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3353 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3354 assert(!src
.abs
&& !src
.negate
);
3355 assert(src
.type
== dst
.type
);
3357 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3358 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3359 /* Trivial, the source is already uniform or the index is a constant.
3360 * We will typically not get here if the optimizer is doing its job, but
3361 * asserting would be mean.
3363 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3365 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3366 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3368 /* From the Haswell PRM section "Register Region Restrictions":
3370 * "The lower bits of the AddressImmediate must not overflow to
3371 * change the register address. The lower 5 bits of Address
3372 * Immediate when added to lower 5 bits of address register gives
3373 * the sub-register offset. The upper bits of Address Immediate
3374 * when added to upper bits of address register gives the register
3375 * address. Any overflow from sub-register offset is dropped."
3377 * Fortunately, for broadcast, we never have a sub-register offset so
3378 * this isn't an issue.
3380 assert(src
.subnr
== 0);
3383 const struct brw_reg addr
=
3384 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3385 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3386 /* Limit in bytes of the signed indirect addressing immediate. */
3387 const unsigned limit
= 512;
3389 brw_push_insn_state(p
);
3390 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3391 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3393 /* Take into account the component size and horizontal stride. */
3394 assert(src
.vstride
== src
.hstride
+ src
.width
);
3395 brw_SHL(p
, addr
, vec1(idx
),
3396 brw_imm_ud(util_logbase2(type_sz(src
.type
)) +
3399 /* We can only address up to limit bytes using the indirect
3400 * addressing immediate, account for the difference if the source
3401 * register is above this limit.
3403 if (offset
>= limit
) {
3404 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3405 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3406 offset
= offset
% limit
;
3409 brw_pop_insn_state(p
);
3411 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3413 /* Use indirect addressing to fetch the specified component. */
3414 if (type_sz(src
.type
) > 4 &&
3415 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3416 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3418 * "When source or destination datatype is 64b or operation is
3419 * integer DWord multiply, indirect addressing must not be
3422 * To work around both of this issue, we do two integer MOVs
3423 * insead of one 64-bit MOV. Because no double value should ever
3424 * cross a register boundary, it's safe to use the immediate
3425 * offset in the indirect here to handle adding 4 bytes to the
3426 * offset and avoid the extra ADD to the register file.
3428 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3429 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3430 BRW_REGISTER_TYPE_D
));
3431 brw_set_default_swsb(p
, tgl_swsb_null());
3432 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3433 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3434 BRW_REGISTER_TYPE_D
));
3437 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3440 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3441 * to all bits of a flag register,
3445 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3446 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3447 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3448 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3450 /* and use predicated SEL to pick the right channel. */
3451 inst
= brw_SEL(p
, dst
,
3452 stride(suboffset(src
, 4), 4, 4, 1),
3453 stride(src
, 4, 4, 1));
3454 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3455 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3459 brw_pop_insn_state(p
);
3463 * This instruction is generated as a single-channel align1 instruction by
3464 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3466 * We can't use the typed atomic op in the FS because that has the execution
3467 * mask ANDed with the pixel mask, but we just want to write the one dword for
3470 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3471 * one u32. So we use the same untyped atomic write message as the pixel
3474 * The untyped atomic operation requires a BUFFER surface type with RAW
3475 * format, and is only accessible through the legacy DATA_CACHE dataport
3478 void brw_shader_time_add(struct brw_codegen
*p
,
3479 struct brw_reg payload
,
3480 uint32_t surf_index
)
3482 const struct gen_device_info
*devinfo
= p
->devinfo
;
3483 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3484 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3485 GEN7_SFID_DATAPORT_DATA_CACHE
);
3486 assert(devinfo
->gen
>= 7);
3488 brw_push_insn_state(p
);
3489 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3490 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3491 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3492 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3494 /* We use brw_vec1_reg and unmasked because we want to increment the given
3497 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3499 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3501 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3502 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3505 brw_inst_set_sfid(devinfo
, send
, sfid
);
3506 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3508 brw_pop_insn_state(p
);
3513 * Emit the SEND message for a barrier
3516 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3518 const struct gen_device_info
*devinfo
= p
->devinfo
;
3519 struct brw_inst
*inst
;
3521 assert(devinfo
->gen
>= 7);
3523 brw_push_insn_state(p
);
3524 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3525 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3526 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3527 brw_set_src0(p
, inst
, src
);
3528 brw_set_src1(p
, inst
, brw_null_reg());
3529 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3531 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3532 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3533 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3535 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3536 brw_pop_insn_state(p
);
3541 * Emit the wait instruction for a barrier
3544 brw_WAIT(struct brw_codegen
*p
)
3546 const struct gen_device_info
*devinfo
= p
->devinfo
;
3547 struct brw_inst
*insn
;
3549 struct brw_reg src
= brw_notification_reg();
3551 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3552 brw_set_dest(p
, insn
, src
);
3553 brw_set_src0(p
, insn
, src
);
3554 brw_set_src1(p
, insn
, brw_null_reg());
3556 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3557 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3561 brw_float_controls_mode(struct brw_codegen
*p
,
3562 unsigned mode
, unsigned mask
)
3564 /* From the Skylake PRM, Volume 7, page 760:
3565 * "Implementation Restriction on Register Access: When the control
3566 * register is used as an explicit source and/or destination, hardware
3567 * does not ensure execution pipeline coherency. Software must set the
3568 * thread control field to ‘switch’ for an instruction that uses
3569 * control register as an explicit operand."
3571 * On Gen12+ this is implemented in terms of SWSB annotations instead.
3573 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3575 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3577 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3578 if (p
->devinfo
->gen
< 12)
3579 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3582 brw_inst
*inst_or
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3584 brw_inst_set_exec_size(p
->devinfo
, inst_or
, BRW_EXECUTE_1
);
3585 if (p
->devinfo
->gen
< 12)
3586 brw_inst_set_thread_control(p
->devinfo
, inst_or
, BRW_THREAD_SWITCH
);
3589 if (p
->devinfo
->gen
>= 12)
3590 brw_SYNC(p
, TGL_SYNC_NOP
);