2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 assert(devinfo
->gen
< 12);
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct gen_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
91 const struct gen_device_info
*devinfo
= p
->devinfo
;
93 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
94 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
95 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
96 assert(dest
.nr
< 128);
98 /* The hardware has a restriction where if the destination is Byte,
99 * the instruction needs to have a stride of 2 (except for packed byte
100 * MOV). This seems to be required even if the destination is the NULL
103 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
104 dest
.nr
== BRW_ARF_NULL
&&
105 type_sz(dest
.type
) == 1) {
106 dest
.hstride
= BRW_HORIZONTAL_STRIDE_2
;
109 gen7_convert_mrf_to_grf(p
, &dest
);
111 if (devinfo
->gen
>= 12 &&
112 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
113 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
114 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
115 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
116 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
117 assert(dest
.subnr
== 0);
118 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
119 (dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
120 dest
.vstride
== dest
.width
+ 1));
121 assert(!dest
.negate
&& !dest
.abs
);
122 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
123 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
125 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
126 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
127 assert(devinfo
->gen
< 12);
128 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
129 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
130 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
131 assert(dest
.subnr
% 16 == 0);
132 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
133 dest
.vstride
== dest
.width
+ 1);
134 assert(!dest
.negate
&& !dest
.abs
);
135 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
136 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
137 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
139 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
140 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
142 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
143 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
145 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
146 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
147 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
148 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
149 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
151 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
152 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
153 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
154 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
155 assert(dest
.writemask
!= 0);
157 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
158 * Although Dst.HorzStride is a don't care for Align16, HW needs
159 * this to be programmed as "01".
161 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
164 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
166 /* These are different sizes in align1 vs align16:
168 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
169 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
170 dest
.indirect_offset
);
171 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
172 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
173 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
175 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
176 dest
.indirect_offset
);
177 /* even ignored in da16, still need to set as '01' */
178 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
183 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
184 * or 16 (SIMD16), as that's normally correct. However, when dealing with
185 * small registers, it can be useful for us to automatically reduce it to
186 * match the register size.
188 if (p
->automatic_exec_sizes
) {
190 * In platforms that support fp64 we can emit instructions with a width
191 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
192 * these cases we need to make sure that these instructions have their
193 * exec sizes set properly when they are emitted and we can't rely on
194 * this code to fix it.
197 if (devinfo
->gen
>= 6)
198 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
200 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
203 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
208 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
210 const struct gen_device_info
*devinfo
= p
->devinfo
;
212 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
213 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
214 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
215 assert(reg
.nr
< 128);
217 gen7_convert_mrf_to_grf(p
, ®
);
219 if (devinfo
->gen
>= 6 &&
220 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
221 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
222 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
223 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
224 /* Any source modifiers or regions will be ignored, since this just
225 * identifies the MRF/GRF to start reading the message contents from.
226 * Check for some likely failures.
230 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
233 if (devinfo
->gen
>= 12 &&
234 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
235 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
236 assert(reg
.file
!= BRW_IMMEDIATE_VALUE
);
237 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
238 assert(reg
.subnr
== 0);
239 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
240 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
241 reg
.vstride
== reg
.width
+ 1));
242 assert(!reg
.negate
&& !reg
.abs
);
243 brw_inst_set_send_src0_reg_file(devinfo
, inst
, reg
.file
);
244 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
246 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
247 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
248 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
249 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
250 assert(reg
.subnr
% 16 == 0);
251 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
252 reg
.vstride
== reg
.width
+ 1);
253 assert(!reg
.negate
&& !reg
.abs
);
254 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
255 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
257 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
258 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
259 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
260 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
262 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
263 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
264 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
265 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
266 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
267 reg
.type
== BRW_REGISTER_TYPE_Q
)
268 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
270 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
272 if (devinfo
->gen
< 12 && type_sz(reg
.type
) < 8) {
273 brw_inst_set_src1_reg_file(devinfo
, inst
,
274 BRW_ARCHITECTURE_REGISTER_FILE
);
275 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
276 brw_inst_src0_reg_hw_type(devinfo
, inst
));
279 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
280 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
281 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
282 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
284 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
287 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
289 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
290 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
292 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
296 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
297 if (reg
.width
== BRW_WIDTH_1
&&
298 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
299 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
300 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
301 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
303 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
304 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
305 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
308 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
309 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
310 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
311 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
312 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
313 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
314 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
315 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
317 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
318 /* This is an oddity of the fact we're using the same
319 * descriptions for registers in align_16 as align_1:
321 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
322 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
323 reg
.type
== BRW_REGISTER_TYPE_DF
&&
324 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
327 * "For Align16 access mode, only encodings of 0000 and 0011
328 * are allowed. Other codes are reserved."
330 * Presumably the DevSNB behavior applies to IVB as well.
332 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
334 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
343 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
345 const struct gen_device_info
*devinfo
= p
->devinfo
;
347 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
348 assert(reg
.nr
< 128);
350 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
351 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
||
352 (devinfo
->gen
>= 12 &&
353 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
354 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
))) {
355 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
356 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
357 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
358 assert(reg
.subnr
== 0);
359 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
360 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
361 reg
.vstride
== reg
.width
+ 1));
362 assert(!reg
.negate
&& !reg
.abs
);
363 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
364 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
366 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
368 * "Accumulator registers may be accessed explicitly as src0
371 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
372 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
374 gen7_convert_mrf_to_grf(p
, ®
);
375 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
377 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
378 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
379 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
381 /* Only src1 can be immediate in two-argument instructions.
383 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
385 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
386 /* two-argument instructions can only use 32-bit immediates */
387 assert(type_sz(reg
.type
) < 8);
388 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
390 /* This is a hardware restriction, which may or may not be lifted
393 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
394 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
396 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
397 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
398 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
400 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
403 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
404 if (reg
.width
== BRW_WIDTH_1
&&
405 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
406 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
407 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
408 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
410 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
411 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
412 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
415 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
416 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
417 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
418 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
419 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
420 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
421 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
422 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
424 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
425 /* This is an oddity of the fact we're using the same
426 * descriptions for registers in align_16 as align_1:
428 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
429 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
430 reg
.type
== BRW_REGISTER_TYPE_DF
&&
431 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
434 * "For Align16 access mode, only encodings of 0000 and 0011
435 * are allowed. Other codes are reserved."
437 * Presumably the DevSNB behavior applies to IVB as well.
439 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
441 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
449 * Specify the descriptor and extended descriptor immediate for a SEND(C)
450 * message instruction.
453 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
454 unsigned desc
, unsigned ex_desc
)
456 const struct gen_device_info
*devinfo
= p
->devinfo
;
457 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
458 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
459 if (devinfo
->gen
< 12)
460 brw_inst_set_src1_file_type(devinfo
, inst
,
461 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
462 brw_inst_set_send_desc(devinfo
, inst
, desc
);
463 if (devinfo
->gen
>= 9)
464 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
467 static void brw_set_math_message( struct brw_codegen
*p
,
470 unsigned integer_type
,
474 const struct gen_device_info
*devinfo
= p
->devinfo
;
476 unsigned response_length
;
478 /* Infer message length from the function */
480 case BRW_MATH_FUNCTION_POW
:
481 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
482 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
483 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
491 /* Infer response length from the function */
493 case BRW_MATH_FUNCTION_SINCOS
:
494 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
502 brw_set_desc(p
, inst
, brw_message_desc(
503 devinfo
, msg_length
, response_length
, false));
505 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
506 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
507 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
508 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
509 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
510 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
511 brw_inst_set_saturate(devinfo
, inst
, 0);
515 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
518 unsigned response_length
,
521 const struct gen_device_info
*devinfo
= p
->devinfo
;
523 brw_set_desc(p
, insn
, brw_message_desc(
524 devinfo
, 1, response_length
, true));
526 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
527 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
528 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
529 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
530 /* The following fields are not used by FF_SYNC: */
531 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
532 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
533 brw_inst_set_urb_used(devinfo
, insn
, 0);
534 brw_inst_set_urb_complete(devinfo
, insn
, 0);
537 static void brw_set_urb_message( struct brw_codegen
*p
,
539 enum brw_urb_write_flags flags
,
541 unsigned response_length
,
543 unsigned swizzle_control
)
545 const struct gen_device_info
*devinfo
= p
->devinfo
;
547 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
548 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
549 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
551 brw_set_desc(p
, insn
, brw_message_desc(
552 devinfo
, msg_length
, response_length
, true));
554 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
555 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
557 if (flags
& BRW_URB_WRITE_OWORD
) {
558 assert(msg_length
== 2); /* header + one OWORD of data */
559 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
561 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
564 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
565 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
567 if (devinfo
->gen
< 8) {
568 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
571 if (devinfo
->gen
< 7) {
572 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
573 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
575 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
576 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
581 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
585 bool invalidate_after_read
,
587 unsigned addr_offset
,
592 const struct gen_device_info
*devinfo
= p
->devinfo
;
593 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
594 (devinfo
->gen
>= 8 && num_regs
== 8));
595 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
598 brw_set_desc(p
, inst
, brw_message_desc(
599 devinfo
, mlen
, rlen
, header_present
));
601 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
602 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
603 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
604 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
605 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
606 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
607 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
611 brw_inst_set_state(const struct gen_device_info
*devinfo
,
613 const struct brw_insn_state
*state
)
615 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
616 brw_inst_set_group(devinfo
, insn
, state
->group
);
617 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
618 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
619 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
620 if (devinfo
->gen
>= 12)
621 brw_inst_set_swsb(devinfo
, insn
, tgl_swsb_encode(state
->swsb
));
622 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
623 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
624 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
626 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
627 state
->access_mode
== BRW_ALIGN_16
) {
628 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
629 if (devinfo
->gen
>= 7)
630 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
632 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
633 if (devinfo
->gen
>= 7)
634 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
637 if (devinfo
->gen
>= 6)
638 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
641 #define next_insn brw_next_insn
643 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
645 const struct gen_device_info
*devinfo
= p
->devinfo
;
648 if (p
->nr_insn
+ 1 > p
->store_size
) {
650 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
653 p
->next_insn_offset
+= 16;
654 insn
= &p
->store
[p
->nr_insn
++];
656 memset(insn
, 0, sizeof(*insn
));
657 brw_inst_set_opcode(devinfo
, insn
, opcode
);
659 /* Apply the default instruction state */
660 brw_inst_set_state(devinfo
, insn
, p
->current
);
666 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
667 struct brw_reg dest
, struct brw_reg src
)
669 brw_inst
*insn
= next_insn(p
, opcode
);
670 brw_set_dest(p
, insn
, dest
);
671 brw_set_src0(p
, insn
, src
);
676 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
677 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
679 /* 64-bit immediates are only supported on 1-src instructions */
680 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
681 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
683 brw_inst
*insn
= next_insn(p
, opcode
);
684 brw_set_dest(p
, insn
, dest
);
685 brw_set_src0(p
, insn
, src0
);
686 brw_set_src1(p
, insn
, src1
);
691 get_3src_subreg_nr(struct brw_reg reg
)
693 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
694 * use 32-bit units (components 0..7). Since they only support F/D/UD
695 * types, this doesn't lose any flexibility, but uses fewer bits.
697 return reg
.subnr
/ 4;
700 static enum gen10_align1_3src_vertical_stride
701 to_3src_align1_vstride(const struct gen_device_info
*devinfo
,
702 enum brw_vertical_stride vstride
)
705 case BRW_VERTICAL_STRIDE_0
:
706 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
707 case BRW_VERTICAL_STRIDE_1
:
708 assert(devinfo
->gen
>= 12);
709 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1
;
710 case BRW_VERTICAL_STRIDE_2
:
711 assert(devinfo
->gen
< 12);
712 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
713 case BRW_VERTICAL_STRIDE_4
:
714 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
715 case BRW_VERTICAL_STRIDE_8
:
716 case BRW_VERTICAL_STRIDE_16
:
717 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
719 unreachable("invalid vstride");
724 static enum gen10_align1_3src_src_horizontal_stride
725 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
728 case BRW_HORIZONTAL_STRIDE_0
:
729 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
730 case BRW_HORIZONTAL_STRIDE_1
:
731 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
732 case BRW_HORIZONTAL_STRIDE_2
:
733 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
734 case BRW_HORIZONTAL_STRIDE_4
:
735 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
737 unreachable("invalid hstride");
742 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
743 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
745 const struct gen_device_info
*devinfo
= p
->devinfo
;
746 brw_inst
*inst
= next_insn(p
, opcode
);
748 gen7_convert_mrf_to_grf(p
, &dest
);
750 assert(dest
.nr
< 128);
751 assert(src0
.file
== BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
752 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
&& src1
.nr
< 128);
753 assert(src2
.file
== BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
754 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
755 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
756 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
757 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
759 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
760 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
761 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
763 if (devinfo
->gen
>= 12) {
764 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
, dest
.file
);
765 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
767 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
768 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
769 BRW_ALIGN1_3SRC_ACCUMULATOR
);
770 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
772 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
773 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
774 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
777 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
779 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
781 if (brw_reg_type_is_floating_point(dest
.type
)) {
782 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
783 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
785 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
786 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
789 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
790 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
791 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
792 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
794 brw_inst_set_3src_a1_src0_vstride(
795 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src0
.vstride
));
796 brw_inst_set_3src_a1_src1_vstride(
797 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src1
.vstride
));
798 /* no vstride on src2 */
800 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
801 to_3src_align1_hstride(src0
.hstride
));
802 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
803 to_3src_align1_hstride(src1
.hstride
));
804 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
805 to_3src_align1_hstride(src2
.hstride
));
807 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
808 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
809 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
811 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
813 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
814 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
816 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
817 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
818 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
820 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
822 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
823 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
825 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
826 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
827 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
828 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
830 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
831 src0
.file
== BRW_IMMEDIATE_VALUE
||
832 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
833 src0
.type
== BRW_REGISTER_TYPE_NF
));
834 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
835 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
836 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
837 src2
.file
== BRW_IMMEDIATE_VALUE
);
839 if (devinfo
->gen
>= 12) {
840 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
, src0
.file
);
841 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
, src1
.file
);
842 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
, src2
.file
);
844 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
845 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
846 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
847 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
848 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
849 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
850 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
851 BRW_ALIGN1_3SRC_ACCUMULATOR
);
852 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
853 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
854 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
855 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
859 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
860 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
861 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
862 dest
.type
== BRW_REGISTER_TYPE_DF
||
863 dest
.type
== BRW_REGISTER_TYPE_D
||
864 dest
.type
== BRW_REGISTER_TYPE_UD
||
865 (dest
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 8));
866 if (devinfo
->gen
== 6) {
867 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
868 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
870 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
871 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
872 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
874 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
875 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
876 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
877 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
878 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
879 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
880 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
881 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
883 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
884 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
885 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
886 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
887 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
888 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
889 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
890 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
892 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
893 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
894 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
895 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
896 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
897 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
898 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
899 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
901 if (devinfo
->gen
>= 7) {
902 /* Set both the source and destination types based on dest.type,
903 * ignoring the source register types. The MAD and LRP emitters ensure
904 * that all four types are float. The BFE and BFI2 emitters, however,
905 * may send us mixed D and UD types and want us to ignore that and use
906 * the destination type.
908 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
909 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
911 /* From the Bspec, 3D Media GPGPU, Instruction fields, srcType:
913 * "Three source instructions can use operands with mixed-mode
914 * precision. When SrcType field is set to :f or :hf it defines
915 * precision for source 0 only, and fields Src1Type and Src2Type
916 * define precision for other source operands:
918 * 0b = :f. Single precision Float (32-bit).
919 * 1b = :hf. Half precision Float (16-bit)."
921 if (src1
.type
== BRW_REGISTER_TYPE_HF
)
922 brw_inst_set_3src_a16_src1_type(devinfo
, inst
, 1);
924 if (src2
.type
== BRW_REGISTER_TYPE_HF
)
925 brw_inst_set_3src_a16_src2_type(devinfo
, inst
, 1);
933 /***********************************************************************
934 * Convenience routines.
937 brw_inst *brw_##OP(struct brw_codegen *p, \
938 struct brw_reg dest, \
939 struct brw_reg src0) \
941 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
945 brw_inst *brw_##OP(struct brw_codegen *p, \
946 struct brw_reg dest, \
947 struct brw_reg src0, \
948 struct brw_reg src1) \
950 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
954 brw_inst *brw_##OP(struct brw_codegen *p, \
955 struct brw_reg dest, \
956 struct brw_reg src0, \
957 struct brw_reg src1, \
958 struct brw_reg src2) \
960 if (p->current->access_mode == BRW_ALIGN_16) { \
961 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
962 src0.swizzle = BRW_SWIZZLE_XXXX; \
963 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
964 src1.swizzle = BRW_SWIZZLE_XXXX; \
965 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
966 src2.swizzle = BRW_SWIZZLE_XXXX; \
968 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
972 brw_inst *brw_##OP(struct brw_codegen *p, \
973 struct brw_reg dest, \
974 struct brw_reg src0, \
975 struct brw_reg src1, \
976 struct brw_reg src2) \
978 assert(dest.type == BRW_REGISTER_TYPE_F || \
979 dest.type == BRW_REGISTER_TYPE_DF); \
980 if (dest.type == BRW_REGISTER_TYPE_F) { \
981 assert(src0.type == BRW_REGISTER_TYPE_F); \
982 assert(src1.type == BRW_REGISTER_TYPE_F); \
983 assert(src2.type == BRW_REGISTER_TYPE_F); \
984 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
985 assert(src0.type == BRW_REGISTER_TYPE_DF); \
986 assert(src1.type == BRW_REGISTER_TYPE_DF); \
987 assert(src2.type == BRW_REGISTER_TYPE_DF); \
990 if (p->current->access_mode == BRW_ALIGN_16) { \
991 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
992 src0.swizzle = BRW_SWIZZLE_XXXX; \
993 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
994 src1.swizzle = BRW_SWIZZLE_XXXX; \
995 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
996 src2.swizzle = BRW_SWIZZLE_XXXX; \
998 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1001 /* Rounding operations (other than RNDD) require two instructions - the first
1002 * stores a rounded value (possibly the wrong way) in the dest register, but
1003 * also sets a per-channel "increment bit" in the flag register. A predicated
1004 * add of 1.0 fixes dest to contain the desired result.
1006 * Sandybridge and later appear to round correctly without an ADD.
1009 void brw_##OP(struct brw_codegen *p, \
1010 struct brw_reg dest, \
1011 struct brw_reg src) \
1013 const struct gen_device_info *devinfo = p->devinfo; \
1014 brw_inst *rnd, *add; \
1015 rnd = next_insn(p, BRW_OPCODE_##OP); \
1016 brw_set_dest(p, rnd, dest); \
1017 brw_set_src0(p, rnd, src); \
1019 if (devinfo->gen < 6) { \
1020 /* turn on round-increments */ \
1021 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
1022 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1023 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1065 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
1067 const struct gen_device_info
*devinfo
= p
->devinfo
;
1069 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
1070 * To avoid the problems that causes, we use an <X,2,0> source region to
1071 * read each element twice.
1073 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1074 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
1075 dest
.type
== BRW_REGISTER_TYPE_DF
&&
1076 (src0
.type
== BRW_REGISTER_TYPE_F
||
1077 src0
.type
== BRW_REGISTER_TYPE_D
||
1078 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
1079 !has_scalar_region(src0
)) {
1080 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
1081 src0
.vstride
= src0
.hstride
;
1082 src0
.width
= BRW_WIDTH_2
;
1083 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1086 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1090 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1091 struct brw_reg src0
, struct brw_reg src1
)
1094 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1095 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1096 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1097 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1098 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1101 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1102 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1103 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1104 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1105 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1108 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1112 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1113 struct brw_reg src0
, struct brw_reg src1
)
1115 assert(dest
.type
== src0
.type
);
1116 assert(src0
.type
== src1
.type
);
1117 switch (src0
.type
) {
1118 case BRW_REGISTER_TYPE_B
:
1119 case BRW_REGISTER_TYPE_UB
:
1120 case BRW_REGISTER_TYPE_W
:
1121 case BRW_REGISTER_TYPE_UW
:
1122 case BRW_REGISTER_TYPE_D
:
1123 case BRW_REGISTER_TYPE_UD
:
1126 unreachable("Bad type for brw_AVG");
1129 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1133 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1134 struct brw_reg src0
, struct brw_reg src1
)
1137 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1138 src0
.type
== BRW_REGISTER_TYPE_UD
||
1139 src1
.type
== BRW_REGISTER_TYPE_D
||
1140 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1141 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1144 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1145 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1146 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1147 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1148 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1151 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1152 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1153 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1154 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1155 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1158 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1159 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1160 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1161 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1163 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1167 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1168 struct brw_reg src0
, struct brw_reg src1
)
1170 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1171 src0
.width
= BRW_WIDTH_1
;
1172 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1173 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1177 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1178 struct brw_reg src0
, struct brw_reg src1
)
1180 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1181 src0
.width
= BRW_WIDTH_1
;
1182 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1183 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1184 src1
.width
= BRW_WIDTH_8
;
1185 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1186 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1190 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1192 const struct gen_device_info
*devinfo
= p
->devinfo
;
1193 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1194 /* The F32TO16 instruction doesn't support 32-bit destination types in
1195 * Align1 mode, and neither does the Gen8 implementation in terms of a
1196 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1197 * an undocumented feature.
1199 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1200 (!align16
|| devinfo
->gen
>= 8));
1204 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1206 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1207 dst
.type
== BRW_REGISTER_TYPE_W
||
1208 dst
.type
== BRW_REGISTER_TYPE_UW
||
1209 dst
.type
== BRW_REGISTER_TYPE_HF
);
1212 brw_push_insn_state(p
);
1214 if (needs_zero_fill
) {
1215 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1216 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1219 if (devinfo
->gen
>= 8) {
1220 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1222 assert(devinfo
->gen
== 7);
1223 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1226 if (needs_zero_fill
) {
1227 if (devinfo
->gen
< 12)
1228 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1229 brw_set_default_swsb(p
, tgl_swsb_null());
1230 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1231 if (devinfo
->gen
< 12)
1232 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1235 brw_pop_insn_state(p
);
1240 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1242 const struct gen_device_info
*devinfo
= p
->devinfo
;
1243 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1246 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1248 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1250 * Because this instruction does not have a 16-bit floating-point
1251 * type, the source data type must be Word (W). The destination type
1252 * must be F (Float).
1254 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1255 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1257 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1258 src
.type
== BRW_REGISTER_TYPE_UW
||
1259 src
.type
== BRW_REGISTER_TYPE_HF
);
1262 if (devinfo
->gen
>= 8) {
1263 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1265 assert(devinfo
->gen
== 7);
1266 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1271 void brw_NOP(struct brw_codegen
*p
)
1273 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1274 memset(insn
, 0, sizeof(*insn
));
1275 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1278 void brw_SYNC(struct brw_codegen
*p
, enum tgl_sync_function func
)
1280 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SYNC
);
1281 brw_inst_set_cond_modifier(p
->devinfo
, insn
, func
);
1284 /***********************************************************************
1285 * Comparisons, if/else/endif
1289 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1290 unsigned predicate_control
)
1292 const struct gen_device_info
*devinfo
= p
->devinfo
;
1293 struct brw_reg ip
= brw_ip_reg();
1294 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1296 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1297 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1298 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1299 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1305 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1307 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1309 p
->if_stack_depth
++;
1310 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1311 p
->if_stack_array_size
*= 2;
1312 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1313 p
->if_stack_array_size
);
1318 pop_if_stack(struct brw_codegen
*p
)
1320 p
->if_stack_depth
--;
1321 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1325 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1327 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1328 p
->loop_stack_array_size
*= 2;
1329 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1330 p
->loop_stack_array_size
);
1331 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1332 p
->loop_stack_array_size
);
1335 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1336 p
->loop_stack_depth
++;
1337 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1341 get_inner_do_insn(struct brw_codegen
*p
)
1343 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1346 /* EU takes the value from the flag register and pushes it onto some
1347 * sort of a stack (presumably merging with any flag value already on
1348 * the stack). Within an if block, the flags at the top of the stack
1349 * control execution on each channel of the unit, eg. on each of the
1350 * 16 pixel values in our wm programs.
1352 * When the matching 'else' instruction is reached (presumably by
1353 * countdown of the instruction count patched in by our ELSE/ENDIF
1354 * functions), the relevant flags are inverted.
1356 * When the matching 'endif' instruction is reached, the flags are
1357 * popped off. If the stack is now empty, normal execution resumes.
1360 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1362 const struct gen_device_info
*devinfo
= p
->devinfo
;
1365 insn
= next_insn(p
, BRW_OPCODE_IF
);
1367 /* Override the defaults for this instruction:
1369 if (devinfo
->gen
< 6) {
1370 brw_set_dest(p
, insn
, brw_ip_reg());
1371 brw_set_src0(p
, insn
, brw_ip_reg());
1372 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1373 } else if (devinfo
->gen
== 6) {
1374 brw_set_dest(p
, insn
, brw_imm_w(0));
1375 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1376 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1377 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1378 } else if (devinfo
->gen
== 7) {
1379 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1380 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1381 brw_set_src1(p
, insn
, brw_imm_w(0));
1382 brw_inst_set_jip(devinfo
, insn
, 0);
1383 brw_inst_set_uip(devinfo
, insn
, 0);
1385 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1386 if (devinfo
->gen
< 12)
1387 brw_set_src0(p
, insn
, brw_imm_d(0));
1388 brw_inst_set_jip(devinfo
, insn
, 0);
1389 brw_inst_set_uip(devinfo
, insn
, 0);
1392 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1393 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1394 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1395 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1396 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1397 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1399 push_if_stack(p
, insn
);
1400 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1404 /* This function is only used for gen6-style IF instructions with an
1405 * embedded comparison (conditional modifier). It is not used on gen7.
1408 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1409 struct brw_reg src0
, struct brw_reg src1
)
1411 const struct gen_device_info
*devinfo
= p
->devinfo
;
1414 insn
= next_insn(p
, BRW_OPCODE_IF
);
1416 brw_set_dest(p
, insn
, brw_imm_w(0));
1417 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1418 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1419 brw_set_src0(p
, insn
, src0
);
1420 brw_set_src1(p
, insn
, src1
);
1422 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1423 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1424 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1426 push_if_stack(p
, insn
);
1431 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1434 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1435 brw_inst
*if_inst
, brw_inst
*else_inst
)
1437 const struct gen_device_info
*devinfo
= p
->devinfo
;
1439 /* The next instruction (where the ENDIF would be, if it existed) */
1440 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1442 assert(p
->single_program_flow
);
1443 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1444 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1445 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1447 /* Convert IF to an ADD instruction that moves the instruction pointer
1448 * to the first instruction of the ELSE block. If there is no ELSE
1449 * block, point to where ENDIF would be. Reverse the predicate.
1451 * There's no need to execute an ENDIF since we don't need to do any
1452 * stack operations, and if we're currently executing, we just want to
1453 * continue normally.
1455 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1456 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1458 if (else_inst
!= NULL
) {
1459 /* Convert ELSE to an ADD instruction that points where the ENDIF
1462 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1464 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1465 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1467 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1472 * Patch IF and ELSE instructions with appropriate jump targets.
1475 patch_IF_ELSE(struct brw_codegen
*p
,
1476 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1478 const struct gen_device_info
*devinfo
= p
->devinfo
;
1480 /* We shouldn't be patching IF and ELSE instructions in single program flow
1481 * mode when gen < 6, because in single program flow mode on those
1482 * platforms, we convert flow control instructions to conditional ADDs that
1483 * operate on IP (see brw_ENDIF).
1485 * However, on Gen6, writing to IP doesn't work in single program flow mode
1486 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1487 * not be updated by non-flow control instructions."). And on later
1488 * platforms, there is no significant benefit to converting control flow
1489 * instructions to conditional ADDs. So we do patch IF and ELSE
1490 * instructions in single program flow mode on those platforms.
1492 if (devinfo
->gen
< 6)
1493 assert(!p
->single_program_flow
);
1495 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1496 assert(endif_inst
!= NULL
);
1497 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1499 unsigned br
= brw_jump_scale(devinfo
);
1501 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1502 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1504 if (else_inst
== NULL
) {
1505 /* Patch IF -> ENDIF */
1506 if (devinfo
->gen
< 6) {
1507 /* Turn it into an IFF, which means no mask stack operations for
1508 * all-false and jumping past the ENDIF.
1510 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1511 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1512 br
* (endif_inst
- if_inst
+ 1));
1513 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1514 } else if (devinfo
->gen
== 6) {
1515 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1516 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1518 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1519 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1522 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1524 /* Patch IF -> ELSE */
1525 if (devinfo
->gen
< 6) {
1526 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1527 br
* (else_inst
- if_inst
));
1528 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1529 } else if (devinfo
->gen
== 6) {
1530 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1531 br
* (else_inst
- if_inst
+ 1));
1534 /* Patch ELSE -> ENDIF */
1535 if (devinfo
->gen
< 6) {
1536 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1539 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1540 br
* (endif_inst
- else_inst
+ 1));
1541 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1542 } else if (devinfo
->gen
== 6) {
1543 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1544 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1545 br
* (endif_inst
- else_inst
));
1547 /* The IF instruction's JIP should point just past the ELSE */
1548 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1549 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1550 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1551 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1552 if (devinfo
->gen
>= 8) {
1553 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1554 * should point to ENDIF.
1556 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1563 brw_ELSE(struct brw_codegen
*p
)
1565 const struct gen_device_info
*devinfo
= p
->devinfo
;
1568 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1570 if (devinfo
->gen
< 6) {
1571 brw_set_dest(p
, insn
, brw_ip_reg());
1572 brw_set_src0(p
, insn
, brw_ip_reg());
1573 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1574 } else if (devinfo
->gen
== 6) {
1575 brw_set_dest(p
, insn
, brw_imm_w(0));
1576 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1577 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1578 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1579 } else if (devinfo
->gen
== 7) {
1580 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1581 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1582 brw_set_src1(p
, insn
, brw_imm_w(0));
1583 brw_inst_set_jip(devinfo
, insn
, 0);
1584 brw_inst_set_uip(devinfo
, insn
, 0);
1586 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1587 if (devinfo
->gen
< 12)
1588 brw_set_src0(p
, insn
, brw_imm_d(0));
1589 brw_inst_set_jip(devinfo
, insn
, 0);
1590 brw_inst_set_uip(devinfo
, insn
, 0);
1593 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1594 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1595 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1596 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1598 push_if_stack(p
, insn
);
1602 brw_ENDIF(struct brw_codegen
*p
)
1604 const struct gen_device_info
*devinfo
= p
->devinfo
;
1605 brw_inst
*insn
= NULL
;
1606 brw_inst
*else_inst
= NULL
;
1607 brw_inst
*if_inst
= NULL
;
1609 bool emit_endif
= true;
1611 /* In single program flow mode, we can express IF and ELSE instructions
1612 * equivalently as ADD instructions that operate on IP. On platforms prior
1613 * to Gen6, flow control instructions cause an implied thread switch, so
1614 * this is a significant savings.
1616 * However, on Gen6, writing to IP doesn't work in single program flow mode
1617 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1618 * not be updated by non-flow control instructions."). And on later
1619 * platforms, there is no significant benefit to converting control flow
1620 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1623 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1627 * A single next_insn() may change the base address of instruction store
1628 * memory(p->store), so call it first before referencing the instruction
1629 * store pointer from an index
1632 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1634 /* Pop the IF and (optional) ELSE instructions from the stack */
1635 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1636 tmp
= pop_if_stack(p
);
1637 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1639 tmp
= pop_if_stack(p
);
1644 /* ENDIF is useless; don't bother emitting it. */
1645 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1649 if (devinfo
->gen
< 6) {
1650 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1651 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1652 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1653 } else if (devinfo
->gen
== 6) {
1654 brw_set_dest(p
, insn
, brw_imm_w(0));
1655 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1656 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1657 } else if (devinfo
->gen
== 7) {
1658 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1659 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1660 brw_set_src1(p
, insn
, brw_imm_w(0));
1662 brw_set_src0(p
, insn
, brw_imm_d(0));
1665 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1666 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1667 if (devinfo
->gen
< 6)
1668 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1670 /* Also pop item off the stack in the endif instruction: */
1671 if (devinfo
->gen
< 6) {
1672 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1673 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1674 } else if (devinfo
->gen
== 6) {
1675 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1677 brw_inst_set_jip(devinfo
, insn
, 2);
1679 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1683 brw_BREAK(struct brw_codegen
*p
)
1685 const struct gen_device_info
*devinfo
= p
->devinfo
;
1688 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1689 if (devinfo
->gen
>= 8) {
1690 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1691 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1692 } else if (devinfo
->gen
>= 6) {
1693 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1694 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1695 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1697 brw_set_dest(p
, insn
, brw_ip_reg());
1698 brw_set_src0(p
, insn
, brw_ip_reg());
1699 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1700 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1701 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1703 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1704 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1710 brw_CONT(struct brw_codegen
*p
)
1712 const struct gen_device_info
*devinfo
= p
->devinfo
;
1715 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1716 brw_set_dest(p
, insn
, brw_ip_reg());
1717 if (devinfo
->gen
>= 8) {
1718 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1720 brw_set_src0(p
, insn
, brw_ip_reg());
1721 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1724 if (devinfo
->gen
< 6) {
1725 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1726 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1728 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1729 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1734 gen6_HALT(struct brw_codegen
*p
)
1736 const struct gen_device_info
*devinfo
= p
->devinfo
;
1739 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1740 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1741 if (devinfo
->gen
< 8) {
1742 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1743 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1744 } else if (devinfo
->gen
< 12) {
1745 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1748 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1749 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1755 * The DO/WHILE is just an unterminated loop -- break or continue are
1756 * used for control within the loop. We have a few ways they can be
1759 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1760 * jip and no DO instruction.
1762 * For non-uniform control flow pre-gen6, there's a DO instruction to
1763 * push the mask, and a WHILE to jump back, and BREAK to get out and
1766 * For gen6, there's no more mask stack, so no need for DO. WHILE
1767 * just points back to the first instruction of the loop.
1770 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1772 const struct gen_device_info
*devinfo
= p
->devinfo
;
1774 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1775 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1776 return &p
->store
[p
->nr_insn
];
1778 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1780 push_loop_stack(p
, insn
);
1782 /* Override the defaults for this instruction:
1784 brw_set_dest(p
, insn
, brw_null_reg());
1785 brw_set_src0(p
, insn
, brw_null_reg());
1786 brw_set_src1(p
, insn
, brw_null_reg());
1788 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1789 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1790 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1797 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1800 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1801 * nesting, since it can always just point to the end of the block/current loop.
1804 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1806 const struct gen_device_info
*devinfo
= p
->devinfo
;
1807 brw_inst
*do_inst
= get_inner_do_insn(p
);
1809 unsigned br
= brw_jump_scale(devinfo
);
1811 assert(devinfo
->gen
< 6);
1813 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1814 /* If the jump count is != 0, that means that this instruction has already
1815 * been patched because it's part of a loop inside of the one we're
1818 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1819 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1820 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1821 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1822 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1823 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1829 brw_WHILE(struct brw_codegen
*p
)
1831 const struct gen_device_info
*devinfo
= p
->devinfo
;
1832 brw_inst
*insn
, *do_insn
;
1833 unsigned br
= brw_jump_scale(devinfo
);
1835 if (devinfo
->gen
>= 6) {
1836 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1837 do_insn
= get_inner_do_insn(p
);
1839 if (devinfo
->gen
>= 8) {
1840 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1841 if (devinfo
->gen
< 12)
1842 brw_set_src0(p
, insn
, brw_imm_d(0));
1843 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1844 } else if (devinfo
->gen
== 7) {
1845 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1846 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1847 brw_set_src1(p
, insn
, brw_imm_w(0));
1848 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1850 brw_set_dest(p
, insn
, brw_imm_w(0));
1851 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1852 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1853 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1856 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1859 if (p
->single_program_flow
) {
1860 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1861 do_insn
= get_inner_do_insn(p
);
1863 brw_set_dest(p
, insn
, brw_ip_reg());
1864 brw_set_src0(p
, insn
, brw_ip_reg());
1865 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1866 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1868 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1869 do_insn
= get_inner_do_insn(p
);
1871 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1873 brw_set_dest(p
, insn
, brw_ip_reg());
1874 brw_set_src0(p
, insn
, brw_ip_reg());
1875 brw_set_src1(p
, insn
, brw_imm_d(0));
1877 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1878 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1879 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1881 brw_patch_break_cont(p
, insn
);
1884 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1886 p
->loop_stack_depth
--;
1893 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1895 const struct gen_device_info
*devinfo
= p
->devinfo
;
1896 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1899 if (devinfo
->gen
>= 5)
1902 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1903 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1905 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1906 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1909 /* To integrate with the above, it makes sense that the comparison
1910 * instruction should populate the flag register. It might be simpler
1911 * just to use the flag reg for most WM tasks?
1913 void brw_CMP(struct brw_codegen
*p
,
1914 struct brw_reg dest
,
1915 unsigned conditional
,
1916 struct brw_reg src0
,
1917 struct brw_reg src1
)
1919 const struct gen_device_info
*devinfo
= p
->devinfo
;
1920 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1922 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1923 brw_set_dest(p
, insn
, dest
);
1924 brw_set_src0(p
, insn
, src0
);
1925 brw_set_src1(p
, insn
, src1
);
1927 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1929 * "Any CMP instruction with a null destination must use a {switch}."
1931 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1932 * mentioned on their work-arounds pages.
1934 if (devinfo
->gen
== 7) {
1935 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1936 dest
.nr
== BRW_ARF_NULL
) {
1937 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1942 /***********************************************************************
1943 * Helpers for the various SEND message types:
1946 /** Extended math function, float[8].
1948 void gen4_math(struct brw_codegen
*p
,
1949 struct brw_reg dest
,
1951 unsigned msg_reg_nr
,
1953 unsigned precision
)
1955 const struct gen_device_info
*devinfo
= p
->devinfo
;
1956 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1958 if (has_scalar_region(src
)) {
1959 data_type
= BRW_MATH_DATA_SCALAR
;
1961 data_type
= BRW_MATH_DATA_VECTOR
;
1964 assert(devinfo
->gen
< 6);
1966 /* Example code doesn't set predicate_control for send
1969 brw_inst_set_pred_control(devinfo
, insn
, 0);
1970 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1972 brw_set_dest(p
, insn
, dest
);
1973 brw_set_src0(p
, insn
, src
);
1974 brw_set_math_message(p
,
1977 src
.type
== BRW_REGISTER_TYPE_D
,
1982 void gen6_math(struct brw_codegen
*p
,
1983 struct brw_reg dest
,
1985 struct brw_reg src0
,
1986 struct brw_reg src1
)
1988 const struct gen_device_info
*devinfo
= p
->devinfo
;
1989 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1991 assert(devinfo
->gen
>= 6);
1993 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1994 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1996 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1997 if (devinfo
->gen
== 6) {
1998 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1999 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2002 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2003 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2004 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2005 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2006 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2007 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2008 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2010 assert(src0
.type
== BRW_REGISTER_TYPE_F
||
2011 (src0
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2012 assert(src1
.type
== BRW_REGISTER_TYPE_F
||
2013 (src1
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2016 /* Source modifiers are ignored for extended math instructions on Gen6. */
2017 if (devinfo
->gen
== 6) {
2018 assert(!src0
.negate
);
2020 assert(!src1
.negate
);
2024 brw_inst_set_math_function(devinfo
, insn
, function
);
2026 brw_set_dest(p
, insn
, dest
);
2027 brw_set_src0(p
, insn
, src0
);
2028 brw_set_src1(p
, insn
, src1
);
2032 * Return the right surface index to access the thread scratch space using
2033 * stateless dataport messages.
2036 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2038 /* The scratch space is thread-local so IA coherency is unnecessary. */
2039 if (p
->devinfo
->gen
>= 8)
2040 return GEN8_BTI_STATELESS_NON_COHERENT
;
2042 return BRW_BTI_STATELESS
;
2046 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2047 * using a constant offset per channel.
2049 * The offset must be aligned to oword size (16 bytes). Used for
2050 * register spilling.
2052 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2057 const struct gen_device_info
*devinfo
= p
->devinfo
;
2058 const unsigned target_cache
=
2059 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2060 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2061 BRW_SFID_DATAPORT_WRITE
);
2062 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2065 if (devinfo
->gen
>= 6)
2068 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2070 const unsigned mlen
= 1 + num_regs
;
2072 /* Set up the message header. This is g0, with g0.2 filled with
2073 * the offset. We don't want to leave our offset around in g0 or
2074 * it'll screw up texture samples, so set it up inside the message
2078 brw_push_insn_state(p
);
2079 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2080 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2081 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2082 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2084 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2086 /* set message header global offset field (reg 0, element 2) */
2087 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2088 brw_set_default_swsb(p
, tgl_swsb_null());
2090 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2092 2), BRW_REGISTER_TYPE_UD
),
2093 brw_imm_ud(offset
));
2095 brw_pop_insn_state(p
);
2096 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2100 struct brw_reg dest
;
2101 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2102 int send_commit_msg
;
2103 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2104 BRW_REGISTER_TYPE_UW
);
2106 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2107 brw_inst_set_compression(devinfo
, insn
, false);
2109 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2110 src_header
= vec16(src_header
);
2112 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2113 if (devinfo
->gen
< 6)
2114 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2116 /* Until gen6, writes followed by reads from the same location
2117 * are not guaranteed to be ordered unless write_commit is set.
2118 * If set, then a no-op write is issued to the destination
2119 * register to set a dependency, and a read from the destination
2120 * can be used to ensure the ordering.
2122 * For gen6, only writes between different threads need ordering
2123 * protection. Our use of DP writes is all about register
2124 * spilling within a thread.
2126 if (devinfo
->gen
>= 6) {
2127 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2128 send_commit_msg
= 0;
2131 send_commit_msg
= 1;
2134 brw_set_dest(p
, insn
, dest
);
2135 if (devinfo
->gen
>= 6) {
2136 brw_set_src0(p
, insn
, mrf
);
2138 brw_set_src0(p
, insn
, brw_null_reg());
2141 if (devinfo
->gen
>= 6)
2142 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2144 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2146 brw_set_desc(p
, insn
,
2147 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2148 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2149 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2150 msg_type
, 0, /* not a render target */
2157 * Read a block of owords (half a GRF each) from the scratch buffer
2158 * using a constant index per channel.
2160 * Offset must be aligned to oword size (16 bytes). Used for register
2164 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2165 struct brw_reg dest
,
2170 const struct gen_device_info
*devinfo
= p
->devinfo
;
2171 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2173 if (devinfo
->gen
>= 6)
2176 if (p
->devinfo
->gen
>= 7) {
2177 /* On gen 7 and above, we no longer have message registers and we can
2178 * send from any register we want. By using the destination register
2179 * for the message, we guarantee that the implied message write won't
2180 * accidentally overwrite anything. This has been a problem because
2181 * the MRF registers and source for the final FB write are both fixed
2184 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2186 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2188 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2190 const unsigned rlen
= num_regs
;
2191 const unsigned target_cache
=
2192 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2193 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2194 BRW_SFID_DATAPORT_READ
);
2197 brw_push_insn_state(p
);
2198 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2199 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2200 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2201 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2203 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2205 /* set message header global offset field (reg 0, element 2) */
2206 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2207 brw_set_default_swsb(p
, tgl_swsb_null());
2208 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2210 brw_pop_insn_state(p
);
2211 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2215 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2217 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2218 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2219 brw_inst_set_compression(devinfo
, insn
, false);
2221 brw_set_dest(p
, insn
, dest
); /* UW? */
2222 if (devinfo
->gen
>= 6) {
2223 brw_set_src0(p
, insn
, mrf
);
2225 brw_set_src0(p
, insn
, brw_null_reg());
2226 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2229 brw_set_desc(p
, insn
,
2230 brw_message_desc(devinfo
, 1, rlen
, true) |
2231 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2232 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2233 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2234 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2239 gen7_block_read_scratch(struct brw_codegen
*p
,
2240 struct brw_reg dest
,
2244 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2245 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2247 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2249 /* The HW requires that the header is present; this is to get the g0.5
2252 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2254 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2255 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2256 * is 32 bytes, which happens to be the size of a register.
2259 assert(offset
< (1 << 12));
2261 gen7_set_dp_scratch_message(p
, insn
,
2262 false, /* scratch read */
2264 false, /* invalidate after read */
2267 1, /* mlen: just g0 */
2268 num_regs
, /* rlen */
2269 true); /* header present */
2273 * Read float[4] vectors from the data port constant cache.
2274 * Location (in buffer) should be a multiple of 16.
2275 * Used for fetching shader constants.
2277 void brw_oword_block_read(struct brw_codegen
*p
,
2278 struct brw_reg dest
,
2281 uint32_t bind_table_index
)
2283 const struct gen_device_info
*devinfo
= p
->devinfo
;
2284 const unsigned target_cache
=
2285 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2286 BRW_SFID_DATAPORT_READ
);
2287 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2288 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2290 /* On newer hardware, offset is in units of owords. */
2291 if (devinfo
->gen
>= 6)
2294 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2296 brw_push_insn_state(p
);
2297 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2298 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2299 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2301 brw_push_insn_state(p
);
2302 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2303 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2304 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2306 /* set message header global offset field (reg 0, element 2) */
2307 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2308 brw_set_default_swsb(p
, tgl_swsb_null());
2310 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2312 2), BRW_REGISTER_TYPE_UD
),
2313 brw_imm_ud(offset
));
2314 brw_pop_insn_state(p
);
2316 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2318 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2320 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2322 /* cast dest to a uword[8] vector */
2323 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2325 brw_set_dest(p
, insn
, dest
);
2326 if (devinfo
->gen
>= 6) {
2327 brw_set_src0(p
, insn
, mrf
);
2329 brw_set_src0(p
, insn
, brw_null_reg());
2330 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2333 brw_set_desc(p
, insn
,
2334 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2335 brw_dp_read_desc(devinfo
, bind_table_index
,
2336 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2337 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2338 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2340 brw_pop_insn_state(p
);
2344 brw_fb_WRITE(struct brw_codegen
*p
,
2345 struct brw_reg payload
,
2346 struct brw_reg implied_header
,
2347 unsigned msg_control
,
2348 unsigned binding_table_index
,
2349 unsigned msg_length
,
2350 unsigned response_length
,
2352 bool last_render_target
,
2353 bool header_present
)
2355 const struct gen_device_info
*devinfo
= p
->devinfo
;
2356 const unsigned target_cache
=
2357 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2358 BRW_SFID_DATAPORT_WRITE
);
2361 struct brw_reg dest
, src0
;
2363 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2364 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2366 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2368 if (devinfo
->gen
>= 6) {
2369 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2371 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2373 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2374 brw_inst_set_compression(devinfo
, insn
, false);
2376 if (devinfo
->gen
>= 6) {
2377 /* headerless version, just submit color payload */
2380 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2382 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2383 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2384 src0
= implied_header
;
2386 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2389 brw_set_dest(p
, insn
, dest
);
2390 brw_set_src0(p
, insn
, src0
);
2391 brw_set_desc(p
, insn
,
2392 brw_message_desc(devinfo
, msg_length
, response_length
,
2394 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2395 msg_type
, last_render_target
,
2396 0 /* send_commit_msg */));
2397 brw_inst_set_eot(devinfo
, insn
, eot
);
2403 gen9_fb_READ(struct brw_codegen
*p
,
2405 struct brw_reg payload
,
2406 unsigned binding_table_index
,
2407 unsigned msg_length
,
2408 unsigned response_length
,
2411 const struct gen_device_info
*devinfo
= p
->devinfo
;
2412 assert(devinfo
->gen
>= 9);
2413 const unsigned msg_subtype
=
2414 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2415 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2417 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2418 brw_set_dest(p
, insn
, dst
);
2419 brw_set_src0(p
, insn
, payload
);
2422 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2423 brw_dp_read_desc(devinfo
, binding_table_index
,
2424 per_sample
<< 5 | msg_subtype
,
2425 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2426 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2427 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2433 * Texture sample instruction.
2434 * Note: the msg_type plus msg_length values determine exactly what kind
2435 * of sampling operation is performed. See volume 4, page 161 of docs.
2437 void brw_SAMPLE(struct brw_codegen
*p
,
2438 struct brw_reg dest
,
2439 unsigned msg_reg_nr
,
2440 struct brw_reg src0
,
2441 unsigned binding_table_index
,
2444 unsigned response_length
,
2445 unsigned msg_length
,
2446 unsigned header_present
,
2448 unsigned return_format
)
2450 const struct gen_device_info
*devinfo
= p
->devinfo
;
2453 if (msg_reg_nr
!= -1)
2454 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2456 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2457 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2458 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2460 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2462 * "Instruction compression is not allowed for this instruction (that
2463 * is, send). The hardware behavior is undefined if this instruction is
2464 * set as compressed. However, compress control can be set to "SecHalf"
2465 * to affect the EMask generation."
2467 * No similar wording is found in later PRMs, but there are examples
2468 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2469 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2470 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2472 brw_inst_set_compression(devinfo
, insn
, false);
2474 if (devinfo
->gen
< 6)
2475 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2477 brw_set_dest(p
, insn
, dest
);
2478 brw_set_src0(p
, insn
, src0
);
2479 brw_set_desc(p
, insn
,
2480 brw_message_desc(devinfo
, msg_length
, response_length
,
2482 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2483 msg_type
, simd_mode
, return_format
));
2486 /* Adjust the message header's sampler state pointer to
2487 * select the correct group of 16 samplers.
2489 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2490 struct brw_reg header
,
2491 struct brw_reg sampler_index
)
2493 /* The "Sampler Index" field can only store values between 0 and 15.
2494 * However, we can add an offset to the "Sampler State Pointer"
2495 * field, effectively selecting a different set of 16 samplers.
2497 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2498 * offset, and each sampler state is only 16-bytes, so we can't
2499 * exclusively use the offset - we have to use both.
2502 const struct gen_device_info
*devinfo
= p
->devinfo
;
2504 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2505 const int sampler_state_size
= 16; /* 16 bytes */
2506 uint32_t sampler
= sampler_index
.ud
;
2508 if (sampler
>= 16) {
2509 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2511 get_element_ud(header
, 3),
2512 get_element_ud(brw_vec8_grf(0, 0), 3),
2513 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2516 /* Non-const sampler array indexing case */
2517 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2521 struct brw_reg temp
= get_element_ud(header
, 3);
2523 brw_push_insn_state(p
);
2524 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2525 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
2526 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2528 get_element_ud(header
, 3),
2529 get_element_ud(brw_vec8_grf(0, 0), 3),
2531 brw_pop_insn_state(p
);
2535 /* All these variables are pretty confusing - we might be better off
2536 * using bitmasks and macros for this, in the old style. Or perhaps
2537 * just having the caller instantiate the fields in dword3 itself.
2539 void brw_urb_WRITE(struct brw_codegen
*p
,
2540 struct brw_reg dest
,
2541 unsigned msg_reg_nr
,
2542 struct brw_reg src0
,
2543 enum brw_urb_write_flags flags
,
2544 unsigned msg_length
,
2545 unsigned response_length
,
2549 const struct gen_device_info
*devinfo
= p
->devinfo
;
2552 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2554 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2555 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2556 brw_push_insn_state(p
);
2557 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2558 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2559 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2560 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2561 BRW_REGISTER_TYPE_UD
),
2562 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2563 brw_imm_ud(0xff00));
2564 brw_pop_insn_state(p
);
2567 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2569 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2571 brw_set_dest(p
, insn
, dest
);
2572 brw_set_src0(p
, insn
, src0
);
2573 brw_set_src1(p
, insn
, brw_imm_d(0));
2575 if (devinfo
->gen
< 6)
2576 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2578 brw_set_urb_message(p
,
2588 brw_send_indirect_message(struct brw_codegen
*p
,
2591 struct brw_reg payload
,
2592 struct brw_reg desc
,
2596 const struct gen_device_info
*devinfo
= p
->devinfo
;
2597 struct brw_inst
*send
;
2599 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2601 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2603 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2604 send
= next_insn(p
, BRW_OPCODE_SEND
);
2605 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2606 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2608 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2609 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2611 brw_push_insn_state(p
);
2612 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2613 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2614 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2615 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2616 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2618 /* Load the indirect descriptor to an address register using OR so the
2619 * caller can specify additional descriptor bits with the desc_imm
2622 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2624 brw_pop_insn_state(p
);
2626 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2627 send
= next_insn(p
, BRW_OPCODE_SEND
);
2628 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2630 if (devinfo
->gen
>= 12)
2631 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, true);
2633 brw_set_src1(p
, send
, addr
);
2636 brw_set_dest(p
, send
, dst
);
2637 brw_inst_set_sfid(devinfo
, send
, sfid
);
2638 brw_inst_set_eot(devinfo
, send
, eot
);
2642 brw_send_indirect_split_message(struct brw_codegen
*p
,
2645 struct brw_reg payload0
,
2646 struct brw_reg payload1
,
2647 struct brw_reg desc
,
2649 struct brw_reg ex_desc
,
2650 unsigned ex_desc_imm
,
2653 const struct gen_device_info
*devinfo
= p
->devinfo
;
2654 struct brw_inst
*send
;
2656 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2658 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2660 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2661 desc
.ud
|= desc_imm
;
2663 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2664 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2666 brw_push_insn_state(p
);
2667 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2668 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2669 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2670 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2671 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2673 /* Load the indirect descriptor to an address register using OR so the
2674 * caller can specify additional descriptor bits with the desc_imm
2677 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2679 brw_pop_insn_state(p
);
2682 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2685 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
&&
2686 (ex_desc
.ud
& INTEL_MASK(15, 12)) == 0) {
2687 ex_desc
.ud
|= ex_desc_imm
;
2689 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2690 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2692 brw_push_insn_state(p
);
2693 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2694 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2695 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2696 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2697 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2699 /* Load the indirect extended descriptor to an address register using OR
2700 * so the caller can specify additional descriptor bits with the
2701 * desc_imm immediate.
2703 * Even though the instruction dispatcher always pulls the SFID and EOT
2704 * fields from the instruction itself, actual external unit which
2705 * processes the message gets the SFID and EOT from the extended
2706 * descriptor which comes from the address register. If we don't OR
2707 * those two bits in, the external unit may get confused and hang.
2709 unsigned imm_part
= ex_desc_imm
| sfid
| eot
<< 5;
2711 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2712 /* ex_desc bits 15:12 don't exist in the instruction encoding, so
2713 * we may have fallen back to an indirect extended descriptor.
2715 brw_MOV(p
, addr
, brw_imm_ud(ex_desc
.ud
| imm_part
));
2717 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(imm_part
));
2720 brw_pop_insn_state(p
);
2723 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2726 send
= next_insn(p
, devinfo
->gen
>= 12 ? BRW_OPCODE_SEND
: BRW_OPCODE_SENDS
);
2727 brw_set_dest(p
, send
, dst
);
2728 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2729 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2731 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2732 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2733 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2735 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2736 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2737 assert(desc
.subnr
== 0);
2738 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2741 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2742 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2743 brw_inst_set_sends_ex_desc(devinfo
, send
, ex_desc
.ud
);
2745 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2746 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2747 assert((ex_desc
.subnr
& 0x3) == 0);
2748 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2749 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2752 brw_inst_set_sfid(devinfo
, send
, sfid
);
2753 brw_inst_set_eot(devinfo
, send
, eot
);
2757 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2760 struct brw_reg payload
,
2761 struct brw_reg surface
,
2764 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2765 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2766 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2768 brw_push_insn_state(p
);
2769 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2770 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2771 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2772 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2773 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2775 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2776 * some surface array is accessed out of bounds.
2779 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2780 BRW_GET_SWZ(surface
.swizzle
, 0)),
2783 brw_pop_insn_state(p
);
2786 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2789 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2793 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2794 brw_inst
*insn
, int while_offset
, int start_offset
)
2796 int scale
= 16 / brw_jump_scale(devinfo
);
2797 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2798 : brw_inst_jip(devinfo
, insn
);
2800 return while_offset
+ jip
* scale
<= start_offset
;
2805 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2808 void *store
= p
->store
;
2809 const struct gen_device_info
*devinfo
= p
->devinfo
;
2813 for (offset
= next_offset(devinfo
, store
, start_offset
);
2814 offset
< p
->next_insn_offset
;
2815 offset
= next_offset(devinfo
, store
, offset
)) {
2816 brw_inst
*insn
= store
+ offset
;
2818 switch (brw_inst_opcode(devinfo
, insn
)) {
2822 case BRW_OPCODE_ENDIF
:
2827 case BRW_OPCODE_WHILE
:
2828 /* If the while doesn't jump before our instruction, it's the end
2829 * of a sibling do...while loop. Ignore it.
2831 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2834 case BRW_OPCODE_ELSE
:
2835 case BRW_OPCODE_HALT
:
2846 /* There is no DO instruction on gen6, so to find the end of the loop
2847 * we have to see if the loop is jumping back before our start
2851 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2853 const struct gen_device_info
*devinfo
= p
->devinfo
;
2855 void *store
= p
->store
;
2857 assert(devinfo
->gen
>= 6);
2859 /* Always start after the instruction (such as a WHILE) we're trying to fix
2862 for (offset
= next_offset(devinfo
, store
, start_offset
);
2863 offset
< p
->next_insn_offset
;
2864 offset
= next_offset(devinfo
, store
, offset
)) {
2865 brw_inst
*insn
= store
+ offset
;
2867 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2868 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2872 assert(!"not reached");
2873 return start_offset
;
2876 /* After program generation, go back and update the UIP and JIP of
2877 * BREAK, CONT, and HALT instructions to their correct locations.
2880 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2882 const struct gen_device_info
*devinfo
= p
->devinfo
;
2884 int br
= brw_jump_scale(devinfo
);
2885 int scale
= 16 / br
;
2886 void *store
= p
->store
;
2888 if (devinfo
->gen
< 6)
2891 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2892 brw_inst
*insn
= store
+ offset
;
2893 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2895 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2896 switch (brw_inst_opcode(devinfo
, insn
)) {
2897 case BRW_OPCODE_BREAK
:
2898 assert(block_end_offset
!= 0);
2899 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2900 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2901 brw_inst_set_uip(devinfo
, insn
,
2902 (brw_find_loop_end(p
, offset
) - offset
+
2903 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2905 case BRW_OPCODE_CONTINUE
:
2906 assert(block_end_offset
!= 0);
2907 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2908 brw_inst_set_uip(devinfo
, insn
,
2909 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2911 assert(brw_inst_uip(devinfo
, insn
) != 0);
2912 assert(brw_inst_jip(devinfo
, insn
) != 0);
2915 case BRW_OPCODE_ENDIF
: {
2916 int32_t jump
= (block_end_offset
== 0) ?
2917 1 * br
: (block_end_offset
- offset
) / scale
;
2918 if (devinfo
->gen
>= 7)
2919 brw_inst_set_jip(devinfo
, insn
, jump
);
2921 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2925 case BRW_OPCODE_HALT
:
2926 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2928 * "In case of the halt instruction not inside any conditional
2929 * code block, the value of <JIP> and <UIP> should be the
2930 * same. In case of the halt instruction inside conditional code
2931 * block, the <UIP> should be the end of the program, and the
2932 * <JIP> should be end of the most inner conditional code block."
2934 * The uip will have already been set by whoever set up the
2937 if (block_end_offset
== 0) {
2938 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2940 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2942 assert(brw_inst_uip(devinfo
, insn
) != 0);
2943 assert(brw_inst_jip(devinfo
, insn
) != 0);
2952 void brw_ff_sync(struct brw_codegen
*p
,
2953 struct brw_reg dest
,
2954 unsigned msg_reg_nr
,
2955 struct brw_reg src0
,
2957 unsigned response_length
,
2960 const struct gen_device_info
*devinfo
= p
->devinfo
;
2963 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2965 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2966 brw_set_dest(p
, insn
, dest
);
2967 brw_set_src0(p
, insn
, src0
);
2968 brw_set_src1(p
, insn
, brw_imm_d(0));
2970 if (devinfo
->gen
< 6)
2971 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2973 brw_set_ff_sync_message(p
,
2981 * Emit the SEND instruction necessary to generate stream output data on Gen6
2982 * (for transform feedback).
2984 * If send_commit_msg is true, this is the last piece of stream output data
2985 * from this thread, so send the data as a committed write. According to the
2986 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2988 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2989 * writes are complete by sending the final write as a committed write."
2992 brw_svb_write(struct brw_codegen
*p
,
2993 struct brw_reg dest
,
2994 unsigned msg_reg_nr
,
2995 struct brw_reg src0
,
2996 unsigned binding_table_index
,
2997 bool send_commit_msg
)
2999 const struct gen_device_info
*devinfo
= p
->devinfo
;
3000 const unsigned target_cache
=
3001 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
3002 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
3003 BRW_SFID_DATAPORT_WRITE
);
3006 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
3008 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3009 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
3010 brw_set_dest(p
, insn
, dest
);
3011 brw_set_src0(p
, insn
, src0
);
3012 brw_set_desc(p
, insn
,
3013 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
3014 brw_dp_write_desc(devinfo
, binding_table_index
,
3015 0, /* msg_control: ignored */
3016 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
3017 0, /* last_render_target: ignored */
3018 send_commit_msg
)); /* send_commit_msg */
3022 brw_surface_payload_size(struct brw_codegen
*p
,
3023 unsigned num_channels
,
3024 unsigned exec_size
/**< 0 for SIMD4x2 */)
3027 return 1; /* SIMD4x2 */
3028 else if (exec_size
<= 8)
3029 return num_channels
;
3031 return 2 * num_channels
;
3035 brw_untyped_atomic(struct brw_codegen
*p
,
3037 struct brw_reg payload
,
3038 struct brw_reg surface
,
3040 unsigned msg_length
,
3041 bool response_expected
,
3042 bool header_present
)
3044 const struct gen_device_info
*devinfo
= p
->devinfo
;
3045 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3046 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3047 GEN7_SFID_DATAPORT_DATA_CACHE
);
3048 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3049 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
3050 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3051 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3052 has_simd4x2
? 0 : 8;
3053 const unsigned response_length
=
3054 brw_surface_payload_size(p
, response_expected
, exec_size
);
3055 const unsigned desc
=
3056 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3057 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
3059 /* Mask out unused components -- This is especially important in Align16
3060 * mode on generations that don't have native support for SIMD4x2 atomics,
3061 * because unused but enabled components will cause the dataport to perform
3062 * additional atomic operations on the addresses that happen to be in the
3063 * uninitialized Y, Z and W coordinates of the payload.
3065 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3067 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
3068 payload
, surface
, desc
);
3072 brw_untyped_surface_read(struct brw_codegen
*p
,
3074 struct brw_reg payload
,
3075 struct brw_reg surface
,
3076 unsigned msg_length
,
3077 unsigned num_channels
)
3079 const struct gen_device_info
*devinfo
= p
->devinfo
;
3080 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3081 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3082 GEN7_SFID_DATAPORT_DATA_CACHE
);
3083 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3084 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
3085 const unsigned response_length
=
3086 brw_surface_payload_size(p
, num_channels
, exec_size
);
3087 const unsigned desc
=
3088 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3089 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
3091 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
3095 brw_untyped_surface_write(struct brw_codegen
*p
,
3096 struct brw_reg payload
,
3097 struct brw_reg surface
,
3098 unsigned msg_length
,
3099 unsigned num_channels
,
3100 bool header_present
)
3102 const struct gen_device_info
*devinfo
= p
->devinfo
;
3103 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3104 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3105 GEN7_SFID_DATAPORT_DATA_CACHE
);
3106 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3107 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
3108 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3109 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3110 has_simd4x2
? 0 : 8;
3111 const unsigned desc
=
3112 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3113 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
3114 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3115 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
3117 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3118 payload
, surface
, desc
);
3122 brw_set_memory_fence_message(struct brw_codegen
*p
,
3123 struct brw_inst
*insn
,
3124 enum brw_message_target sfid
,
3128 const struct gen_device_info
*devinfo
= p
->devinfo
;
3130 brw_set_desc(p
, insn
, brw_message_desc(
3131 devinfo
, 1, (commit_enable
? 1 : 0), true));
3133 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3136 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3137 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3139 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3140 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3143 unreachable("Not reached");
3147 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3149 assert(devinfo
->gen
>= 11 || bti
== 0);
3150 brw_inst_set_binding_table_index(devinfo
, insn
, bti
);
3154 brw_memory_fence(struct brw_codegen
*p
,
3157 enum opcode send_op
,
3161 const struct gen_device_info
*devinfo
= p
->devinfo
;
3162 const bool commit_enable
= stall
||
3163 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3164 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3165 struct brw_inst
*insn
;
3167 brw_push_insn_state(p
);
3168 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3169 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3170 dst
= retype(vec1(dst
), BRW_REGISTER_TYPE_UW
);
3171 src
= retype(vec1(src
), BRW_REGISTER_TYPE_UD
);
3173 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3174 * message doesn't write anything back.
3176 insn
= next_insn(p
, send_op
);
3177 brw_set_dest(p
, insn
, dst
);
3178 brw_set_src0(p
, insn
, src
);
3179 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3180 commit_enable
, bti
);
3182 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3183 /* IVB does typed surface access through the render cache, so we need to
3184 * flush it too. Use a different register so both flushes can be
3185 * pipelined by the hardware.
3187 insn
= next_insn(p
, send_op
);
3188 brw_set_dest(p
, insn
, offset(dst
, 1));
3189 brw_set_src0(p
, insn
, src
);
3190 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3191 commit_enable
, bti
);
3193 /* Now write the response of the second message into the response of the
3194 * first to trigger a pipeline stall -- This way future render and data
3195 * cache messages will be properly ordered with respect to past data and
3196 * render cache messages.
3198 brw_MOV(p
, dst
, offset(dst
, 1));
3202 brw_set_default_swsb(p
, tgl_swsb_sbid(TGL_SBID_DST
,
3203 brw_get_default_swsb(p
).sbid
));
3205 brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
), dst
);
3208 brw_pop_insn_state(p
);
3212 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3213 struct brw_reg dest
,
3217 struct brw_reg data
,
3218 unsigned msg_length
,
3219 unsigned response_length
)
3221 const struct gen_device_info
*devinfo
= p
->devinfo
;
3222 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3223 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3224 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3225 const unsigned desc
=
3226 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3227 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3230 /* brw_send_indirect_message will automatically use a direct send message
3231 * if data is actually immediate.
3233 brw_send_indirect_message(p
,
3234 GEN7_SFID_PIXEL_INTERPOLATOR
,
3243 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3244 struct brw_reg mask
)
3246 const struct gen_device_info
*devinfo
= p
->devinfo
;
3247 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3248 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3251 assert(devinfo
->gen
>= 7);
3252 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3254 brw_push_insn_state(p
);
3256 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3257 * unnecessary bits in the instruction words, get the information we need
3258 * and reset the default flag register. This allows more instructions to be
3261 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3262 brw_set_default_flag_reg(p
, 0, 0);
3264 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3265 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3267 if (devinfo
->gen
>= 8) {
3268 /* Getting the first active channel index is easy on Gen8: Just find
3269 * the first bit set in the execution mask. The register exists on
3270 * HSW already but it reads back as all ones when the current
3271 * instruction has execution masking disabled, so it's kind of
3274 struct brw_reg exec_mask
=
3275 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3277 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3278 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3279 /* Unfortunately, ce0 does not take into account the thread
3280 * dispatch mask, which may be a problem in cases where it's not
3281 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3282 * some n). Combine ce0 with the given dispatch (or vector) mask
3283 * to mask off those channels which were never dispatched by the
3286 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3287 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3288 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3289 exec_mask
= vec1(dst
);
3292 /* Quarter control has the effect of magically shifting the value of
3293 * ce0 so you'll get the first active channel relative to the
3294 * specified quarter control as result.
3296 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3298 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3300 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3301 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3303 /* Run enough instructions returning zero with execution masking and
3304 * a conditional modifier enabled in order to get the full execution
3305 * mask in f1.0. We could use a single 32-wide move here if it
3306 * weren't because of the hardware bug that causes channel enables to
3307 * be applied incorrectly to the second half of 32-wide instructions
3310 const unsigned lower_size
= MIN2(16, exec_size
);
3311 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3312 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3314 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3315 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3316 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3317 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3318 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3319 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3322 /* Find the first bit set in the exec_size-wide portion of the flag
3323 * register that was updated by the last sequence of MOV
3326 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3327 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3328 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3331 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3333 if (devinfo
->gen
>= 8 &&
3334 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3335 /* In SIMD4x2 mode the first active channel index is just the
3336 * negation of the first bit of the mask register. Note that ce0
3337 * doesn't take into account the dispatch mask, so the Gen7 path
3338 * should be used instead unless you have the guarantee that the
3339 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3342 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3343 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3347 /* Overwrite the destination without and with execution masking to
3348 * find out which of the channels is active.
3350 brw_push_insn_state(p
);
3351 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3352 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3355 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3357 brw_pop_insn_state(p
);
3358 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3362 brw_pop_insn_state(p
);
3366 brw_broadcast(struct brw_codegen
*p
,
3371 const struct gen_device_info
*devinfo
= p
->devinfo
;
3372 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3375 brw_push_insn_state(p
);
3376 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3377 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3379 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3380 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3381 assert(!src
.abs
&& !src
.negate
);
3382 assert(src
.type
== dst
.type
);
3384 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3385 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3386 /* Trivial, the source is already uniform or the index is a constant.
3387 * We will typically not get here if the optimizer is doing its job, but
3388 * asserting would be mean.
3390 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3392 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3393 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3395 /* From the Haswell PRM section "Register Region Restrictions":
3397 * "The lower bits of the AddressImmediate must not overflow to
3398 * change the register address. The lower 5 bits of Address
3399 * Immediate when added to lower 5 bits of address register gives
3400 * the sub-register offset. The upper bits of Address Immediate
3401 * when added to upper bits of address register gives the register
3402 * address. Any overflow from sub-register offset is dropped."
3404 * Fortunately, for broadcast, we never have a sub-register offset so
3405 * this isn't an issue.
3407 assert(src
.subnr
== 0);
3410 const struct brw_reg addr
=
3411 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3412 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3413 /* Limit in bytes of the signed indirect addressing immediate. */
3414 const unsigned limit
= 512;
3416 brw_push_insn_state(p
);
3417 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3418 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3420 /* Take into account the component size and horizontal stride. */
3421 assert(src
.vstride
== src
.hstride
+ src
.width
);
3422 brw_SHL(p
, addr
, vec1(idx
),
3423 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3426 /* We can only address up to limit bytes using the indirect
3427 * addressing immediate, account for the difference if the source
3428 * register is above this limit.
3430 if (offset
>= limit
) {
3431 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3432 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3433 offset
= offset
% limit
;
3436 brw_pop_insn_state(p
);
3438 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3440 /* Use indirect addressing to fetch the specified component. */
3441 if (type_sz(src
.type
) > 4 &&
3442 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3443 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3445 * "When source or destination datatype is 64b or operation is
3446 * integer DWord multiply, indirect addressing must not be
3449 * To work around both of this issue, we do two integer MOVs
3450 * insead of one 64-bit MOV. Because no double value should ever
3451 * cross a register boundary, it's safe to use the immediate
3452 * offset in the indirect here to handle adding 4 bytes to the
3453 * offset and avoid the extra ADD to the register file.
3455 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3456 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3457 BRW_REGISTER_TYPE_D
));
3458 brw_set_default_swsb(p
, tgl_swsb_null());
3459 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3460 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3461 BRW_REGISTER_TYPE_D
));
3464 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3467 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3468 * to all bits of a flag register,
3472 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3473 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3474 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3475 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3477 /* and use predicated SEL to pick the right channel. */
3478 inst
= brw_SEL(p
, dst
,
3479 stride(suboffset(src
, 4), 4, 4, 1),
3480 stride(src
, 4, 4, 1));
3481 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3482 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3486 brw_pop_insn_state(p
);
3490 * This instruction is generated as a single-channel align1 instruction by
3491 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3493 * We can't use the typed atomic op in the FS because that has the execution
3494 * mask ANDed with the pixel mask, but we just want to write the one dword for
3497 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3498 * one u32. So we use the same untyped atomic write message as the pixel
3501 * The untyped atomic operation requires a BUFFER surface type with RAW
3502 * format, and is only accessible through the legacy DATA_CACHE dataport
3505 void brw_shader_time_add(struct brw_codegen
*p
,
3506 struct brw_reg payload
,
3507 uint32_t surf_index
)
3509 const struct gen_device_info
*devinfo
= p
->devinfo
;
3510 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3511 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3512 GEN7_SFID_DATAPORT_DATA_CACHE
);
3513 assert(devinfo
->gen
>= 7);
3515 brw_push_insn_state(p
);
3516 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3517 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3518 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3519 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3521 /* We use brw_vec1_reg and unmasked because we want to increment the given
3524 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3526 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3528 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3529 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3532 brw_inst_set_sfid(devinfo
, send
, sfid
);
3533 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3535 brw_pop_insn_state(p
);
3540 * Emit the SEND message for a barrier
3543 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3545 const struct gen_device_info
*devinfo
= p
->devinfo
;
3546 struct brw_inst
*inst
;
3548 assert(devinfo
->gen
>= 7);
3550 brw_push_insn_state(p
);
3551 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3552 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3553 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3554 brw_set_src0(p
, inst
, src
);
3555 brw_set_src1(p
, inst
, brw_null_reg());
3556 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3558 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3559 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3560 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3562 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3563 brw_pop_insn_state(p
);
3568 * Emit the wait instruction for a barrier
3571 brw_WAIT(struct brw_codegen
*p
)
3573 const struct gen_device_info
*devinfo
= p
->devinfo
;
3574 struct brw_inst
*insn
;
3576 struct brw_reg src
= brw_notification_reg();
3578 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3579 brw_set_dest(p
, insn
, src
);
3580 brw_set_src0(p
, insn
, src
);
3581 brw_set_src1(p
, insn
, brw_null_reg());
3583 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3584 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3588 brw_float_controls_mode(struct brw_codegen
*p
,
3589 unsigned mode
, unsigned mask
)
3591 /* From the Skylake PRM, Volume 7, page 760:
3592 * "Implementation Restriction on Register Access: When the control
3593 * register is used as an explicit source and/or destination, hardware
3594 * does not ensure execution pipeline coherency. Software must set the
3595 * thread control field to ‘switch’ for an instruction that uses
3596 * control register as an explicit operand."
3598 * On Gen12+ this is implemented in terms of SWSB annotations instead.
3600 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3602 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3604 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3605 if (p
->devinfo
->gen
< 12)
3606 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3609 brw_inst
*inst_or
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3611 brw_inst_set_exec_size(p
->devinfo
, inst_or
, BRW_EXECUTE_1
);
3612 if (p
->devinfo
->gen
< 12)
3613 brw_inst_set_thread_control(p
->devinfo
, inst_or
, BRW_THREAD_SWITCH
);
3616 if (p
->devinfo
->gen
>= 12)
3617 brw_SYNC(p
, TGL_SYNC_NOP
);