2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 assert(devinfo
->gen
< 12);
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct gen_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
91 const struct gen_device_info
*devinfo
= p
->devinfo
;
93 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
94 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
95 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
96 assert(dest
.nr
< 128);
98 /* The hardware has a restriction where if the destination is Byte,
99 * the instruction needs to have a stride of 2 (except for packed byte
100 * MOV). This seems to be required even if the destination is the NULL
103 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
104 dest
.nr
== BRW_ARF_NULL
&&
105 type_sz(dest
.type
) == 1) {
106 dest
.hstride
= BRW_HORIZONTAL_STRIDE_2
;
109 gen7_convert_mrf_to_grf(p
, &dest
);
111 if (devinfo
->gen
>= 12 &&
112 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
113 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
114 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
115 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
116 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
117 assert(dest
.subnr
== 0);
118 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
119 (dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
120 dest
.vstride
== dest
.width
+ 1));
121 assert(!dest
.negate
&& !dest
.abs
);
122 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
123 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
125 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
126 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
127 assert(devinfo
->gen
< 12);
128 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
129 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
130 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
131 assert(dest
.subnr
% 16 == 0);
132 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
133 dest
.vstride
== dest
.width
+ 1);
134 assert(!dest
.negate
&& !dest
.abs
);
135 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
136 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
137 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
139 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
140 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
142 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
143 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
145 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
146 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
147 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
148 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
149 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
151 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
152 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
153 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
154 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
155 assert(dest
.writemask
!= 0);
157 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
158 * Although Dst.HorzStride is a don't care for Align16, HW needs
159 * this to be programmed as "01".
161 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
164 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
166 /* These are different sizes in align1 vs align16:
168 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
169 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
170 dest
.indirect_offset
);
171 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
172 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
173 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
175 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
176 dest
.indirect_offset
);
177 /* even ignored in da16, still need to set as '01' */
178 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
183 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
184 * or 16 (SIMD16), as that's normally correct. However, when dealing with
185 * small registers, it can be useful for us to automatically reduce it to
186 * match the register size.
188 if (p
->automatic_exec_sizes
) {
190 * In platforms that support fp64 we can emit instructions with a width
191 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
192 * these cases we need to make sure that these instructions have their
193 * exec sizes set properly when they are emitted and we can't rely on
194 * this code to fix it.
197 if (devinfo
->gen
>= 6)
198 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
200 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
203 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
208 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
210 const struct gen_device_info
*devinfo
= p
->devinfo
;
212 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
213 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
214 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
215 assert(reg
.nr
< 128);
217 gen7_convert_mrf_to_grf(p
, ®
);
219 if (devinfo
->gen
>= 6 &&
220 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
221 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
222 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
223 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
224 /* Any source modifiers or regions will be ignored, since this just
225 * identifies the MRF/GRF to start reading the message contents from.
226 * Check for some likely failures.
230 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
233 if (devinfo
->gen
>= 12 &&
234 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
235 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
236 assert(reg
.file
!= BRW_IMMEDIATE_VALUE
);
237 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
238 assert(reg
.subnr
== 0);
239 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
240 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
241 reg
.vstride
== reg
.width
+ 1));
242 assert(!reg
.negate
&& !reg
.abs
);
243 brw_inst_set_send_src0_reg_file(devinfo
, inst
, reg
.file
);
244 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
246 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
247 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
248 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
249 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
250 assert(reg
.subnr
% 16 == 0);
251 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
252 reg
.vstride
== reg
.width
+ 1);
253 assert(!reg
.negate
&& !reg
.abs
);
254 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
255 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
257 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
258 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
259 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
260 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
262 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
263 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
264 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
265 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
266 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
267 reg
.type
== BRW_REGISTER_TYPE_Q
)
268 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
270 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
272 if (devinfo
->gen
< 12 && type_sz(reg
.type
) < 8) {
273 brw_inst_set_src1_reg_file(devinfo
, inst
,
274 BRW_ARCHITECTURE_REGISTER_FILE
);
275 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
276 brw_inst_src0_reg_hw_type(devinfo
, inst
));
279 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
280 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
281 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
282 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
284 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
287 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
289 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
290 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
292 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
296 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
297 if (reg
.width
== BRW_WIDTH_1
&&
298 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
299 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
300 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
301 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
303 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
304 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
305 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
308 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
309 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
310 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
311 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
312 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
313 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
314 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
315 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
317 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
318 /* This is an oddity of the fact we're using the same
319 * descriptions for registers in align_16 as align_1:
321 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
322 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
323 reg
.type
== BRW_REGISTER_TYPE_DF
&&
324 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
327 * "For Align16 access mode, only encodings of 0000 and 0011
328 * are allowed. Other codes are reserved."
330 * Presumably the DevSNB behavior applies to IVB as well.
332 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
334 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
343 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
345 const struct gen_device_info
*devinfo
= p
->devinfo
;
347 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
348 assert(reg
.nr
< 128);
350 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
351 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
||
352 (devinfo
->gen
>= 12 &&
353 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
354 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
))) {
355 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
356 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
357 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
358 assert(reg
.subnr
== 0);
359 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
360 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
361 reg
.vstride
== reg
.width
+ 1));
362 assert(!reg
.negate
&& !reg
.abs
);
363 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
364 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
366 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
368 * "Accumulator registers may be accessed explicitly as src0
371 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
372 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
374 gen7_convert_mrf_to_grf(p
, ®
);
375 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
377 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
378 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
379 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
381 /* Only src1 can be immediate in two-argument instructions.
383 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
385 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
386 /* two-argument instructions can only use 32-bit immediates */
387 assert(type_sz(reg
.type
) < 8);
388 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
390 /* This is a hardware restriction, which may or may not be lifted
393 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
394 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
396 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
397 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
398 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
400 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
403 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
404 if (reg
.width
== BRW_WIDTH_1
&&
405 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
406 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
407 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
408 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
410 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
411 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
412 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
415 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
416 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
417 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
418 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
419 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
420 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
421 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
422 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
424 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
425 /* This is an oddity of the fact we're using the same
426 * descriptions for registers in align_16 as align_1:
428 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
429 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
430 reg
.type
== BRW_REGISTER_TYPE_DF
&&
431 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
434 * "For Align16 access mode, only encodings of 0000 and 0011
435 * are allowed. Other codes are reserved."
437 * Presumably the DevSNB behavior applies to IVB as well.
439 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
441 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
449 * Specify the descriptor and extended descriptor immediate for a SEND(C)
450 * message instruction.
453 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
454 unsigned desc
, unsigned ex_desc
)
456 const struct gen_device_info
*devinfo
= p
->devinfo
;
457 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
458 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
459 if (devinfo
->gen
< 12)
460 brw_inst_set_src1_file_type(devinfo
, inst
,
461 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
462 brw_inst_set_send_desc(devinfo
, inst
, desc
);
463 if (devinfo
->gen
>= 9)
464 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
467 static void brw_set_math_message( struct brw_codegen
*p
,
470 unsigned integer_type
,
474 const struct gen_device_info
*devinfo
= p
->devinfo
;
476 unsigned response_length
;
478 /* Infer message length from the function */
480 case BRW_MATH_FUNCTION_POW
:
481 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
482 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
483 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
491 /* Infer response length from the function */
493 case BRW_MATH_FUNCTION_SINCOS
:
494 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
502 brw_set_desc(p
, inst
, brw_message_desc(
503 devinfo
, msg_length
, response_length
, false));
505 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
506 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
507 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
508 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
509 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
510 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
511 brw_inst_set_saturate(devinfo
, inst
, 0);
515 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
518 unsigned response_length
,
521 const struct gen_device_info
*devinfo
= p
->devinfo
;
523 brw_set_desc(p
, insn
, brw_message_desc(
524 devinfo
, 1, response_length
, true));
526 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
527 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
528 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
529 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
530 /* The following fields are not used by FF_SYNC: */
531 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
532 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
533 brw_inst_set_urb_used(devinfo
, insn
, 0);
534 brw_inst_set_urb_complete(devinfo
, insn
, 0);
537 static void brw_set_urb_message( struct brw_codegen
*p
,
539 enum brw_urb_write_flags flags
,
541 unsigned response_length
,
543 unsigned swizzle_control
)
545 const struct gen_device_info
*devinfo
= p
->devinfo
;
547 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
548 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
549 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
551 brw_set_desc(p
, insn
, brw_message_desc(
552 devinfo
, msg_length
, response_length
, true));
554 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
555 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
557 if (flags
& BRW_URB_WRITE_OWORD
) {
558 assert(msg_length
== 2); /* header + one OWORD of data */
559 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
561 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
564 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
565 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
567 if (devinfo
->gen
< 8) {
568 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
571 if (devinfo
->gen
< 7) {
572 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
573 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
575 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
576 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
581 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
585 bool invalidate_after_read
,
587 unsigned addr_offset
,
592 const struct gen_device_info
*devinfo
= p
->devinfo
;
593 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
594 (devinfo
->gen
>= 8 && num_regs
== 8));
595 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
598 brw_set_desc(p
, inst
, brw_message_desc(
599 devinfo
, mlen
, rlen
, header_present
));
601 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
602 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
603 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
604 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
605 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
606 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
607 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
611 brw_inst_set_state(const struct gen_device_info
*devinfo
,
613 const struct brw_insn_state
*state
)
615 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
616 brw_inst_set_group(devinfo
, insn
, state
->group
);
617 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
618 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
619 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
620 if (devinfo
->gen
>= 12)
621 brw_inst_set_swsb(devinfo
, insn
, tgl_swsb_encode(state
->swsb
));
622 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
623 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
624 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
626 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
627 state
->access_mode
== BRW_ALIGN_16
) {
628 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
629 if (devinfo
->gen
>= 7)
630 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
632 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
633 if (devinfo
->gen
>= 7)
634 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
637 if (devinfo
->gen
>= 6)
638 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
641 #define next_insn brw_next_insn
643 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
645 const struct gen_device_info
*devinfo
= p
->devinfo
;
648 if (p
->nr_insn
+ 1 > p
->store_size
) {
650 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
653 p
->next_insn_offset
+= 16;
654 insn
= &p
->store
[p
->nr_insn
++];
656 memset(insn
, 0, sizeof(*insn
));
657 brw_inst_set_opcode(devinfo
, insn
, opcode
);
659 /* Apply the default instruction state */
660 brw_inst_set_state(devinfo
, insn
, p
->current
);
666 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
667 struct brw_reg dest
, struct brw_reg src
)
669 brw_inst
*insn
= next_insn(p
, opcode
);
670 brw_set_dest(p
, insn
, dest
);
671 brw_set_src0(p
, insn
, src
);
676 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
677 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
679 /* 64-bit immediates are only supported on 1-src instructions */
680 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
681 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
683 brw_inst
*insn
= next_insn(p
, opcode
);
684 brw_set_dest(p
, insn
, dest
);
685 brw_set_src0(p
, insn
, src0
);
686 brw_set_src1(p
, insn
, src1
);
691 get_3src_subreg_nr(struct brw_reg reg
)
693 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
694 * use 32-bit units (components 0..7). Since they only support F/D/UD
695 * types, this doesn't lose any flexibility, but uses fewer bits.
697 return reg
.subnr
/ 4;
700 static enum gen10_align1_3src_vertical_stride
701 to_3src_align1_vstride(const struct gen_device_info
*devinfo
,
702 enum brw_vertical_stride vstride
)
705 case BRW_VERTICAL_STRIDE_0
:
706 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
707 case BRW_VERTICAL_STRIDE_1
:
708 assert(devinfo
->gen
>= 12);
709 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1
;
710 case BRW_VERTICAL_STRIDE_2
:
711 assert(devinfo
->gen
< 12);
712 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
713 case BRW_VERTICAL_STRIDE_4
:
714 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
715 case BRW_VERTICAL_STRIDE_8
:
716 case BRW_VERTICAL_STRIDE_16
:
717 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
719 unreachable("invalid vstride");
724 static enum gen10_align1_3src_src_horizontal_stride
725 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
728 case BRW_HORIZONTAL_STRIDE_0
:
729 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
730 case BRW_HORIZONTAL_STRIDE_1
:
731 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
732 case BRW_HORIZONTAL_STRIDE_2
:
733 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
734 case BRW_HORIZONTAL_STRIDE_4
:
735 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
737 unreachable("invalid hstride");
742 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
743 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
745 const struct gen_device_info
*devinfo
= p
->devinfo
;
746 brw_inst
*inst
= next_insn(p
, opcode
);
748 gen7_convert_mrf_to_grf(p
, &dest
);
750 assert(dest
.nr
< 128);
752 if (devinfo
->gen
>= 10)
753 assert(!(src0
.file
== BRW_IMMEDIATE_VALUE
&&
754 src2
.file
== BRW_IMMEDIATE_VALUE
));
756 assert(src0
.file
== BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
757 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
&& src1
.nr
< 128);
758 assert(src2
.file
== BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
759 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
760 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
761 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
762 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
764 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
765 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
766 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
768 if (devinfo
->gen
>= 12) {
769 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
, dest
.file
);
770 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
772 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
773 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
774 BRW_ALIGN1_3SRC_ACCUMULATOR
);
775 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
777 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
778 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
779 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
782 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
784 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
786 if (brw_reg_type_is_floating_point(dest
.type
)) {
787 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
788 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
790 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
791 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
794 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
795 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
796 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
797 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
799 if (src0
.file
== BRW_IMMEDIATE_VALUE
) {
800 brw_inst_set_3src_a1_src0_imm(devinfo
, inst
, src0
.ud
);
802 brw_inst_set_3src_a1_src0_vstride(
803 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src0
.vstride
));
804 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
805 to_3src_align1_hstride(src0
.hstride
));
806 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
807 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
808 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
810 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
812 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
813 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
815 brw_inst_set_3src_a1_src1_vstride(
816 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src1
.vstride
));
817 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
818 to_3src_align1_hstride(src1
.hstride
));
820 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
821 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
822 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
824 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
826 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
827 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
829 if (src2
.file
== BRW_IMMEDIATE_VALUE
) {
830 brw_inst_set_3src_a1_src2_imm(devinfo
, inst
, src2
.ud
);
832 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
833 to_3src_align1_hstride(src2
.hstride
));
834 /* no vstride on src2 */
835 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
836 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
837 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
838 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
841 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
842 src0
.file
== BRW_IMMEDIATE_VALUE
||
843 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
844 src0
.type
== BRW_REGISTER_TYPE_NF
));
845 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
846 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
847 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
848 src2
.file
== BRW_IMMEDIATE_VALUE
);
850 if (devinfo
->gen
>= 12) {
851 if (src0
.file
== BRW_IMMEDIATE_VALUE
) {
852 brw_inst_set_3src_a1_src0_is_imm(devinfo
, inst
, 1);
854 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
, src0
.file
);
857 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
, src1
.file
);
859 if (src2
.file
== BRW_IMMEDIATE_VALUE
) {
860 brw_inst_set_3src_a1_src2_is_imm(devinfo
, inst
, 1);
862 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
, src2
.file
);
865 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
866 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
867 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
868 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
869 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
870 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
871 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
872 BRW_ALIGN1_3SRC_ACCUMULATOR
);
873 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
874 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
875 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
876 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
880 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
881 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
882 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
883 dest
.type
== BRW_REGISTER_TYPE_DF
||
884 dest
.type
== BRW_REGISTER_TYPE_D
||
885 dest
.type
== BRW_REGISTER_TYPE_UD
||
886 (dest
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 8));
887 if (devinfo
->gen
== 6) {
888 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
889 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
891 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
892 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
893 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
895 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
896 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
897 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
898 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
899 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
900 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
901 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
902 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
904 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
905 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
906 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
907 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
908 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
909 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
910 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
911 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
913 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
914 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
915 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
916 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
917 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
918 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
919 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
920 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
922 if (devinfo
->gen
>= 7) {
923 /* Set both the source and destination types based on dest.type,
924 * ignoring the source register types. The MAD and LRP emitters ensure
925 * that all four types are float. The BFE and BFI2 emitters, however,
926 * may send us mixed D and UD types and want us to ignore that and use
927 * the destination type.
929 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
930 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
932 /* From the Bspec, 3D Media GPGPU, Instruction fields, srcType:
934 * "Three source instructions can use operands with mixed-mode
935 * precision. When SrcType field is set to :f or :hf it defines
936 * precision for source 0 only, and fields Src1Type and Src2Type
937 * define precision for other source operands:
939 * 0b = :f. Single precision Float (32-bit).
940 * 1b = :hf. Half precision Float (16-bit)."
942 if (src1
.type
== BRW_REGISTER_TYPE_HF
)
943 brw_inst_set_3src_a16_src1_type(devinfo
, inst
, 1);
945 if (src2
.type
== BRW_REGISTER_TYPE_HF
)
946 brw_inst_set_3src_a16_src2_type(devinfo
, inst
, 1);
954 /***********************************************************************
955 * Convenience routines.
958 brw_inst *brw_##OP(struct brw_codegen *p, \
959 struct brw_reg dest, \
960 struct brw_reg src0) \
962 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
966 brw_inst *brw_##OP(struct brw_codegen *p, \
967 struct brw_reg dest, \
968 struct brw_reg src0, \
969 struct brw_reg src1) \
971 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
975 brw_inst *brw_##OP(struct brw_codegen *p, \
976 struct brw_reg dest, \
977 struct brw_reg src0, \
978 struct brw_reg src1, \
979 struct brw_reg src2) \
981 if (p->current->access_mode == BRW_ALIGN_16) { \
982 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
983 src0.swizzle = BRW_SWIZZLE_XXXX; \
984 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
985 src1.swizzle = BRW_SWIZZLE_XXXX; \
986 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
987 src2.swizzle = BRW_SWIZZLE_XXXX; \
989 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
993 brw_inst *brw_##OP(struct brw_codegen *p, \
994 struct brw_reg dest, \
995 struct brw_reg src0, \
996 struct brw_reg src1, \
997 struct brw_reg src2) \
999 assert(dest.type == BRW_REGISTER_TYPE_F || \
1000 dest.type == BRW_REGISTER_TYPE_DF); \
1001 if (dest.type == BRW_REGISTER_TYPE_F) { \
1002 assert(src0.type == BRW_REGISTER_TYPE_F); \
1003 assert(src1.type == BRW_REGISTER_TYPE_F); \
1004 assert(src2.type == BRW_REGISTER_TYPE_F); \
1005 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
1006 assert(src0.type == BRW_REGISTER_TYPE_DF); \
1007 assert(src1.type == BRW_REGISTER_TYPE_DF); \
1008 assert(src2.type == BRW_REGISTER_TYPE_DF); \
1011 if (p->current->access_mode == BRW_ALIGN_16) { \
1012 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
1013 src0.swizzle = BRW_SWIZZLE_XXXX; \
1014 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
1015 src1.swizzle = BRW_SWIZZLE_XXXX; \
1016 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
1017 src2.swizzle = BRW_SWIZZLE_XXXX; \
1019 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1022 /* Rounding operations (other than RNDD) require two instructions - the first
1023 * stores a rounded value (possibly the wrong way) in the dest register, but
1024 * also sets a per-channel "increment bit" in the flag register. A predicated
1025 * add of 1.0 fixes dest to contain the desired result.
1027 * Sandybridge and later appear to round correctly without an ADD.
1030 void brw_##OP(struct brw_codegen *p, \
1031 struct brw_reg dest, \
1032 struct brw_reg src) \
1034 const struct gen_device_info *devinfo = p->devinfo; \
1035 brw_inst *rnd, *add; \
1036 rnd = next_insn(p, BRW_OPCODE_##OP); \
1037 brw_set_dest(p, rnd, dest); \
1038 brw_set_src0(p, rnd, src); \
1040 if (devinfo->gen < 6) { \
1041 /* turn on round-increments */ \
1042 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
1043 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1044 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1086 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
1088 const struct gen_device_info
*devinfo
= p
->devinfo
;
1090 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
1091 * To avoid the problems that causes, we use an <X,2,0> source region to
1092 * read each element twice.
1094 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1095 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
1096 dest
.type
== BRW_REGISTER_TYPE_DF
&&
1097 (src0
.type
== BRW_REGISTER_TYPE_F
||
1098 src0
.type
== BRW_REGISTER_TYPE_D
||
1099 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
1100 !has_scalar_region(src0
)) {
1101 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
1102 src0
.vstride
= src0
.hstride
;
1103 src0
.width
= BRW_WIDTH_2
;
1104 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1107 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1111 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1112 struct brw_reg src0
, struct brw_reg src1
)
1115 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1116 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1117 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1118 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1119 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1122 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1123 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1124 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1125 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1126 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1129 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1133 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1134 struct brw_reg src0
, struct brw_reg src1
)
1136 assert(dest
.type
== src0
.type
);
1137 assert(src0
.type
== src1
.type
);
1138 switch (src0
.type
) {
1139 case BRW_REGISTER_TYPE_B
:
1140 case BRW_REGISTER_TYPE_UB
:
1141 case BRW_REGISTER_TYPE_W
:
1142 case BRW_REGISTER_TYPE_UW
:
1143 case BRW_REGISTER_TYPE_D
:
1144 case BRW_REGISTER_TYPE_UD
:
1147 unreachable("Bad type for brw_AVG");
1150 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1154 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1155 struct brw_reg src0
, struct brw_reg src1
)
1158 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1159 src0
.type
== BRW_REGISTER_TYPE_UD
||
1160 src1
.type
== BRW_REGISTER_TYPE_D
||
1161 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1162 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1165 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1166 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1167 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1168 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1169 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1172 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1173 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1174 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1175 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1176 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1179 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1180 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1181 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1182 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1184 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1188 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1189 struct brw_reg src0
, struct brw_reg src1
)
1191 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1192 src0
.width
= BRW_WIDTH_1
;
1193 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1194 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1198 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1199 struct brw_reg src0
, struct brw_reg src1
)
1201 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1202 src0
.width
= BRW_WIDTH_1
;
1203 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1204 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1205 src1
.width
= BRW_WIDTH_8
;
1206 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1207 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1211 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1213 const struct gen_device_info
*devinfo
= p
->devinfo
;
1214 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1215 /* The F32TO16 instruction doesn't support 32-bit destination types in
1216 * Align1 mode, and neither does the Gen8 implementation in terms of a
1217 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1218 * an undocumented feature.
1220 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1221 (!align16
|| devinfo
->gen
>= 8));
1225 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1227 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1228 dst
.type
== BRW_REGISTER_TYPE_W
||
1229 dst
.type
== BRW_REGISTER_TYPE_UW
||
1230 dst
.type
== BRW_REGISTER_TYPE_HF
);
1233 brw_push_insn_state(p
);
1235 if (needs_zero_fill
) {
1236 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1237 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1240 if (devinfo
->gen
>= 8) {
1241 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1243 assert(devinfo
->gen
== 7);
1244 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1247 if (needs_zero_fill
) {
1248 if (devinfo
->gen
< 12)
1249 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1250 brw_set_default_swsb(p
, tgl_swsb_null());
1251 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1252 if (devinfo
->gen
< 12)
1253 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1256 brw_pop_insn_state(p
);
1261 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1263 const struct gen_device_info
*devinfo
= p
->devinfo
;
1264 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1267 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1269 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1271 * Because this instruction does not have a 16-bit floating-point
1272 * type, the source data type must be Word (W). The destination type
1273 * must be F (Float).
1275 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1276 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1278 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1279 src
.type
== BRW_REGISTER_TYPE_UW
||
1280 src
.type
== BRW_REGISTER_TYPE_HF
);
1283 if (devinfo
->gen
>= 8) {
1284 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1286 assert(devinfo
->gen
== 7);
1287 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1292 void brw_NOP(struct brw_codegen
*p
)
1294 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1295 memset(insn
, 0, sizeof(*insn
));
1296 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1299 void brw_SYNC(struct brw_codegen
*p
, enum tgl_sync_function func
)
1301 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SYNC
);
1302 brw_inst_set_cond_modifier(p
->devinfo
, insn
, func
);
1305 /***********************************************************************
1306 * Comparisons, if/else/endif
1310 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1311 unsigned predicate_control
)
1313 const struct gen_device_info
*devinfo
= p
->devinfo
;
1314 struct brw_reg ip
= brw_ip_reg();
1315 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1317 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1318 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1319 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1320 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1326 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1328 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1330 p
->if_stack_depth
++;
1331 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1332 p
->if_stack_array_size
*= 2;
1333 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1334 p
->if_stack_array_size
);
1339 pop_if_stack(struct brw_codegen
*p
)
1341 p
->if_stack_depth
--;
1342 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1346 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1348 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1349 p
->loop_stack_array_size
*= 2;
1350 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1351 p
->loop_stack_array_size
);
1352 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1353 p
->loop_stack_array_size
);
1356 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1357 p
->loop_stack_depth
++;
1358 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1362 get_inner_do_insn(struct brw_codegen
*p
)
1364 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1367 /* EU takes the value from the flag register and pushes it onto some
1368 * sort of a stack (presumably merging with any flag value already on
1369 * the stack). Within an if block, the flags at the top of the stack
1370 * control execution on each channel of the unit, eg. on each of the
1371 * 16 pixel values in our wm programs.
1373 * When the matching 'else' instruction is reached (presumably by
1374 * countdown of the instruction count patched in by our ELSE/ENDIF
1375 * functions), the relevant flags are inverted.
1377 * When the matching 'endif' instruction is reached, the flags are
1378 * popped off. If the stack is now empty, normal execution resumes.
1381 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1383 const struct gen_device_info
*devinfo
= p
->devinfo
;
1386 insn
= next_insn(p
, BRW_OPCODE_IF
);
1388 /* Override the defaults for this instruction:
1390 if (devinfo
->gen
< 6) {
1391 brw_set_dest(p
, insn
, brw_ip_reg());
1392 brw_set_src0(p
, insn
, brw_ip_reg());
1393 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1394 } else if (devinfo
->gen
== 6) {
1395 brw_set_dest(p
, insn
, brw_imm_w(0));
1396 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1397 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1398 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1399 } else if (devinfo
->gen
== 7) {
1400 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1401 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1402 brw_set_src1(p
, insn
, brw_imm_w(0));
1403 brw_inst_set_jip(devinfo
, insn
, 0);
1404 brw_inst_set_uip(devinfo
, insn
, 0);
1406 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1407 if (devinfo
->gen
< 12)
1408 brw_set_src0(p
, insn
, brw_imm_d(0));
1409 brw_inst_set_jip(devinfo
, insn
, 0);
1410 brw_inst_set_uip(devinfo
, insn
, 0);
1413 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1414 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1415 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1416 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1417 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1418 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1420 push_if_stack(p
, insn
);
1421 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1425 /* This function is only used for gen6-style IF instructions with an
1426 * embedded comparison (conditional modifier). It is not used on gen7.
1429 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1430 struct brw_reg src0
, struct brw_reg src1
)
1432 const struct gen_device_info
*devinfo
= p
->devinfo
;
1435 insn
= next_insn(p
, BRW_OPCODE_IF
);
1437 brw_set_dest(p
, insn
, brw_imm_w(0));
1438 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1439 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1440 brw_set_src0(p
, insn
, src0
);
1441 brw_set_src1(p
, insn
, src1
);
1443 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1444 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1445 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1447 push_if_stack(p
, insn
);
1452 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1455 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1456 brw_inst
*if_inst
, brw_inst
*else_inst
)
1458 const struct gen_device_info
*devinfo
= p
->devinfo
;
1460 /* The next instruction (where the ENDIF would be, if it existed) */
1461 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1463 assert(p
->single_program_flow
);
1464 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1465 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1466 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1468 /* Convert IF to an ADD instruction that moves the instruction pointer
1469 * to the first instruction of the ELSE block. If there is no ELSE
1470 * block, point to where ENDIF would be. Reverse the predicate.
1472 * There's no need to execute an ENDIF since we don't need to do any
1473 * stack operations, and if we're currently executing, we just want to
1474 * continue normally.
1476 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1477 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1479 if (else_inst
!= NULL
) {
1480 /* Convert ELSE to an ADD instruction that points where the ENDIF
1483 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1485 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1486 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1488 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1493 * Patch IF and ELSE instructions with appropriate jump targets.
1496 patch_IF_ELSE(struct brw_codegen
*p
,
1497 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1499 const struct gen_device_info
*devinfo
= p
->devinfo
;
1501 /* We shouldn't be patching IF and ELSE instructions in single program flow
1502 * mode when gen < 6, because in single program flow mode on those
1503 * platforms, we convert flow control instructions to conditional ADDs that
1504 * operate on IP (see brw_ENDIF).
1506 * However, on Gen6, writing to IP doesn't work in single program flow mode
1507 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1508 * not be updated by non-flow control instructions."). And on later
1509 * platforms, there is no significant benefit to converting control flow
1510 * instructions to conditional ADDs. So we do patch IF and ELSE
1511 * instructions in single program flow mode on those platforms.
1513 if (devinfo
->gen
< 6)
1514 assert(!p
->single_program_flow
);
1516 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1517 assert(endif_inst
!= NULL
);
1518 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1520 unsigned br
= brw_jump_scale(devinfo
);
1522 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1523 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1525 if (else_inst
== NULL
) {
1526 /* Patch IF -> ENDIF */
1527 if (devinfo
->gen
< 6) {
1528 /* Turn it into an IFF, which means no mask stack operations for
1529 * all-false and jumping past the ENDIF.
1531 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1532 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1533 br
* (endif_inst
- if_inst
+ 1));
1534 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1535 } else if (devinfo
->gen
== 6) {
1536 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1537 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1539 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1540 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1543 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1545 /* Patch IF -> ELSE */
1546 if (devinfo
->gen
< 6) {
1547 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1548 br
* (else_inst
- if_inst
));
1549 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1550 } else if (devinfo
->gen
== 6) {
1551 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1552 br
* (else_inst
- if_inst
+ 1));
1555 /* Patch ELSE -> ENDIF */
1556 if (devinfo
->gen
< 6) {
1557 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1560 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1561 br
* (endif_inst
- else_inst
+ 1));
1562 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1563 } else if (devinfo
->gen
== 6) {
1564 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1565 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1566 br
* (endif_inst
- else_inst
));
1568 /* The IF instruction's JIP should point just past the ELSE */
1569 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1570 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1571 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1572 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1573 if (devinfo
->gen
>= 8) {
1574 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1575 * should point to ENDIF.
1577 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1584 brw_ELSE(struct brw_codegen
*p
)
1586 const struct gen_device_info
*devinfo
= p
->devinfo
;
1589 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1591 if (devinfo
->gen
< 6) {
1592 brw_set_dest(p
, insn
, brw_ip_reg());
1593 brw_set_src0(p
, insn
, brw_ip_reg());
1594 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1595 } else if (devinfo
->gen
== 6) {
1596 brw_set_dest(p
, insn
, brw_imm_w(0));
1597 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1598 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1599 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1600 } else if (devinfo
->gen
== 7) {
1601 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1602 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1603 brw_set_src1(p
, insn
, brw_imm_w(0));
1604 brw_inst_set_jip(devinfo
, insn
, 0);
1605 brw_inst_set_uip(devinfo
, insn
, 0);
1607 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1608 if (devinfo
->gen
< 12)
1609 brw_set_src0(p
, insn
, brw_imm_d(0));
1610 brw_inst_set_jip(devinfo
, insn
, 0);
1611 brw_inst_set_uip(devinfo
, insn
, 0);
1614 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1615 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1616 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1617 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1619 push_if_stack(p
, insn
);
1623 brw_ENDIF(struct brw_codegen
*p
)
1625 const struct gen_device_info
*devinfo
= p
->devinfo
;
1626 brw_inst
*insn
= NULL
;
1627 brw_inst
*else_inst
= NULL
;
1628 brw_inst
*if_inst
= NULL
;
1630 bool emit_endif
= true;
1632 /* In single program flow mode, we can express IF and ELSE instructions
1633 * equivalently as ADD instructions that operate on IP. On platforms prior
1634 * to Gen6, flow control instructions cause an implied thread switch, so
1635 * this is a significant savings.
1637 * However, on Gen6, writing to IP doesn't work in single program flow mode
1638 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1639 * not be updated by non-flow control instructions."). And on later
1640 * platforms, there is no significant benefit to converting control flow
1641 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1644 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1648 * A single next_insn() may change the base address of instruction store
1649 * memory(p->store), so call it first before referencing the instruction
1650 * store pointer from an index
1653 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1655 /* Pop the IF and (optional) ELSE instructions from the stack */
1656 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1657 tmp
= pop_if_stack(p
);
1658 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1660 tmp
= pop_if_stack(p
);
1665 /* ENDIF is useless; don't bother emitting it. */
1666 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1670 if (devinfo
->gen
< 6) {
1671 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1672 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1673 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1674 } else if (devinfo
->gen
== 6) {
1675 brw_set_dest(p
, insn
, brw_imm_w(0));
1676 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1677 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1678 } else if (devinfo
->gen
== 7) {
1679 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1680 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1681 brw_set_src1(p
, insn
, brw_imm_w(0));
1683 brw_set_src0(p
, insn
, brw_imm_d(0));
1686 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1687 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1688 if (devinfo
->gen
< 6)
1689 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1691 /* Also pop item off the stack in the endif instruction: */
1692 if (devinfo
->gen
< 6) {
1693 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1694 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1695 } else if (devinfo
->gen
== 6) {
1696 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1698 brw_inst_set_jip(devinfo
, insn
, 2);
1700 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1704 brw_BREAK(struct brw_codegen
*p
)
1706 const struct gen_device_info
*devinfo
= p
->devinfo
;
1709 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1710 if (devinfo
->gen
>= 8) {
1711 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1712 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1713 } else if (devinfo
->gen
>= 6) {
1714 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1715 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1716 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1718 brw_set_dest(p
, insn
, brw_ip_reg());
1719 brw_set_src0(p
, insn
, brw_ip_reg());
1720 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1721 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1722 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1724 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1725 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1731 brw_CONT(struct brw_codegen
*p
)
1733 const struct gen_device_info
*devinfo
= p
->devinfo
;
1736 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1737 brw_set_dest(p
, insn
, brw_ip_reg());
1738 if (devinfo
->gen
>= 8) {
1739 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1741 brw_set_src0(p
, insn
, brw_ip_reg());
1742 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1745 if (devinfo
->gen
< 6) {
1746 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1747 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1749 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1750 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1755 gen6_HALT(struct brw_codegen
*p
)
1757 const struct gen_device_info
*devinfo
= p
->devinfo
;
1760 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1761 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1762 if (devinfo
->gen
< 8) {
1763 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1764 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1765 } else if (devinfo
->gen
< 12) {
1766 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1769 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1770 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1776 * The DO/WHILE is just an unterminated loop -- break or continue are
1777 * used for control within the loop. We have a few ways they can be
1780 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1781 * jip and no DO instruction.
1783 * For non-uniform control flow pre-gen6, there's a DO instruction to
1784 * push the mask, and a WHILE to jump back, and BREAK to get out and
1787 * For gen6, there's no more mask stack, so no need for DO. WHILE
1788 * just points back to the first instruction of the loop.
1791 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1793 const struct gen_device_info
*devinfo
= p
->devinfo
;
1795 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1796 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1797 return &p
->store
[p
->nr_insn
];
1799 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1801 push_loop_stack(p
, insn
);
1803 /* Override the defaults for this instruction:
1805 brw_set_dest(p
, insn
, brw_null_reg());
1806 brw_set_src0(p
, insn
, brw_null_reg());
1807 brw_set_src1(p
, insn
, brw_null_reg());
1809 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1810 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1811 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1818 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1821 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1822 * nesting, since it can always just point to the end of the block/current loop.
1825 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1827 const struct gen_device_info
*devinfo
= p
->devinfo
;
1828 brw_inst
*do_inst
= get_inner_do_insn(p
);
1830 unsigned br
= brw_jump_scale(devinfo
);
1832 assert(devinfo
->gen
< 6);
1834 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1835 /* If the jump count is != 0, that means that this instruction has already
1836 * been patched because it's part of a loop inside of the one we're
1839 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1840 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1841 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1842 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1843 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1844 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1850 brw_WHILE(struct brw_codegen
*p
)
1852 const struct gen_device_info
*devinfo
= p
->devinfo
;
1853 brw_inst
*insn
, *do_insn
;
1854 unsigned br
= brw_jump_scale(devinfo
);
1856 if (devinfo
->gen
>= 6) {
1857 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1858 do_insn
= get_inner_do_insn(p
);
1860 if (devinfo
->gen
>= 8) {
1861 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1862 if (devinfo
->gen
< 12)
1863 brw_set_src0(p
, insn
, brw_imm_d(0));
1864 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1865 } else if (devinfo
->gen
== 7) {
1866 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1867 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1868 brw_set_src1(p
, insn
, brw_imm_w(0));
1869 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1871 brw_set_dest(p
, insn
, brw_imm_w(0));
1872 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1873 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1874 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1877 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1880 if (p
->single_program_flow
) {
1881 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1882 do_insn
= get_inner_do_insn(p
);
1884 brw_set_dest(p
, insn
, brw_ip_reg());
1885 brw_set_src0(p
, insn
, brw_ip_reg());
1886 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1887 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1889 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1890 do_insn
= get_inner_do_insn(p
);
1892 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1894 brw_set_dest(p
, insn
, brw_ip_reg());
1895 brw_set_src0(p
, insn
, brw_ip_reg());
1896 brw_set_src1(p
, insn
, brw_imm_d(0));
1898 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1899 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1900 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1902 brw_patch_break_cont(p
, insn
);
1905 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1907 p
->loop_stack_depth
--;
1914 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1916 const struct gen_device_info
*devinfo
= p
->devinfo
;
1917 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1920 if (devinfo
->gen
>= 5)
1923 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1924 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1926 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1927 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1930 /* To integrate with the above, it makes sense that the comparison
1931 * instruction should populate the flag register. It might be simpler
1932 * just to use the flag reg for most WM tasks?
1934 void brw_CMP(struct brw_codegen
*p
,
1935 struct brw_reg dest
,
1936 unsigned conditional
,
1937 struct brw_reg src0
,
1938 struct brw_reg src1
)
1940 const struct gen_device_info
*devinfo
= p
->devinfo
;
1941 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1943 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1944 brw_set_dest(p
, insn
, dest
);
1945 brw_set_src0(p
, insn
, src0
);
1946 brw_set_src1(p
, insn
, src1
);
1948 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1950 * "Any CMP instruction with a null destination must use a {switch}."
1952 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1953 * mentioned on their work-arounds pages.
1955 if (devinfo
->gen
== 7) {
1956 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1957 dest
.nr
== BRW_ARF_NULL
) {
1958 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1963 /***********************************************************************
1964 * Helpers for the various SEND message types:
1967 /** Extended math function, float[8].
1969 void gen4_math(struct brw_codegen
*p
,
1970 struct brw_reg dest
,
1972 unsigned msg_reg_nr
,
1974 unsigned precision
)
1976 const struct gen_device_info
*devinfo
= p
->devinfo
;
1977 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1979 if (has_scalar_region(src
)) {
1980 data_type
= BRW_MATH_DATA_SCALAR
;
1982 data_type
= BRW_MATH_DATA_VECTOR
;
1985 assert(devinfo
->gen
< 6);
1987 /* Example code doesn't set predicate_control for send
1990 brw_inst_set_pred_control(devinfo
, insn
, 0);
1991 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1993 brw_set_dest(p
, insn
, dest
);
1994 brw_set_src0(p
, insn
, src
);
1995 brw_set_math_message(p
,
1998 src
.type
== BRW_REGISTER_TYPE_D
,
2003 void gen6_math(struct brw_codegen
*p
,
2004 struct brw_reg dest
,
2006 struct brw_reg src0
,
2007 struct brw_reg src1
)
2009 const struct gen_device_info
*devinfo
= p
->devinfo
;
2010 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
2012 assert(devinfo
->gen
>= 6);
2014 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
2015 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
2017 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2018 if (devinfo
->gen
== 6) {
2019 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2020 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2023 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2024 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2025 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2026 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2027 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2028 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2029 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2031 assert(src0
.type
== BRW_REGISTER_TYPE_F
||
2032 (src0
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2033 assert(src1
.type
== BRW_REGISTER_TYPE_F
||
2034 (src1
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2037 /* Source modifiers are ignored for extended math instructions on Gen6. */
2038 if (devinfo
->gen
== 6) {
2039 assert(!src0
.negate
);
2041 assert(!src1
.negate
);
2045 brw_inst_set_math_function(devinfo
, insn
, function
);
2047 brw_set_dest(p
, insn
, dest
);
2048 brw_set_src0(p
, insn
, src0
);
2049 brw_set_src1(p
, insn
, src1
);
2053 * Return the right surface index to access the thread scratch space using
2054 * stateless dataport messages.
2057 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2059 /* The scratch space is thread-local so IA coherency is unnecessary. */
2060 if (p
->devinfo
->gen
>= 8)
2061 return GEN8_BTI_STATELESS_NON_COHERENT
;
2063 return BRW_BTI_STATELESS
;
2067 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2068 * using a constant offset per channel.
2070 * The offset must be aligned to oword size (16 bytes). Used for
2071 * register spilling.
2073 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2078 const struct gen_device_info
*devinfo
= p
->devinfo
;
2079 const unsigned target_cache
=
2080 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2081 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2082 BRW_SFID_DATAPORT_WRITE
);
2083 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2086 if (devinfo
->gen
>= 6)
2089 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2091 const unsigned mlen
= 1 + num_regs
;
2093 /* Set up the message header. This is g0, with g0.2 filled with
2094 * the offset. We don't want to leave our offset around in g0 or
2095 * it'll screw up texture samples, so set it up inside the message
2099 brw_push_insn_state(p
);
2100 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2101 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2102 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2103 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2105 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2107 /* set message header global offset field (reg 0, element 2) */
2108 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2109 brw_set_default_swsb(p
, tgl_swsb_null());
2111 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2113 2), BRW_REGISTER_TYPE_UD
),
2114 brw_imm_ud(offset
));
2116 brw_pop_insn_state(p
);
2117 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2121 struct brw_reg dest
;
2122 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2123 int send_commit_msg
;
2124 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2125 BRW_REGISTER_TYPE_UW
);
2127 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2128 brw_inst_set_compression(devinfo
, insn
, false);
2130 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2131 src_header
= vec16(src_header
);
2133 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2134 if (devinfo
->gen
< 6)
2135 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2137 /* Until gen6, writes followed by reads from the same location
2138 * are not guaranteed to be ordered unless write_commit is set.
2139 * If set, then a no-op write is issued to the destination
2140 * register to set a dependency, and a read from the destination
2141 * can be used to ensure the ordering.
2143 * For gen6, only writes between different threads need ordering
2144 * protection. Our use of DP writes is all about register
2145 * spilling within a thread.
2147 if (devinfo
->gen
>= 6) {
2148 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2149 send_commit_msg
= 0;
2152 send_commit_msg
= 1;
2155 brw_set_dest(p
, insn
, dest
);
2156 if (devinfo
->gen
>= 6) {
2157 brw_set_src0(p
, insn
, mrf
);
2159 brw_set_src0(p
, insn
, brw_null_reg());
2162 if (devinfo
->gen
>= 6)
2163 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2165 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2167 brw_set_desc(p
, insn
,
2168 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2169 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2170 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2171 msg_type
, 0, /* not a render target */
2178 * Read a block of owords (half a GRF each) from the scratch buffer
2179 * using a constant index per channel.
2181 * Offset must be aligned to oword size (16 bytes). Used for register
2185 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2186 struct brw_reg dest
,
2191 const struct gen_device_info
*devinfo
= p
->devinfo
;
2192 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2194 if (devinfo
->gen
>= 6)
2197 if (p
->devinfo
->gen
>= 7) {
2198 /* On gen 7 and above, we no longer have message registers and we can
2199 * send from any register we want. By using the destination register
2200 * for the message, we guarantee that the implied message write won't
2201 * accidentally overwrite anything. This has been a problem because
2202 * the MRF registers and source for the final FB write are both fixed
2205 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2207 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2209 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2211 const unsigned rlen
= num_regs
;
2212 const unsigned target_cache
=
2213 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2214 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2215 BRW_SFID_DATAPORT_READ
);
2218 brw_push_insn_state(p
);
2219 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2220 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2221 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2222 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2224 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2226 /* set message header global offset field (reg 0, element 2) */
2227 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2228 brw_set_default_swsb(p
, tgl_swsb_null());
2229 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2231 brw_pop_insn_state(p
);
2232 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2236 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2238 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2239 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2240 brw_inst_set_compression(devinfo
, insn
, false);
2242 brw_set_dest(p
, insn
, dest
); /* UW? */
2243 if (devinfo
->gen
>= 6) {
2244 brw_set_src0(p
, insn
, mrf
);
2246 brw_set_src0(p
, insn
, brw_null_reg());
2247 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2250 brw_set_desc(p
, insn
,
2251 brw_message_desc(devinfo
, 1, rlen
, true) |
2252 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2253 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2254 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2255 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2260 gen7_block_read_scratch(struct brw_codegen
*p
,
2261 struct brw_reg dest
,
2265 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2266 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2268 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2270 /* The HW requires that the header is present; this is to get the g0.5
2273 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2275 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2276 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2277 * is 32 bytes, which happens to be the size of a register.
2280 assert(offset
< (1 << 12));
2282 gen7_set_dp_scratch_message(p
, insn
,
2283 false, /* scratch read */
2285 false, /* invalidate after read */
2288 1, /* mlen: just g0 */
2289 num_regs
, /* rlen */
2290 true); /* header present */
2294 * Read float[4] vectors from the data port constant cache.
2295 * Location (in buffer) should be a multiple of 16.
2296 * Used for fetching shader constants.
2298 void brw_oword_block_read(struct brw_codegen
*p
,
2299 struct brw_reg dest
,
2302 uint32_t bind_table_index
)
2304 const struct gen_device_info
*devinfo
= p
->devinfo
;
2305 const unsigned target_cache
=
2306 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2307 BRW_SFID_DATAPORT_READ
);
2308 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2309 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2311 /* On newer hardware, offset is in units of owords. */
2312 if (devinfo
->gen
>= 6)
2315 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2317 brw_push_insn_state(p
);
2318 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2319 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2320 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2322 brw_push_insn_state(p
);
2323 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2324 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2325 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2327 /* set message header global offset field (reg 0, element 2) */
2328 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2329 brw_set_default_swsb(p
, tgl_swsb_null());
2331 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2333 2), BRW_REGISTER_TYPE_UD
),
2334 brw_imm_ud(offset
));
2335 brw_pop_insn_state(p
);
2337 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2339 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2341 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2343 /* cast dest to a uword[8] vector */
2344 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2346 brw_set_dest(p
, insn
, dest
);
2347 if (devinfo
->gen
>= 6) {
2348 brw_set_src0(p
, insn
, mrf
);
2350 brw_set_src0(p
, insn
, brw_null_reg());
2351 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2354 brw_set_desc(p
, insn
,
2355 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2356 brw_dp_read_desc(devinfo
, bind_table_index
,
2357 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2358 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2359 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2361 brw_pop_insn_state(p
);
2365 brw_fb_WRITE(struct brw_codegen
*p
,
2366 struct brw_reg payload
,
2367 struct brw_reg implied_header
,
2368 unsigned msg_control
,
2369 unsigned binding_table_index
,
2370 unsigned msg_length
,
2371 unsigned response_length
,
2373 bool last_render_target
,
2374 bool header_present
)
2376 const struct gen_device_info
*devinfo
= p
->devinfo
;
2377 const unsigned target_cache
=
2378 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2379 BRW_SFID_DATAPORT_WRITE
);
2382 struct brw_reg dest
, src0
;
2384 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2385 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2387 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2389 if (devinfo
->gen
>= 6) {
2390 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2392 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2394 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2395 brw_inst_set_compression(devinfo
, insn
, false);
2397 if (devinfo
->gen
>= 6) {
2398 /* headerless version, just submit color payload */
2401 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2403 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2404 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2405 src0
= implied_header
;
2407 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2410 brw_set_dest(p
, insn
, dest
);
2411 brw_set_src0(p
, insn
, src0
);
2412 brw_set_desc(p
, insn
,
2413 brw_message_desc(devinfo
, msg_length
, response_length
,
2415 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2416 msg_type
, last_render_target
,
2417 0 /* send_commit_msg */));
2418 brw_inst_set_eot(devinfo
, insn
, eot
);
2424 gen9_fb_READ(struct brw_codegen
*p
,
2426 struct brw_reg payload
,
2427 unsigned binding_table_index
,
2428 unsigned msg_length
,
2429 unsigned response_length
,
2432 const struct gen_device_info
*devinfo
= p
->devinfo
;
2433 assert(devinfo
->gen
>= 9);
2434 const unsigned msg_subtype
=
2435 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2436 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2438 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2439 brw_set_dest(p
, insn
, dst
);
2440 brw_set_src0(p
, insn
, payload
);
2443 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2444 brw_dp_read_desc(devinfo
, binding_table_index
,
2445 per_sample
<< 5 | msg_subtype
,
2446 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2447 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2448 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2454 * Texture sample instruction.
2455 * Note: the msg_type plus msg_length values determine exactly what kind
2456 * of sampling operation is performed. See volume 4, page 161 of docs.
2458 void brw_SAMPLE(struct brw_codegen
*p
,
2459 struct brw_reg dest
,
2460 unsigned msg_reg_nr
,
2461 struct brw_reg src0
,
2462 unsigned binding_table_index
,
2465 unsigned response_length
,
2466 unsigned msg_length
,
2467 unsigned header_present
,
2469 unsigned return_format
)
2471 const struct gen_device_info
*devinfo
= p
->devinfo
;
2474 if (msg_reg_nr
!= -1)
2475 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2477 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2478 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2479 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2481 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2483 * "Instruction compression is not allowed for this instruction (that
2484 * is, send). The hardware behavior is undefined if this instruction is
2485 * set as compressed. However, compress control can be set to "SecHalf"
2486 * to affect the EMask generation."
2488 * No similar wording is found in later PRMs, but there are examples
2489 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2490 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2491 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2493 brw_inst_set_compression(devinfo
, insn
, false);
2495 if (devinfo
->gen
< 6)
2496 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2498 brw_set_dest(p
, insn
, dest
);
2499 brw_set_src0(p
, insn
, src0
);
2500 brw_set_desc(p
, insn
,
2501 brw_message_desc(devinfo
, msg_length
, response_length
,
2503 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2504 msg_type
, simd_mode
, return_format
));
2507 /* Adjust the message header's sampler state pointer to
2508 * select the correct group of 16 samplers.
2510 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2511 struct brw_reg header
,
2512 struct brw_reg sampler_index
)
2514 /* The "Sampler Index" field can only store values between 0 and 15.
2515 * However, we can add an offset to the "Sampler State Pointer"
2516 * field, effectively selecting a different set of 16 samplers.
2518 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2519 * offset, and each sampler state is only 16-bytes, so we can't
2520 * exclusively use the offset - we have to use both.
2523 const struct gen_device_info
*devinfo
= p
->devinfo
;
2525 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2526 const int sampler_state_size
= 16; /* 16 bytes */
2527 uint32_t sampler
= sampler_index
.ud
;
2529 if (sampler
>= 16) {
2530 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2532 get_element_ud(header
, 3),
2533 get_element_ud(brw_vec8_grf(0, 0), 3),
2534 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2537 /* Non-const sampler array indexing case */
2538 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2542 struct brw_reg temp
= get_element_ud(header
, 3);
2544 brw_push_insn_state(p
);
2545 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2546 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
2547 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2549 get_element_ud(header
, 3),
2550 get_element_ud(brw_vec8_grf(0, 0), 3),
2552 brw_pop_insn_state(p
);
2556 /* All these variables are pretty confusing - we might be better off
2557 * using bitmasks and macros for this, in the old style. Or perhaps
2558 * just having the caller instantiate the fields in dword3 itself.
2560 void brw_urb_WRITE(struct brw_codegen
*p
,
2561 struct brw_reg dest
,
2562 unsigned msg_reg_nr
,
2563 struct brw_reg src0
,
2564 enum brw_urb_write_flags flags
,
2565 unsigned msg_length
,
2566 unsigned response_length
,
2570 const struct gen_device_info
*devinfo
= p
->devinfo
;
2573 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2575 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2576 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2577 brw_push_insn_state(p
);
2578 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2579 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2580 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2581 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2582 BRW_REGISTER_TYPE_UD
),
2583 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2584 brw_imm_ud(0xff00));
2585 brw_pop_insn_state(p
);
2588 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2590 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2592 brw_set_dest(p
, insn
, dest
);
2593 brw_set_src0(p
, insn
, src0
);
2594 brw_set_src1(p
, insn
, brw_imm_d(0));
2596 if (devinfo
->gen
< 6)
2597 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2599 brw_set_urb_message(p
,
2609 brw_send_indirect_message(struct brw_codegen
*p
,
2612 struct brw_reg payload
,
2613 struct brw_reg desc
,
2617 const struct gen_device_info
*devinfo
= p
->devinfo
;
2618 struct brw_inst
*send
;
2620 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2622 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2624 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2625 send
= next_insn(p
, BRW_OPCODE_SEND
);
2626 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2627 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2629 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2630 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2632 brw_push_insn_state(p
);
2633 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2634 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2635 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2636 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2637 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2639 /* Load the indirect descriptor to an address register using OR so the
2640 * caller can specify additional descriptor bits with the desc_imm
2643 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2645 brw_pop_insn_state(p
);
2647 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2648 send
= next_insn(p
, BRW_OPCODE_SEND
);
2649 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2651 if (devinfo
->gen
>= 12)
2652 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, true);
2654 brw_set_src1(p
, send
, addr
);
2657 brw_set_dest(p
, send
, dst
);
2658 brw_inst_set_sfid(devinfo
, send
, sfid
);
2659 brw_inst_set_eot(devinfo
, send
, eot
);
2663 brw_send_indirect_split_message(struct brw_codegen
*p
,
2666 struct brw_reg payload0
,
2667 struct brw_reg payload1
,
2668 struct brw_reg desc
,
2670 struct brw_reg ex_desc
,
2671 unsigned ex_desc_imm
,
2674 const struct gen_device_info
*devinfo
= p
->devinfo
;
2675 struct brw_inst
*send
;
2677 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2679 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2681 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2682 desc
.ud
|= desc_imm
;
2684 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2685 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2687 brw_push_insn_state(p
);
2688 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2689 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2690 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2691 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2692 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2694 /* Load the indirect descriptor to an address register using OR so the
2695 * caller can specify additional descriptor bits with the desc_imm
2698 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2700 brw_pop_insn_state(p
);
2703 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2706 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
&&
2707 (ex_desc
.ud
& INTEL_MASK(15, 12)) == 0) {
2708 ex_desc
.ud
|= ex_desc_imm
;
2710 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2711 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2713 brw_push_insn_state(p
);
2714 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2715 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2716 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2717 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2718 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2720 /* Load the indirect extended descriptor to an address register using OR
2721 * so the caller can specify additional descriptor bits with the
2722 * desc_imm immediate.
2724 * Even though the instruction dispatcher always pulls the SFID and EOT
2725 * fields from the instruction itself, actual external unit which
2726 * processes the message gets the SFID and EOT from the extended
2727 * descriptor which comes from the address register. If we don't OR
2728 * those two bits in, the external unit may get confused and hang.
2730 unsigned imm_part
= ex_desc_imm
| sfid
| eot
<< 5;
2732 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2733 /* ex_desc bits 15:12 don't exist in the instruction encoding, so
2734 * we may have fallen back to an indirect extended descriptor.
2736 brw_MOV(p
, addr
, brw_imm_ud(ex_desc
.ud
| imm_part
));
2738 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(imm_part
));
2741 brw_pop_insn_state(p
);
2744 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2747 send
= next_insn(p
, devinfo
->gen
>= 12 ? BRW_OPCODE_SEND
: BRW_OPCODE_SENDS
);
2748 brw_set_dest(p
, send
, dst
);
2749 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2750 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2752 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2753 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2754 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2756 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2757 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2758 assert(desc
.subnr
== 0);
2759 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2762 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2763 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2764 brw_inst_set_sends_ex_desc(devinfo
, send
, ex_desc
.ud
);
2766 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2767 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2768 assert((ex_desc
.subnr
& 0x3) == 0);
2769 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2770 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2773 brw_inst_set_sfid(devinfo
, send
, sfid
);
2774 brw_inst_set_eot(devinfo
, send
, eot
);
2778 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2781 struct brw_reg payload
,
2782 struct brw_reg surface
,
2785 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2786 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2787 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2789 brw_push_insn_state(p
);
2790 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2791 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2792 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2793 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2794 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2796 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2797 * some surface array is accessed out of bounds.
2800 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2801 BRW_GET_SWZ(surface
.swizzle
, 0)),
2804 brw_pop_insn_state(p
);
2807 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2810 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2814 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2815 brw_inst
*insn
, int while_offset
, int start_offset
)
2817 int scale
= 16 / brw_jump_scale(devinfo
);
2818 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2819 : brw_inst_jip(devinfo
, insn
);
2821 return while_offset
+ jip
* scale
<= start_offset
;
2826 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2829 void *store
= p
->store
;
2830 const struct gen_device_info
*devinfo
= p
->devinfo
;
2834 for (offset
= next_offset(devinfo
, store
, start_offset
);
2835 offset
< p
->next_insn_offset
;
2836 offset
= next_offset(devinfo
, store
, offset
)) {
2837 brw_inst
*insn
= store
+ offset
;
2839 switch (brw_inst_opcode(devinfo
, insn
)) {
2843 case BRW_OPCODE_ENDIF
:
2848 case BRW_OPCODE_WHILE
:
2849 /* If the while doesn't jump before our instruction, it's the end
2850 * of a sibling do...while loop. Ignore it.
2852 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2855 case BRW_OPCODE_ELSE
:
2856 case BRW_OPCODE_HALT
:
2867 /* There is no DO instruction on gen6, so to find the end of the loop
2868 * we have to see if the loop is jumping back before our start
2872 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2874 const struct gen_device_info
*devinfo
= p
->devinfo
;
2876 void *store
= p
->store
;
2878 assert(devinfo
->gen
>= 6);
2880 /* Always start after the instruction (such as a WHILE) we're trying to fix
2883 for (offset
= next_offset(devinfo
, store
, start_offset
);
2884 offset
< p
->next_insn_offset
;
2885 offset
= next_offset(devinfo
, store
, offset
)) {
2886 brw_inst
*insn
= store
+ offset
;
2888 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2889 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2893 assert(!"not reached");
2894 return start_offset
;
2897 /* After program generation, go back and update the UIP and JIP of
2898 * BREAK, CONT, and HALT instructions to their correct locations.
2901 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2903 const struct gen_device_info
*devinfo
= p
->devinfo
;
2905 int br
= brw_jump_scale(devinfo
);
2906 int scale
= 16 / br
;
2907 void *store
= p
->store
;
2909 if (devinfo
->gen
< 6)
2912 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2913 brw_inst
*insn
= store
+ offset
;
2914 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2916 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2917 switch (brw_inst_opcode(devinfo
, insn
)) {
2918 case BRW_OPCODE_BREAK
:
2919 assert(block_end_offset
!= 0);
2920 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2921 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2922 brw_inst_set_uip(devinfo
, insn
,
2923 (brw_find_loop_end(p
, offset
) - offset
+
2924 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2926 case BRW_OPCODE_CONTINUE
:
2927 assert(block_end_offset
!= 0);
2928 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2929 brw_inst_set_uip(devinfo
, insn
,
2930 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2932 assert(brw_inst_uip(devinfo
, insn
) != 0);
2933 assert(brw_inst_jip(devinfo
, insn
) != 0);
2936 case BRW_OPCODE_ENDIF
: {
2937 int32_t jump
= (block_end_offset
== 0) ?
2938 1 * br
: (block_end_offset
- offset
) / scale
;
2939 if (devinfo
->gen
>= 7)
2940 brw_inst_set_jip(devinfo
, insn
, jump
);
2942 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2946 case BRW_OPCODE_HALT
:
2947 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2949 * "In case of the halt instruction not inside any conditional
2950 * code block, the value of <JIP> and <UIP> should be the
2951 * same. In case of the halt instruction inside conditional code
2952 * block, the <UIP> should be the end of the program, and the
2953 * <JIP> should be end of the most inner conditional code block."
2955 * The uip will have already been set by whoever set up the
2958 if (block_end_offset
== 0) {
2959 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2961 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2963 assert(brw_inst_uip(devinfo
, insn
) != 0);
2964 assert(brw_inst_jip(devinfo
, insn
) != 0);
2973 void brw_ff_sync(struct brw_codegen
*p
,
2974 struct brw_reg dest
,
2975 unsigned msg_reg_nr
,
2976 struct brw_reg src0
,
2978 unsigned response_length
,
2981 const struct gen_device_info
*devinfo
= p
->devinfo
;
2984 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2986 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2987 brw_set_dest(p
, insn
, dest
);
2988 brw_set_src0(p
, insn
, src0
);
2989 brw_set_src1(p
, insn
, brw_imm_d(0));
2991 if (devinfo
->gen
< 6)
2992 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2994 brw_set_ff_sync_message(p
,
3002 * Emit the SEND instruction necessary to generate stream output data on Gen6
3003 * (for transform feedback).
3005 * If send_commit_msg is true, this is the last piece of stream output data
3006 * from this thread, so send the data as a committed write. According to the
3007 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
3009 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
3010 * writes are complete by sending the final write as a committed write."
3013 brw_svb_write(struct brw_codegen
*p
,
3014 struct brw_reg dest
,
3015 unsigned msg_reg_nr
,
3016 struct brw_reg src0
,
3017 unsigned binding_table_index
,
3018 bool send_commit_msg
)
3020 const struct gen_device_info
*devinfo
= p
->devinfo
;
3021 const unsigned target_cache
=
3022 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
3023 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
3024 BRW_SFID_DATAPORT_WRITE
);
3027 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
3029 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3030 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
3031 brw_set_dest(p
, insn
, dest
);
3032 brw_set_src0(p
, insn
, src0
);
3033 brw_set_desc(p
, insn
,
3034 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
3035 brw_dp_write_desc(devinfo
, binding_table_index
,
3036 0, /* msg_control: ignored */
3037 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
3038 0, /* last_render_target: ignored */
3039 send_commit_msg
)); /* send_commit_msg */
3043 brw_surface_payload_size(struct brw_codegen
*p
,
3044 unsigned num_channels
,
3045 unsigned exec_size
/**< 0 for SIMD4x2 */)
3048 return 1; /* SIMD4x2 */
3049 else if (exec_size
<= 8)
3050 return num_channels
;
3052 return 2 * num_channels
;
3056 brw_untyped_atomic(struct brw_codegen
*p
,
3058 struct brw_reg payload
,
3059 struct brw_reg surface
,
3061 unsigned msg_length
,
3062 bool response_expected
,
3063 bool header_present
)
3065 const struct gen_device_info
*devinfo
= p
->devinfo
;
3066 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3067 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3068 GEN7_SFID_DATAPORT_DATA_CACHE
);
3069 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3070 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
3071 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3072 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3073 has_simd4x2
? 0 : 8;
3074 const unsigned response_length
=
3075 brw_surface_payload_size(p
, response_expected
, exec_size
);
3076 const unsigned desc
=
3077 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3078 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
3080 /* Mask out unused components -- This is especially important in Align16
3081 * mode on generations that don't have native support for SIMD4x2 atomics,
3082 * because unused but enabled components will cause the dataport to perform
3083 * additional atomic operations on the addresses that happen to be in the
3084 * uninitialized Y, Z and W coordinates of the payload.
3086 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3088 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
3089 payload
, surface
, desc
);
3093 brw_untyped_surface_read(struct brw_codegen
*p
,
3095 struct brw_reg payload
,
3096 struct brw_reg surface
,
3097 unsigned msg_length
,
3098 unsigned num_channels
)
3100 const struct gen_device_info
*devinfo
= p
->devinfo
;
3101 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3102 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3103 GEN7_SFID_DATAPORT_DATA_CACHE
);
3104 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3105 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
3106 const unsigned response_length
=
3107 brw_surface_payload_size(p
, num_channels
, exec_size
);
3108 const unsigned desc
=
3109 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3110 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
3112 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
3116 brw_untyped_surface_write(struct brw_codegen
*p
,
3117 struct brw_reg payload
,
3118 struct brw_reg surface
,
3119 unsigned msg_length
,
3120 unsigned num_channels
,
3121 bool header_present
)
3123 const struct gen_device_info
*devinfo
= p
->devinfo
;
3124 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3125 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3126 GEN7_SFID_DATAPORT_DATA_CACHE
);
3127 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3128 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
3129 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3130 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3131 has_simd4x2
? 0 : 8;
3132 const unsigned desc
=
3133 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3134 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
3135 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3136 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
3138 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3139 payload
, surface
, desc
);
3143 brw_set_memory_fence_message(struct brw_codegen
*p
,
3144 struct brw_inst
*insn
,
3145 enum brw_message_target sfid
,
3149 const struct gen_device_info
*devinfo
= p
->devinfo
;
3151 brw_set_desc(p
, insn
, brw_message_desc(
3152 devinfo
, 1, (commit_enable
? 1 : 0), true));
3154 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3157 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3158 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3160 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3161 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3164 unreachable("Not reached");
3168 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3170 assert(devinfo
->gen
>= 11 || bti
== 0);
3171 brw_inst_set_binding_table_index(devinfo
, insn
, bti
);
3175 brw_memory_fence(struct brw_codegen
*p
,
3178 enum opcode send_op
,
3182 const struct gen_device_info
*devinfo
= p
->devinfo
;
3183 const bool commit_enable
= stall
||
3184 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3185 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3186 struct brw_inst
*insn
;
3188 brw_push_insn_state(p
);
3189 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3190 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3191 dst
= retype(vec1(dst
), BRW_REGISTER_TYPE_UW
);
3192 src
= retype(vec1(src
), BRW_REGISTER_TYPE_UD
);
3194 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3195 * message doesn't write anything back.
3197 insn
= next_insn(p
, send_op
);
3198 brw_set_dest(p
, insn
, dst
);
3199 brw_set_src0(p
, insn
, src
);
3200 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3201 commit_enable
, bti
);
3203 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3204 /* IVB does typed surface access through the render cache, so we need to
3205 * flush it too. Use a different register so both flushes can be
3206 * pipelined by the hardware.
3208 insn
= next_insn(p
, send_op
);
3209 brw_set_dest(p
, insn
, offset(dst
, 1));
3210 brw_set_src0(p
, insn
, src
);
3211 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3212 commit_enable
, bti
);
3214 /* Now write the response of the second message into the response of the
3215 * first to trigger a pipeline stall -- This way future render and data
3216 * cache messages will be properly ordered with respect to past data and
3217 * render cache messages.
3219 brw_MOV(p
, dst
, offset(dst
, 1));
3223 brw_set_default_swsb(p
, tgl_swsb_sbid(TGL_SBID_DST
,
3224 brw_get_default_swsb(p
).sbid
));
3226 brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
), dst
);
3229 brw_pop_insn_state(p
);
3233 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3234 struct brw_reg dest
,
3238 struct brw_reg data
,
3239 unsigned msg_length
,
3240 unsigned response_length
)
3242 const struct gen_device_info
*devinfo
= p
->devinfo
;
3243 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3244 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3245 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3246 const unsigned desc
=
3247 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3248 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3251 /* brw_send_indirect_message will automatically use a direct send message
3252 * if data is actually immediate.
3254 brw_send_indirect_message(p
,
3255 GEN7_SFID_PIXEL_INTERPOLATOR
,
3264 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3265 struct brw_reg mask
)
3267 const struct gen_device_info
*devinfo
= p
->devinfo
;
3268 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3269 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3272 assert(devinfo
->gen
>= 7);
3273 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3275 brw_push_insn_state(p
);
3277 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3278 * unnecessary bits in the instruction words, get the information we need
3279 * and reset the default flag register. This allows more instructions to be
3282 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3283 brw_set_default_flag_reg(p
, 0, 0);
3285 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3286 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3288 if (devinfo
->gen
>= 8) {
3289 /* Getting the first active channel index is easy on Gen8: Just find
3290 * the first bit set in the execution mask. The register exists on
3291 * HSW already but it reads back as all ones when the current
3292 * instruction has execution masking disabled, so it's kind of
3295 struct brw_reg exec_mask
=
3296 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3298 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3299 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3300 /* Unfortunately, ce0 does not take into account the thread
3301 * dispatch mask, which may be a problem in cases where it's not
3302 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3303 * some n). Combine ce0 with the given dispatch (or vector) mask
3304 * to mask off those channels which were never dispatched by the
3307 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3308 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3309 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3310 exec_mask
= vec1(dst
);
3313 /* Quarter control has the effect of magically shifting the value of
3314 * ce0 so you'll get the first active channel relative to the
3315 * specified quarter control as result.
3317 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3319 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3321 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3322 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3324 /* Run enough instructions returning zero with execution masking and
3325 * a conditional modifier enabled in order to get the full execution
3326 * mask in f1.0. We could use a single 32-wide move here if it
3327 * weren't because of the hardware bug that causes channel enables to
3328 * be applied incorrectly to the second half of 32-wide instructions
3331 const unsigned lower_size
= MIN2(16, exec_size
);
3332 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3333 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3335 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3336 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3337 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3338 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3339 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3340 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3343 /* Find the first bit set in the exec_size-wide portion of the flag
3344 * register that was updated by the last sequence of MOV
3347 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3348 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3349 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3352 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3354 if (devinfo
->gen
>= 8 &&
3355 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3356 /* In SIMD4x2 mode the first active channel index is just the
3357 * negation of the first bit of the mask register. Note that ce0
3358 * doesn't take into account the dispatch mask, so the Gen7 path
3359 * should be used instead unless you have the guarantee that the
3360 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3363 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3364 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3368 /* Overwrite the destination without and with execution masking to
3369 * find out which of the channels is active.
3371 brw_push_insn_state(p
);
3372 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3373 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3376 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3378 brw_pop_insn_state(p
);
3379 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3383 brw_pop_insn_state(p
);
3387 brw_broadcast(struct brw_codegen
*p
,
3392 const struct gen_device_info
*devinfo
= p
->devinfo
;
3393 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3396 brw_push_insn_state(p
);
3397 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3398 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3400 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3401 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3402 assert(!src
.abs
&& !src
.negate
);
3403 assert(src
.type
== dst
.type
);
3405 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3406 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3407 /* Trivial, the source is already uniform or the index is a constant.
3408 * We will typically not get here if the optimizer is doing its job, but
3409 * asserting would be mean.
3411 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3413 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3414 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3416 /* From the Haswell PRM section "Register Region Restrictions":
3418 * "The lower bits of the AddressImmediate must not overflow to
3419 * change the register address. The lower 5 bits of Address
3420 * Immediate when added to lower 5 bits of address register gives
3421 * the sub-register offset. The upper bits of Address Immediate
3422 * when added to upper bits of address register gives the register
3423 * address. Any overflow from sub-register offset is dropped."
3425 * Fortunately, for broadcast, we never have a sub-register offset so
3426 * this isn't an issue.
3428 assert(src
.subnr
== 0);
3431 const struct brw_reg addr
=
3432 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3433 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3434 /* Limit in bytes of the signed indirect addressing immediate. */
3435 const unsigned limit
= 512;
3437 brw_push_insn_state(p
);
3438 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3439 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3441 /* Take into account the component size and horizontal stride. */
3442 assert(src
.vstride
== src
.hstride
+ src
.width
);
3443 brw_SHL(p
, addr
, vec1(idx
),
3444 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3447 /* We can only address up to limit bytes using the indirect
3448 * addressing immediate, account for the difference if the source
3449 * register is above this limit.
3451 if (offset
>= limit
) {
3452 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3453 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3454 offset
= offset
% limit
;
3457 brw_pop_insn_state(p
);
3459 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3461 /* Use indirect addressing to fetch the specified component. */
3462 if (type_sz(src
.type
) > 4 &&
3463 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3464 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3466 * "When source or destination datatype is 64b or operation is
3467 * integer DWord multiply, indirect addressing must not be
3470 * To work around both of this issue, we do two integer MOVs
3471 * insead of one 64-bit MOV. Because no double value should ever
3472 * cross a register boundary, it's safe to use the immediate
3473 * offset in the indirect here to handle adding 4 bytes to the
3474 * offset and avoid the extra ADD to the register file.
3476 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3477 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3478 BRW_REGISTER_TYPE_D
));
3479 brw_set_default_swsb(p
, tgl_swsb_null());
3480 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3481 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3482 BRW_REGISTER_TYPE_D
));
3485 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3488 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3489 * to all bits of a flag register,
3493 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3494 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3495 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3496 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3498 /* and use predicated SEL to pick the right channel. */
3499 inst
= brw_SEL(p
, dst
,
3500 stride(suboffset(src
, 4), 4, 4, 1),
3501 stride(src
, 4, 4, 1));
3502 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3503 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3507 brw_pop_insn_state(p
);
3511 * This instruction is generated as a single-channel align1 instruction by
3512 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3514 * We can't use the typed atomic op in the FS because that has the execution
3515 * mask ANDed with the pixel mask, but we just want to write the one dword for
3518 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3519 * one u32. So we use the same untyped atomic write message as the pixel
3522 * The untyped atomic operation requires a BUFFER surface type with RAW
3523 * format, and is only accessible through the legacy DATA_CACHE dataport
3526 void brw_shader_time_add(struct brw_codegen
*p
,
3527 struct brw_reg payload
,
3528 uint32_t surf_index
)
3530 const struct gen_device_info
*devinfo
= p
->devinfo
;
3531 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3532 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3533 GEN7_SFID_DATAPORT_DATA_CACHE
);
3534 assert(devinfo
->gen
>= 7);
3536 brw_push_insn_state(p
);
3537 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3538 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3539 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3540 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3542 /* We use brw_vec1_reg and unmasked because we want to increment the given
3545 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3547 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3549 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3550 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3553 brw_inst_set_sfid(devinfo
, send
, sfid
);
3554 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3556 brw_pop_insn_state(p
);
3561 * Emit the SEND message for a barrier
3564 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3566 const struct gen_device_info
*devinfo
= p
->devinfo
;
3567 struct brw_inst
*inst
;
3569 assert(devinfo
->gen
>= 7);
3571 brw_push_insn_state(p
);
3572 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3573 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3574 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3575 brw_set_src0(p
, inst
, src
);
3576 brw_set_src1(p
, inst
, brw_null_reg());
3577 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3579 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3580 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3581 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3583 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3584 brw_pop_insn_state(p
);
3589 * Emit the wait instruction for a barrier
3592 brw_WAIT(struct brw_codegen
*p
)
3594 const struct gen_device_info
*devinfo
= p
->devinfo
;
3595 struct brw_inst
*insn
;
3597 struct brw_reg src
= brw_notification_reg();
3599 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3600 brw_set_dest(p
, insn
, src
);
3601 brw_set_src0(p
, insn
, src
);
3602 brw_set_src1(p
, insn
, brw_null_reg());
3604 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3605 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3609 brw_float_controls_mode(struct brw_codegen
*p
,
3610 unsigned mode
, unsigned mask
)
3612 /* From the Skylake PRM, Volume 7, page 760:
3613 * "Implementation Restriction on Register Access: When the control
3614 * register is used as an explicit source and/or destination, hardware
3615 * does not ensure execution pipeline coherency. Software must set the
3616 * thread control field to ‘switch’ for an instruction that uses
3617 * control register as an explicit operand."
3619 * On Gen12+ this is implemented in terms of SWSB annotations instead.
3621 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3623 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3625 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3626 if (p
->devinfo
->gen
< 12)
3627 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3630 brw_inst
*inst_or
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3632 brw_inst_set_exec_size(p
->devinfo
, inst_or
, BRW_EXECUTE_1
);
3633 if (p
->devinfo
->gen
< 12)
3634 brw_inst_set_thread_control(p
->devinfo
, inst_or
, BRW_THREAD_SWITCH
);
3637 if (p
->devinfo
->gen
>= 12)
3638 brw_SYNC(p
, TGL_SYNC_NOP
);