2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 assert(devinfo
->gen
< 12);
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct gen_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
91 const struct gen_device_info
*devinfo
= p
->devinfo
;
93 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
94 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
95 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
96 assert(dest
.nr
< 128);
98 /* The hardware has a restriction where a destination of size Byte with
99 * a stride of 1 is only allowed for a packed byte MOV. For any other
100 * instruction, the stride must be at least 2, even when the destination
101 * is the NULL register.
103 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
104 dest
.nr
== BRW_ARF_NULL
&&
105 type_sz(dest
.type
) == 1 &&
106 dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
) {
107 dest
.hstride
= BRW_HORIZONTAL_STRIDE_2
;
110 gen7_convert_mrf_to_grf(p
, &dest
);
112 if (devinfo
->gen
>= 12 &&
113 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
114 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
115 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
116 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
117 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
118 assert(dest
.subnr
== 0);
119 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
120 (dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
121 dest
.vstride
== dest
.width
+ 1));
122 assert(!dest
.negate
&& !dest
.abs
);
123 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
124 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
126 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
127 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
128 assert(devinfo
->gen
< 12);
129 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
130 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
131 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
132 assert(dest
.subnr
% 16 == 0);
133 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
134 dest
.vstride
== dest
.width
+ 1);
135 assert(!dest
.negate
&& !dest
.abs
);
136 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
137 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
138 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
140 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
141 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
143 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
144 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
146 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
147 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
148 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
149 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
150 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
152 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
153 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
154 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
155 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
156 assert(dest
.writemask
!= 0);
158 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
159 * Although Dst.HorzStride is a don't care for Align16, HW needs
160 * this to be programmed as "01".
162 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
165 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
167 /* These are different sizes in align1 vs align16:
169 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
170 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
171 dest
.indirect_offset
);
172 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
173 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
174 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
176 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
177 dest
.indirect_offset
);
178 /* even ignored in da16, still need to set as '01' */
179 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
184 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
185 * or 16 (SIMD16), as that's normally correct. However, when dealing with
186 * small registers, it can be useful for us to automatically reduce it to
187 * match the register size.
189 if (p
->automatic_exec_sizes
) {
191 * In platforms that support fp64 we can emit instructions with a width
192 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
193 * these cases we need to make sure that these instructions have their
194 * exec sizes set properly when they are emitted and we can't rely on
195 * this code to fix it.
198 if (devinfo
->gen
>= 6)
199 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
201 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
204 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
209 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
211 const struct gen_device_info
*devinfo
= p
->devinfo
;
213 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
214 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
215 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
216 assert(reg
.nr
< 128);
218 gen7_convert_mrf_to_grf(p
, ®
);
220 if (devinfo
->gen
>= 6 &&
221 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
222 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
223 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
224 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
225 /* Any source modifiers or regions will be ignored, since this just
226 * identifies the MRF/GRF to start reading the message contents from.
227 * Check for some likely failures.
231 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
234 if (devinfo
->gen
>= 12 &&
235 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
236 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
237 assert(reg
.file
!= BRW_IMMEDIATE_VALUE
);
238 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
239 assert(reg
.subnr
== 0);
240 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
241 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
242 reg
.vstride
== reg
.width
+ 1));
243 assert(!reg
.negate
&& !reg
.abs
);
244 brw_inst_set_send_src0_reg_file(devinfo
, inst
, reg
.file
);
245 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
247 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
248 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
249 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
250 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
251 assert(reg
.subnr
% 16 == 0);
252 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
253 reg
.vstride
== reg
.width
+ 1);
254 assert(!reg
.negate
&& !reg
.abs
);
255 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
256 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
258 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
259 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
260 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
261 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
263 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
264 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
265 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
266 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
267 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
268 reg
.type
== BRW_REGISTER_TYPE_Q
)
269 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
271 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
273 if (devinfo
->gen
< 12 && type_sz(reg
.type
) < 8) {
274 brw_inst_set_src1_reg_file(devinfo
, inst
,
275 BRW_ARCHITECTURE_REGISTER_FILE
);
276 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
277 brw_inst_src0_reg_hw_type(devinfo
, inst
));
280 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
281 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
282 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
283 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
285 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
288 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
290 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
291 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
293 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
297 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
298 if (reg
.width
== BRW_WIDTH_1
&&
299 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
300 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
301 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
302 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
304 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
305 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
306 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
309 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
310 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
311 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
312 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
313 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
314 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
315 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
316 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
318 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
319 /* This is an oddity of the fact we're using the same
320 * descriptions for registers in align_16 as align_1:
322 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
323 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
324 reg
.type
== BRW_REGISTER_TYPE_DF
&&
325 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
328 * "For Align16 access mode, only encodings of 0000 and 0011
329 * are allowed. Other codes are reserved."
331 * Presumably the DevSNB behavior applies to IVB as well.
333 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
335 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
344 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
346 const struct gen_device_info
*devinfo
= p
->devinfo
;
348 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
349 assert(reg
.nr
< 128);
351 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
352 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
||
353 (devinfo
->gen
>= 12 &&
354 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
355 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
))) {
356 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
357 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
358 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
359 assert(reg
.subnr
== 0);
360 assert(brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
||
361 (reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
362 reg
.vstride
== reg
.width
+ 1));
363 assert(!reg
.negate
&& !reg
.abs
);
364 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
365 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
367 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
369 * "Accumulator registers may be accessed explicitly as src0
372 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
373 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
375 gen7_convert_mrf_to_grf(p
, ®
);
376 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
378 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
379 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
380 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
382 /* Only src1 can be immediate in two-argument instructions.
384 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
386 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
387 /* two-argument instructions can only use 32-bit immediates */
388 assert(type_sz(reg
.type
) < 8);
389 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
391 /* This is a hardware restriction, which may or may not be lifted
394 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
395 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
397 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
398 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
399 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
401 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
404 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
405 if (reg
.width
== BRW_WIDTH_1
&&
406 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
407 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
408 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
409 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
411 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
412 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
413 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
416 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
417 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
418 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
419 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
420 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
421 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
422 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
423 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
425 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
426 /* This is an oddity of the fact we're using the same
427 * descriptions for registers in align_16 as align_1:
429 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
430 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
431 reg
.type
== BRW_REGISTER_TYPE_DF
&&
432 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
435 * "For Align16 access mode, only encodings of 0000 and 0011
436 * are allowed. Other codes are reserved."
438 * Presumably the DevSNB behavior applies to IVB as well.
440 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
442 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
450 * Specify the descriptor and extended descriptor immediate for a SEND(C)
451 * message instruction.
454 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
455 unsigned desc
, unsigned ex_desc
)
457 const struct gen_device_info
*devinfo
= p
->devinfo
;
458 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
459 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
460 if (devinfo
->gen
< 12)
461 brw_inst_set_src1_file_type(devinfo
, inst
,
462 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
463 brw_inst_set_send_desc(devinfo
, inst
, desc
);
464 if (devinfo
->gen
>= 9)
465 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
468 static void brw_set_math_message( struct brw_codegen
*p
,
471 unsigned integer_type
,
475 const struct gen_device_info
*devinfo
= p
->devinfo
;
477 unsigned response_length
;
479 /* Infer message length from the function */
481 case BRW_MATH_FUNCTION_POW
:
482 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
483 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
484 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
492 /* Infer response length from the function */
494 case BRW_MATH_FUNCTION_SINCOS
:
495 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
503 brw_set_desc(p
, inst
, brw_message_desc(
504 devinfo
, msg_length
, response_length
, false));
506 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
507 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
508 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
509 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
510 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
511 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
512 brw_inst_set_saturate(devinfo
, inst
, 0);
516 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
519 unsigned response_length
,
522 const struct gen_device_info
*devinfo
= p
->devinfo
;
524 brw_set_desc(p
, insn
, brw_message_desc(
525 devinfo
, 1, response_length
, true));
527 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
528 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
529 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
530 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
531 /* The following fields are not used by FF_SYNC: */
532 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
533 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
534 brw_inst_set_urb_used(devinfo
, insn
, 0);
535 brw_inst_set_urb_complete(devinfo
, insn
, 0);
538 static void brw_set_urb_message( struct brw_codegen
*p
,
540 enum brw_urb_write_flags flags
,
542 unsigned response_length
,
544 unsigned swizzle_control
)
546 const struct gen_device_info
*devinfo
= p
->devinfo
;
548 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
549 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
550 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
552 brw_set_desc(p
, insn
, brw_message_desc(
553 devinfo
, msg_length
, response_length
, true));
555 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
556 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
558 if (flags
& BRW_URB_WRITE_OWORD
) {
559 assert(msg_length
== 2); /* header + one OWORD of data */
560 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
562 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
565 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
566 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
568 if (devinfo
->gen
< 8) {
569 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
572 if (devinfo
->gen
< 7) {
573 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
574 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
576 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
577 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
582 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
586 bool invalidate_after_read
,
588 unsigned addr_offset
,
593 const struct gen_device_info
*devinfo
= p
->devinfo
;
594 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
595 (devinfo
->gen
>= 8 && num_regs
== 8));
596 const unsigned block_size
= (devinfo
->gen
>= 8 ? util_logbase2(num_regs
) :
599 brw_set_desc(p
, inst
, brw_message_desc(
600 devinfo
, mlen
, rlen
, header_present
));
602 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
603 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
604 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
605 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
606 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
607 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
608 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
612 brw_inst_set_state(const struct gen_device_info
*devinfo
,
614 const struct brw_insn_state
*state
)
616 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
617 brw_inst_set_group(devinfo
, insn
, state
->group
);
618 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
619 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
620 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
621 if (devinfo
->gen
>= 12)
622 brw_inst_set_swsb(devinfo
, insn
, tgl_swsb_encode(state
->swsb
));
623 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
624 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
625 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
627 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
628 state
->access_mode
== BRW_ALIGN_16
) {
629 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
630 if (devinfo
->gen
>= 7)
631 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
633 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
634 if (devinfo
->gen
>= 7)
635 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
638 if (devinfo
->gen
>= 6)
639 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
642 #define next_insn brw_next_insn
644 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
646 const struct gen_device_info
*devinfo
= p
->devinfo
;
649 if (p
->nr_insn
+ 1 > p
->store_size
) {
651 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
654 p
->next_insn_offset
+= 16;
655 insn
= &p
->store
[p
->nr_insn
++];
657 memset(insn
, 0, sizeof(*insn
));
658 brw_inst_set_opcode(devinfo
, insn
, opcode
);
660 /* Apply the default instruction state */
661 brw_inst_set_state(devinfo
, insn
, p
->current
);
667 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
668 struct brw_reg dest
, struct brw_reg src
)
670 brw_inst
*insn
= next_insn(p
, opcode
);
671 brw_set_dest(p
, insn
, dest
);
672 brw_set_src0(p
, insn
, src
);
677 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
678 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
680 /* 64-bit immediates are only supported on 1-src instructions */
681 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
682 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
684 brw_inst
*insn
= next_insn(p
, opcode
);
685 brw_set_dest(p
, insn
, dest
);
686 brw_set_src0(p
, insn
, src0
);
687 brw_set_src1(p
, insn
, src1
);
692 get_3src_subreg_nr(struct brw_reg reg
)
694 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
695 * use 32-bit units (components 0..7). Since they only support F/D/UD
696 * types, this doesn't lose any flexibility, but uses fewer bits.
698 return reg
.subnr
/ 4;
701 static enum gen10_align1_3src_vertical_stride
702 to_3src_align1_vstride(const struct gen_device_info
*devinfo
,
703 enum brw_vertical_stride vstride
)
706 case BRW_VERTICAL_STRIDE_0
:
707 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
708 case BRW_VERTICAL_STRIDE_1
:
709 assert(devinfo
->gen
>= 12);
710 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1
;
711 case BRW_VERTICAL_STRIDE_2
:
712 assert(devinfo
->gen
< 12);
713 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
714 case BRW_VERTICAL_STRIDE_4
:
715 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
716 case BRW_VERTICAL_STRIDE_8
:
717 case BRW_VERTICAL_STRIDE_16
:
718 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
720 unreachable("invalid vstride");
725 static enum gen10_align1_3src_src_horizontal_stride
726 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
729 case BRW_HORIZONTAL_STRIDE_0
:
730 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
731 case BRW_HORIZONTAL_STRIDE_1
:
732 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
733 case BRW_HORIZONTAL_STRIDE_2
:
734 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
735 case BRW_HORIZONTAL_STRIDE_4
:
736 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
738 unreachable("invalid hstride");
743 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
744 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
746 const struct gen_device_info
*devinfo
= p
->devinfo
;
747 brw_inst
*inst
= next_insn(p
, opcode
);
749 gen7_convert_mrf_to_grf(p
, &dest
);
751 assert(dest
.nr
< 128);
753 if (devinfo
->gen
>= 10)
754 assert(!(src0
.file
== BRW_IMMEDIATE_VALUE
&&
755 src2
.file
== BRW_IMMEDIATE_VALUE
));
757 assert(src0
.file
== BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
758 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
&& src1
.nr
< 128);
759 assert(src2
.file
== BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
760 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
761 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
762 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
763 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
765 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
766 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
767 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
769 if (devinfo
->gen
>= 12) {
770 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
, dest
.file
);
771 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
773 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
774 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
775 BRW_ALIGN1_3SRC_ACCUMULATOR
);
776 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
778 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
779 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
780 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
783 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
785 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
787 if (brw_reg_type_is_floating_point(dest
.type
)) {
788 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
789 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
791 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
792 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
795 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
796 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
797 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
798 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
800 if (src0
.file
== BRW_IMMEDIATE_VALUE
) {
801 brw_inst_set_3src_a1_src0_imm(devinfo
, inst
, src0
.ud
);
803 brw_inst_set_3src_a1_src0_vstride(
804 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src0
.vstride
));
805 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
806 to_3src_align1_hstride(src0
.hstride
));
807 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
808 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
809 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
811 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
813 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
814 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
816 brw_inst_set_3src_a1_src1_vstride(
817 devinfo
, inst
, to_3src_align1_vstride(devinfo
, src1
.vstride
));
818 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
819 to_3src_align1_hstride(src1
.hstride
));
821 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
822 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
823 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
825 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
827 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
828 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
830 if (src2
.file
== BRW_IMMEDIATE_VALUE
) {
831 brw_inst_set_3src_a1_src2_imm(devinfo
, inst
, src2
.ud
);
833 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
834 to_3src_align1_hstride(src2
.hstride
));
835 /* no vstride on src2 */
836 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
837 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
838 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
839 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
842 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
843 src0
.file
== BRW_IMMEDIATE_VALUE
||
844 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
845 src0
.type
== BRW_REGISTER_TYPE_NF
));
846 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
847 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
848 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
849 src2
.file
== BRW_IMMEDIATE_VALUE
);
851 if (devinfo
->gen
>= 12) {
852 if (src0
.file
== BRW_IMMEDIATE_VALUE
) {
853 brw_inst_set_3src_a1_src0_is_imm(devinfo
, inst
, 1);
855 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
, src0
.file
);
858 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
, src1
.file
);
860 if (src2
.file
== BRW_IMMEDIATE_VALUE
) {
861 brw_inst_set_3src_a1_src2_is_imm(devinfo
, inst
, 1);
863 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
, src2
.file
);
866 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
867 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
868 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
869 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
870 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
871 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
872 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
873 BRW_ALIGN1_3SRC_ACCUMULATOR
);
874 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
875 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
876 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
877 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
881 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
882 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
883 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
884 dest
.type
== BRW_REGISTER_TYPE_DF
||
885 dest
.type
== BRW_REGISTER_TYPE_D
||
886 dest
.type
== BRW_REGISTER_TYPE_UD
||
887 (dest
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 8));
888 if (devinfo
->gen
== 6) {
889 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
890 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
892 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
893 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 4);
894 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
896 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
897 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
898 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
899 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
900 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
901 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
902 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
903 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
905 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
906 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
907 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
908 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
909 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
910 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
911 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
912 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
914 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
915 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
916 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
917 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
918 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
919 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
920 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
921 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
923 if (devinfo
->gen
>= 7) {
924 /* Set both the source and destination types based on dest.type,
925 * ignoring the source register types. The MAD and LRP emitters ensure
926 * that all four types are float. The BFE and BFI2 emitters, however,
927 * may send us mixed D and UD types and want us to ignore that and use
928 * the destination type.
930 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
931 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
933 /* From the Bspec, 3D Media GPGPU, Instruction fields, srcType:
935 * "Three source instructions can use operands with mixed-mode
936 * precision. When SrcType field is set to :f or :hf it defines
937 * precision for source 0 only, and fields Src1Type and Src2Type
938 * define precision for other source operands:
940 * 0b = :f. Single precision Float (32-bit).
941 * 1b = :hf. Half precision Float (16-bit)."
943 if (src1
.type
== BRW_REGISTER_TYPE_HF
)
944 brw_inst_set_3src_a16_src1_type(devinfo
, inst
, 1);
946 if (src2
.type
== BRW_REGISTER_TYPE_HF
)
947 brw_inst_set_3src_a16_src2_type(devinfo
, inst
, 1);
955 /***********************************************************************
956 * Convenience routines.
959 brw_inst *brw_##OP(struct brw_codegen *p, \
960 struct brw_reg dest, \
961 struct brw_reg src0) \
963 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
967 brw_inst *brw_##OP(struct brw_codegen *p, \
968 struct brw_reg dest, \
969 struct brw_reg src0, \
970 struct brw_reg src1) \
972 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
976 brw_inst *brw_##OP(struct brw_codegen *p, \
977 struct brw_reg dest, \
978 struct brw_reg src0, \
979 struct brw_reg src1, \
980 struct brw_reg src2) \
982 if (p->current->access_mode == BRW_ALIGN_16) { \
983 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
984 src0.swizzle = BRW_SWIZZLE_XXXX; \
985 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
986 src1.swizzle = BRW_SWIZZLE_XXXX; \
987 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
988 src2.swizzle = BRW_SWIZZLE_XXXX; \
990 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
994 brw_inst *brw_##OP(struct brw_codegen *p, \
995 struct brw_reg dest, \
996 struct brw_reg src0, \
997 struct brw_reg src1, \
998 struct brw_reg src2) \
1000 assert(dest.type == BRW_REGISTER_TYPE_F || \
1001 dest.type == BRW_REGISTER_TYPE_DF); \
1002 if (dest.type == BRW_REGISTER_TYPE_F) { \
1003 assert(src0.type == BRW_REGISTER_TYPE_F); \
1004 assert(src1.type == BRW_REGISTER_TYPE_F); \
1005 assert(src2.type == BRW_REGISTER_TYPE_F); \
1006 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
1007 assert(src0.type == BRW_REGISTER_TYPE_DF); \
1008 assert(src1.type == BRW_REGISTER_TYPE_DF); \
1009 assert(src2.type == BRW_REGISTER_TYPE_DF); \
1012 if (p->current->access_mode == BRW_ALIGN_16) { \
1013 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
1014 src0.swizzle = BRW_SWIZZLE_XXXX; \
1015 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
1016 src1.swizzle = BRW_SWIZZLE_XXXX; \
1017 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
1018 src2.swizzle = BRW_SWIZZLE_XXXX; \
1020 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1060 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
1062 const struct gen_device_info
*devinfo
= p
->devinfo
;
1064 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
1065 * To avoid the problems that causes, we use an <X,2,0> source region to
1066 * read each element twice.
1068 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1069 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
1070 dest
.type
== BRW_REGISTER_TYPE_DF
&&
1071 (src0
.type
== BRW_REGISTER_TYPE_F
||
1072 src0
.type
== BRW_REGISTER_TYPE_D
||
1073 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
1074 !has_scalar_region(src0
)) {
1075 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
1076 src0
.vstride
= src0
.hstride
;
1077 src0
.width
= BRW_WIDTH_2
;
1078 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1081 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1085 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1086 struct brw_reg src0
, struct brw_reg src1
)
1089 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1090 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1091 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1092 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1093 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1096 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1097 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1098 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1099 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1100 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1103 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1107 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1108 struct brw_reg src0
, struct brw_reg src1
)
1110 assert(dest
.type
== src0
.type
);
1111 assert(src0
.type
== src1
.type
);
1112 switch (src0
.type
) {
1113 case BRW_REGISTER_TYPE_B
:
1114 case BRW_REGISTER_TYPE_UB
:
1115 case BRW_REGISTER_TYPE_W
:
1116 case BRW_REGISTER_TYPE_UW
:
1117 case BRW_REGISTER_TYPE_D
:
1118 case BRW_REGISTER_TYPE_UD
:
1121 unreachable("Bad type for brw_AVG");
1124 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1128 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1129 struct brw_reg src0
, struct brw_reg src1
)
1132 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1133 src0
.type
== BRW_REGISTER_TYPE_UD
||
1134 src1
.type
== BRW_REGISTER_TYPE_D
||
1135 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1136 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1139 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1140 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1141 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1142 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1143 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1146 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1147 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1148 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1149 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1150 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1153 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1154 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1155 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1156 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1158 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1162 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1163 struct brw_reg src0
, struct brw_reg src1
)
1165 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1166 src0
.width
= BRW_WIDTH_1
;
1167 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1168 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1172 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1173 struct brw_reg src0
, struct brw_reg src1
)
1175 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1176 src0
.width
= BRW_WIDTH_1
;
1177 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1178 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1179 src1
.width
= BRW_WIDTH_8
;
1180 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1181 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1185 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1187 const struct gen_device_info
*devinfo
= p
->devinfo
;
1188 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1189 /* The F32TO16 instruction doesn't support 32-bit destination types in
1190 * Align1 mode, and neither does the Gen8 implementation in terms of a
1191 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1192 * an undocumented feature.
1194 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1195 (!align16
|| devinfo
->gen
>= 8));
1199 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1201 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1202 dst
.type
== BRW_REGISTER_TYPE_W
||
1203 dst
.type
== BRW_REGISTER_TYPE_UW
||
1204 dst
.type
== BRW_REGISTER_TYPE_HF
);
1207 brw_push_insn_state(p
);
1209 if (needs_zero_fill
) {
1210 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1211 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1214 if (devinfo
->gen
>= 8) {
1215 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1217 assert(devinfo
->gen
== 7);
1218 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1221 if (needs_zero_fill
) {
1222 if (devinfo
->gen
< 12)
1223 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1224 brw_set_default_swsb(p
, tgl_swsb_null());
1225 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1226 if (devinfo
->gen
< 12)
1227 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1230 brw_pop_insn_state(p
);
1235 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1237 const struct gen_device_info
*devinfo
= p
->devinfo
;
1238 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1241 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1243 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1245 * Because this instruction does not have a 16-bit floating-point
1246 * type, the source data type must be Word (W). The destination type
1247 * must be F (Float).
1249 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1250 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1252 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1253 src
.type
== BRW_REGISTER_TYPE_UW
||
1254 src
.type
== BRW_REGISTER_TYPE_HF
);
1257 if (devinfo
->gen
>= 8) {
1258 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1260 assert(devinfo
->gen
== 7);
1261 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1266 void brw_NOP(struct brw_codegen
*p
)
1268 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1269 memset(insn
, 0, sizeof(*insn
));
1270 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1273 void brw_SYNC(struct brw_codegen
*p
, enum tgl_sync_function func
)
1275 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SYNC
);
1276 brw_inst_set_cond_modifier(p
->devinfo
, insn
, func
);
1279 /***********************************************************************
1280 * Comparisons, if/else/endif
1284 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1285 unsigned predicate_control
)
1287 const struct gen_device_info
*devinfo
= p
->devinfo
;
1288 struct brw_reg ip
= brw_ip_reg();
1289 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1291 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1292 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1293 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1294 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1300 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1302 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1304 p
->if_stack_depth
++;
1305 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1306 p
->if_stack_array_size
*= 2;
1307 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1308 p
->if_stack_array_size
);
1313 pop_if_stack(struct brw_codegen
*p
)
1315 p
->if_stack_depth
--;
1316 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1320 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1322 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1323 p
->loop_stack_array_size
*= 2;
1324 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1325 p
->loop_stack_array_size
);
1326 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1327 p
->loop_stack_array_size
);
1330 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1331 p
->loop_stack_depth
++;
1332 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1336 get_inner_do_insn(struct brw_codegen
*p
)
1338 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1341 /* EU takes the value from the flag register and pushes it onto some
1342 * sort of a stack (presumably merging with any flag value already on
1343 * the stack). Within an if block, the flags at the top of the stack
1344 * control execution on each channel of the unit, eg. on each of the
1345 * 16 pixel values in our wm programs.
1347 * When the matching 'else' instruction is reached (presumably by
1348 * countdown of the instruction count patched in by our ELSE/ENDIF
1349 * functions), the relevant flags are inverted.
1351 * When the matching 'endif' instruction is reached, the flags are
1352 * popped off. If the stack is now empty, normal execution resumes.
1355 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1357 const struct gen_device_info
*devinfo
= p
->devinfo
;
1360 insn
= next_insn(p
, BRW_OPCODE_IF
);
1362 /* Override the defaults for this instruction:
1364 if (devinfo
->gen
< 6) {
1365 brw_set_dest(p
, insn
, brw_ip_reg());
1366 brw_set_src0(p
, insn
, brw_ip_reg());
1367 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1368 } else if (devinfo
->gen
== 6) {
1369 brw_set_dest(p
, insn
, brw_imm_w(0));
1370 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1371 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1372 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1373 } else if (devinfo
->gen
== 7) {
1374 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1375 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1376 brw_set_src1(p
, insn
, brw_imm_w(0));
1377 brw_inst_set_jip(devinfo
, insn
, 0);
1378 brw_inst_set_uip(devinfo
, insn
, 0);
1380 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1381 if (devinfo
->gen
< 12)
1382 brw_set_src0(p
, insn
, brw_imm_d(0));
1383 brw_inst_set_jip(devinfo
, insn
, 0);
1384 brw_inst_set_uip(devinfo
, insn
, 0);
1387 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1388 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1389 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1390 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1391 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1392 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1394 push_if_stack(p
, insn
);
1395 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1399 /* This function is only used for gen6-style IF instructions with an
1400 * embedded comparison (conditional modifier). It is not used on gen7.
1403 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1404 struct brw_reg src0
, struct brw_reg src1
)
1406 const struct gen_device_info
*devinfo
= p
->devinfo
;
1409 insn
= next_insn(p
, BRW_OPCODE_IF
);
1411 brw_set_dest(p
, insn
, brw_imm_w(0));
1412 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1413 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1414 brw_set_src0(p
, insn
, src0
);
1415 brw_set_src1(p
, insn
, src1
);
1417 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1418 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1419 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1421 push_if_stack(p
, insn
);
1426 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1429 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1430 brw_inst
*if_inst
, brw_inst
*else_inst
)
1432 const struct gen_device_info
*devinfo
= p
->devinfo
;
1434 /* The next instruction (where the ENDIF would be, if it existed) */
1435 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1437 assert(p
->single_program_flow
);
1438 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1439 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1440 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1442 /* Convert IF to an ADD instruction that moves the instruction pointer
1443 * to the first instruction of the ELSE block. If there is no ELSE
1444 * block, point to where ENDIF would be. Reverse the predicate.
1446 * There's no need to execute an ENDIF since we don't need to do any
1447 * stack operations, and if we're currently executing, we just want to
1448 * continue normally.
1450 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1451 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1453 if (else_inst
!= NULL
) {
1454 /* Convert ELSE to an ADD instruction that points where the ENDIF
1457 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1459 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1460 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1462 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1467 * Patch IF and ELSE instructions with appropriate jump targets.
1470 patch_IF_ELSE(struct brw_codegen
*p
,
1471 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1473 const struct gen_device_info
*devinfo
= p
->devinfo
;
1475 /* We shouldn't be patching IF and ELSE instructions in single program flow
1476 * mode when gen < 6, because in single program flow mode on those
1477 * platforms, we convert flow control instructions to conditional ADDs that
1478 * operate on IP (see brw_ENDIF).
1480 * However, on Gen6, writing to IP doesn't work in single program flow mode
1481 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1482 * not be updated by non-flow control instructions."). And on later
1483 * platforms, there is no significant benefit to converting control flow
1484 * instructions to conditional ADDs. So we do patch IF and ELSE
1485 * instructions in single program flow mode on those platforms.
1487 if (devinfo
->gen
< 6)
1488 assert(!p
->single_program_flow
);
1490 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1491 assert(endif_inst
!= NULL
);
1492 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1494 unsigned br
= brw_jump_scale(devinfo
);
1496 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1497 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1499 if (else_inst
== NULL
) {
1500 /* Patch IF -> ENDIF */
1501 if (devinfo
->gen
< 6) {
1502 /* Turn it into an IFF, which means no mask stack operations for
1503 * all-false and jumping past the ENDIF.
1505 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1506 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1507 br
* (endif_inst
- if_inst
+ 1));
1508 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1509 } else if (devinfo
->gen
== 6) {
1510 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1511 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1513 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1514 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1517 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1519 /* Patch IF -> ELSE */
1520 if (devinfo
->gen
< 6) {
1521 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1522 br
* (else_inst
- if_inst
));
1523 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1524 } else if (devinfo
->gen
== 6) {
1525 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1526 br
* (else_inst
- if_inst
+ 1));
1529 /* Patch ELSE -> ENDIF */
1530 if (devinfo
->gen
< 6) {
1531 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1534 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1535 br
* (endif_inst
- else_inst
+ 1));
1536 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1537 } else if (devinfo
->gen
== 6) {
1538 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1539 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1540 br
* (endif_inst
- else_inst
));
1542 /* The IF instruction's JIP should point just past the ELSE */
1543 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1544 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1545 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1546 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1547 if (devinfo
->gen
>= 8) {
1548 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1549 * should point to ENDIF.
1551 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1558 brw_ELSE(struct brw_codegen
*p
)
1560 const struct gen_device_info
*devinfo
= p
->devinfo
;
1563 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1565 if (devinfo
->gen
< 6) {
1566 brw_set_dest(p
, insn
, brw_ip_reg());
1567 brw_set_src0(p
, insn
, brw_ip_reg());
1568 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1569 } else if (devinfo
->gen
== 6) {
1570 brw_set_dest(p
, insn
, brw_imm_w(0));
1571 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1572 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1573 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1574 } else if (devinfo
->gen
== 7) {
1575 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1576 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1577 brw_set_src1(p
, insn
, brw_imm_w(0));
1578 brw_inst_set_jip(devinfo
, insn
, 0);
1579 brw_inst_set_uip(devinfo
, insn
, 0);
1581 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1582 if (devinfo
->gen
< 12)
1583 brw_set_src0(p
, insn
, brw_imm_d(0));
1584 brw_inst_set_jip(devinfo
, insn
, 0);
1585 brw_inst_set_uip(devinfo
, insn
, 0);
1588 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1589 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1590 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1591 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1593 push_if_stack(p
, insn
);
1597 brw_ENDIF(struct brw_codegen
*p
)
1599 const struct gen_device_info
*devinfo
= p
->devinfo
;
1600 brw_inst
*insn
= NULL
;
1601 brw_inst
*else_inst
= NULL
;
1602 brw_inst
*if_inst
= NULL
;
1604 bool emit_endif
= true;
1606 /* In single program flow mode, we can express IF and ELSE instructions
1607 * equivalently as ADD instructions that operate on IP. On platforms prior
1608 * to Gen6, flow control instructions cause an implied thread switch, so
1609 * this is a significant savings.
1611 * However, on Gen6, writing to IP doesn't work in single program flow mode
1612 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1613 * not be updated by non-flow control instructions."). And on later
1614 * platforms, there is no significant benefit to converting control flow
1615 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1618 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1622 * A single next_insn() may change the base address of instruction store
1623 * memory(p->store), so call it first before referencing the instruction
1624 * store pointer from an index
1627 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1629 /* Pop the IF and (optional) ELSE instructions from the stack */
1630 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1631 tmp
= pop_if_stack(p
);
1632 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1634 tmp
= pop_if_stack(p
);
1639 /* ENDIF is useless; don't bother emitting it. */
1640 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1644 if (devinfo
->gen
< 6) {
1645 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1646 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1647 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1648 } else if (devinfo
->gen
== 6) {
1649 brw_set_dest(p
, insn
, brw_imm_w(0));
1650 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1651 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1652 } else if (devinfo
->gen
== 7) {
1653 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1654 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1655 brw_set_src1(p
, insn
, brw_imm_w(0));
1657 brw_set_src0(p
, insn
, brw_imm_d(0));
1660 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1661 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1662 if (devinfo
->gen
< 6)
1663 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1665 /* Also pop item off the stack in the endif instruction: */
1666 if (devinfo
->gen
< 6) {
1667 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1668 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1669 } else if (devinfo
->gen
== 6) {
1670 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1672 brw_inst_set_jip(devinfo
, insn
, 2);
1674 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1678 brw_BREAK(struct brw_codegen
*p
)
1680 const struct gen_device_info
*devinfo
= p
->devinfo
;
1683 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1684 if (devinfo
->gen
>= 8) {
1685 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1686 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1687 } else if (devinfo
->gen
>= 6) {
1688 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1689 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1690 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1692 brw_set_dest(p
, insn
, brw_ip_reg());
1693 brw_set_src0(p
, insn
, brw_ip_reg());
1694 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1695 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1696 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1698 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1699 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1705 brw_CONT(struct brw_codegen
*p
)
1707 const struct gen_device_info
*devinfo
= p
->devinfo
;
1710 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1711 brw_set_dest(p
, insn
, brw_ip_reg());
1712 if (devinfo
->gen
>= 8) {
1713 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1715 brw_set_src0(p
, insn
, brw_ip_reg());
1716 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1719 if (devinfo
->gen
< 6) {
1720 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1721 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1723 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1724 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1729 brw_HALT(struct brw_codegen
*p
)
1731 const struct gen_device_info
*devinfo
= p
->devinfo
;
1734 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1735 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1736 if (devinfo
->gen
< 6) {
1737 /* From the Gen4 PRM:
1739 * "IP register must be put (for example, by the assembler) at <dst>
1740 * and <src0> locations.
1742 brw_set_dest(p
, insn
, brw_ip_reg());
1743 brw_set_src0(p
, insn
, brw_ip_reg());
1744 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* exitcode updated later. */
1745 } else if (devinfo
->gen
< 8) {
1746 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1747 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1748 } else if (devinfo
->gen
< 12) {
1749 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1752 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1753 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1759 * The DO/WHILE is just an unterminated loop -- break or continue are
1760 * used for control within the loop. We have a few ways they can be
1763 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1764 * jip and no DO instruction.
1766 * For non-uniform control flow pre-gen6, there's a DO instruction to
1767 * push the mask, and a WHILE to jump back, and BREAK to get out and
1770 * For gen6, there's no more mask stack, so no need for DO. WHILE
1771 * just points back to the first instruction of the loop.
1774 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1776 const struct gen_device_info
*devinfo
= p
->devinfo
;
1778 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1779 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1780 return &p
->store
[p
->nr_insn
];
1782 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1784 push_loop_stack(p
, insn
);
1786 /* Override the defaults for this instruction:
1788 brw_set_dest(p
, insn
, brw_null_reg());
1789 brw_set_src0(p
, insn
, brw_null_reg());
1790 brw_set_src1(p
, insn
, brw_null_reg());
1792 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1793 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1794 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1801 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1804 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1805 * nesting, since it can always just point to the end of the block/current loop.
1808 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1810 const struct gen_device_info
*devinfo
= p
->devinfo
;
1811 brw_inst
*do_inst
= get_inner_do_insn(p
);
1813 unsigned br
= brw_jump_scale(devinfo
);
1815 assert(devinfo
->gen
< 6);
1817 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1818 /* If the jump count is != 0, that means that this instruction has already
1819 * been patched because it's part of a loop inside of the one we're
1822 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1823 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1824 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1825 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1826 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1827 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1833 brw_WHILE(struct brw_codegen
*p
)
1835 const struct gen_device_info
*devinfo
= p
->devinfo
;
1836 brw_inst
*insn
, *do_insn
;
1837 unsigned br
= brw_jump_scale(devinfo
);
1839 if (devinfo
->gen
>= 6) {
1840 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1841 do_insn
= get_inner_do_insn(p
);
1843 if (devinfo
->gen
>= 8) {
1844 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1845 if (devinfo
->gen
< 12)
1846 brw_set_src0(p
, insn
, brw_imm_d(0));
1847 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1848 } else if (devinfo
->gen
== 7) {
1849 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1850 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1851 brw_set_src1(p
, insn
, brw_imm_w(0));
1852 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1854 brw_set_dest(p
, insn
, brw_imm_w(0));
1855 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1856 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1857 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1860 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1863 if (p
->single_program_flow
) {
1864 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1865 do_insn
= get_inner_do_insn(p
);
1867 brw_set_dest(p
, insn
, brw_ip_reg());
1868 brw_set_src0(p
, insn
, brw_ip_reg());
1869 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1870 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1872 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1873 do_insn
= get_inner_do_insn(p
);
1875 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1877 brw_set_dest(p
, insn
, brw_ip_reg());
1878 brw_set_src0(p
, insn
, brw_ip_reg());
1879 brw_set_src1(p
, insn
, brw_imm_d(0));
1881 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1882 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1883 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1885 brw_patch_break_cont(p
, insn
);
1888 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1890 p
->loop_stack_depth
--;
1897 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1899 const struct gen_device_info
*devinfo
= p
->devinfo
;
1900 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1903 if (devinfo
->gen
>= 5)
1906 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1907 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1909 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1910 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1913 /* To integrate with the above, it makes sense that the comparison
1914 * instruction should populate the flag register. It might be simpler
1915 * just to use the flag reg for most WM tasks?
1917 void brw_CMP(struct brw_codegen
*p
,
1918 struct brw_reg dest
,
1919 unsigned conditional
,
1920 struct brw_reg src0
,
1921 struct brw_reg src1
)
1923 const struct gen_device_info
*devinfo
= p
->devinfo
;
1924 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1926 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1927 brw_set_dest(p
, insn
, dest
);
1928 brw_set_src0(p
, insn
, src0
);
1929 brw_set_src1(p
, insn
, src1
);
1931 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1933 * "Any CMP instruction with a null destination must use a {switch}."
1935 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1936 * mentioned on their work-arounds pages.
1938 if (devinfo
->gen
== 7) {
1939 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1940 dest
.nr
== BRW_ARF_NULL
) {
1941 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1946 /***********************************************************************
1947 * Helpers for the various SEND message types:
1950 /** Extended math function, float[8].
1952 void gen4_math(struct brw_codegen
*p
,
1953 struct brw_reg dest
,
1955 unsigned msg_reg_nr
,
1957 unsigned precision
)
1959 const struct gen_device_info
*devinfo
= p
->devinfo
;
1960 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1962 if (has_scalar_region(src
)) {
1963 data_type
= BRW_MATH_DATA_SCALAR
;
1965 data_type
= BRW_MATH_DATA_VECTOR
;
1968 assert(devinfo
->gen
< 6);
1970 /* Example code doesn't set predicate_control for send
1973 brw_inst_set_pred_control(devinfo
, insn
, 0);
1974 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1976 brw_set_dest(p
, insn
, dest
);
1977 brw_set_src0(p
, insn
, src
);
1978 brw_set_math_message(p
,
1981 src
.type
== BRW_REGISTER_TYPE_D
,
1986 void gen6_math(struct brw_codegen
*p
,
1987 struct brw_reg dest
,
1989 struct brw_reg src0
,
1990 struct brw_reg src1
)
1992 const struct gen_device_info
*devinfo
= p
->devinfo
;
1993 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1995 assert(devinfo
->gen
>= 6);
1997 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1998 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
2000 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2001 if (devinfo
->gen
== 6) {
2002 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2003 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
2006 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
2007 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
2008 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
2009 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
2010 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
2011 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
2012 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
2014 assert(src0
.type
== BRW_REGISTER_TYPE_F
||
2015 (src0
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2016 assert(src1
.type
== BRW_REGISTER_TYPE_F
||
2017 (src1
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
2020 /* Source modifiers are ignored for extended math instructions on Gen6. */
2021 if (devinfo
->gen
== 6) {
2022 assert(!src0
.negate
);
2024 assert(!src1
.negate
);
2028 brw_inst_set_math_function(devinfo
, insn
, function
);
2030 brw_set_dest(p
, insn
, dest
);
2031 brw_set_src0(p
, insn
, src0
);
2032 brw_set_src1(p
, insn
, src1
);
2036 * Return the right surface index to access the thread scratch space using
2037 * stateless dataport messages.
2040 brw_scratch_surface_idx(const struct brw_codegen
*p
)
2042 /* The scratch space is thread-local so IA coherency is unnecessary. */
2043 if (p
->devinfo
->gen
>= 8)
2044 return GEN8_BTI_STATELESS_NON_COHERENT
;
2046 return BRW_BTI_STATELESS
;
2050 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2051 * using a constant offset per channel.
2053 * The offset must be aligned to oword size (16 bytes). Used for
2054 * register spilling.
2056 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
2061 const struct gen_device_info
*devinfo
= p
->devinfo
;
2062 const unsigned target_cache
=
2063 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2064 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2065 BRW_SFID_DATAPORT_WRITE
);
2066 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2069 if (devinfo
->gen
>= 6)
2072 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2074 const unsigned mlen
= 1 + num_regs
;
2076 /* Set up the message header. This is g0, with g0.2 filled with
2077 * the offset. We don't want to leave our offset around in g0 or
2078 * it'll screw up texture samples, so set it up inside the message
2082 brw_push_insn_state(p
);
2083 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2084 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2085 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2086 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2088 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2090 /* set message header global offset field (reg 0, element 2) */
2091 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2092 brw_set_default_swsb(p
, tgl_swsb_null());
2094 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2096 2), BRW_REGISTER_TYPE_UD
),
2097 brw_imm_ud(offset
));
2099 brw_pop_insn_state(p
);
2100 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2104 struct brw_reg dest
;
2105 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2106 int send_commit_msg
;
2107 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2108 BRW_REGISTER_TYPE_UW
);
2110 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2111 brw_inst_set_compression(devinfo
, insn
, false);
2113 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2114 src_header
= vec16(src_header
);
2116 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2117 if (devinfo
->gen
< 6)
2118 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2120 /* Until gen6, writes followed by reads from the same location
2121 * are not guaranteed to be ordered unless write_commit is set.
2122 * If set, then a no-op write is issued to the destination
2123 * register to set a dependency, and a read from the destination
2124 * can be used to ensure the ordering.
2126 * For gen6, only writes between different threads need ordering
2127 * protection. Our use of DP writes is all about register
2128 * spilling within a thread.
2130 if (devinfo
->gen
>= 6) {
2131 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2132 send_commit_msg
= 0;
2135 send_commit_msg
= 1;
2138 brw_set_dest(p
, insn
, dest
);
2139 if (devinfo
->gen
>= 6) {
2140 brw_set_src0(p
, insn
, mrf
);
2142 brw_set_src0(p
, insn
, brw_null_reg());
2145 if (devinfo
->gen
>= 6)
2146 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2148 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2150 brw_set_desc(p
, insn
,
2151 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2152 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2153 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2154 msg_type
, 0, /* not a render target */
2161 * Read a block of owords (half a GRF each) from the scratch buffer
2162 * using a constant index per channel.
2164 * Offset must be aligned to oword size (16 bytes). Used for register
2168 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2169 struct brw_reg dest
,
2174 const struct gen_device_info
*devinfo
= p
->devinfo
;
2175 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2177 if (devinfo
->gen
>= 6)
2180 if (p
->devinfo
->gen
>= 7) {
2181 /* On gen 7 and above, we no longer have message registers and we can
2182 * send from any register we want. By using the destination register
2183 * for the message, we guarantee that the implied message write won't
2184 * accidentally overwrite anything. This has been a problem because
2185 * the MRF registers and source for the final FB write are both fixed
2188 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2190 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2192 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2194 const unsigned rlen
= num_regs
;
2195 const unsigned target_cache
=
2196 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2197 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2198 BRW_SFID_DATAPORT_READ
);
2201 brw_push_insn_state(p
);
2202 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2203 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2204 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2205 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2207 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2209 /* set message header global offset field (reg 0, element 2) */
2210 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2211 brw_set_default_swsb(p
, tgl_swsb_null());
2212 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2214 brw_pop_insn_state(p
);
2215 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2219 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2221 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2222 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2223 brw_inst_set_compression(devinfo
, insn
, false);
2225 brw_set_dest(p
, insn
, dest
); /* UW? */
2226 if (devinfo
->gen
>= 6) {
2227 brw_set_src0(p
, insn
, mrf
);
2229 brw_set_src0(p
, insn
, brw_null_reg());
2230 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2233 brw_set_desc(p
, insn
,
2234 brw_message_desc(devinfo
, 1, rlen
, true) |
2235 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2236 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2237 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2238 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2243 gen7_block_read_scratch(struct brw_codegen
*p
,
2244 struct brw_reg dest
,
2248 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2249 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2251 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2253 /* The HW requires that the header is present; this is to get the g0.5
2256 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2258 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2259 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2260 * is 32 bytes, which happens to be the size of a register.
2263 assert(offset
< (1 << 12));
2265 gen7_set_dp_scratch_message(p
, insn
,
2266 false, /* scratch read */
2268 false, /* invalidate after read */
2271 1, /* mlen: just g0 */
2272 num_regs
, /* rlen */
2273 true); /* header present */
2277 * Read float[4] vectors from the data port constant cache.
2278 * Location (in buffer) should be a multiple of 16.
2279 * Used for fetching shader constants.
2281 void brw_oword_block_read(struct brw_codegen
*p
,
2282 struct brw_reg dest
,
2285 uint32_t bind_table_index
)
2287 const struct gen_device_info
*devinfo
= p
->devinfo
;
2288 const unsigned target_cache
=
2289 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2290 BRW_SFID_DATAPORT_READ
);
2291 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2292 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2294 /* On newer hardware, offset is in units of owords. */
2295 if (devinfo
->gen
>= 6)
2298 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2300 brw_push_insn_state(p
);
2301 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2302 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2303 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2305 brw_push_insn_state(p
);
2306 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2307 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2308 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2310 /* set message header global offset field (reg 0, element 2) */
2311 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2312 brw_set_default_swsb(p
, tgl_swsb_null());
2314 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2316 2), BRW_REGISTER_TYPE_UD
),
2317 brw_imm_ud(offset
));
2318 brw_pop_insn_state(p
);
2320 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2322 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2324 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2326 /* cast dest to a uword[8] vector */
2327 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2329 brw_set_dest(p
, insn
, dest
);
2330 if (devinfo
->gen
>= 6) {
2331 brw_set_src0(p
, insn
, mrf
);
2333 brw_set_src0(p
, insn
, brw_null_reg());
2334 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2337 brw_set_desc(p
, insn
,
2338 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2339 brw_dp_read_desc(devinfo
, bind_table_index
,
2340 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2341 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2342 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2344 brw_pop_insn_state(p
);
2348 brw_fb_WRITE(struct brw_codegen
*p
,
2349 struct brw_reg payload
,
2350 struct brw_reg implied_header
,
2351 unsigned msg_control
,
2352 unsigned binding_table_index
,
2353 unsigned msg_length
,
2354 unsigned response_length
,
2356 bool last_render_target
,
2357 bool header_present
)
2359 const struct gen_device_info
*devinfo
= p
->devinfo
;
2360 const unsigned target_cache
=
2361 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2362 BRW_SFID_DATAPORT_WRITE
);
2365 struct brw_reg dest
, src0
;
2367 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2368 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2370 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2372 if (devinfo
->gen
>= 6) {
2373 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2375 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2377 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2378 brw_inst_set_compression(devinfo
, insn
, false);
2380 if (devinfo
->gen
>= 6) {
2381 /* headerless version, just submit color payload */
2384 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2386 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2387 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2388 src0
= implied_header
;
2390 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2393 brw_set_dest(p
, insn
, dest
);
2394 brw_set_src0(p
, insn
, src0
);
2395 brw_set_desc(p
, insn
,
2396 brw_message_desc(devinfo
, msg_length
, response_length
,
2398 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2399 msg_type
, last_render_target
,
2400 0 /* send_commit_msg */));
2401 brw_inst_set_eot(devinfo
, insn
, eot
);
2407 gen9_fb_READ(struct brw_codegen
*p
,
2409 struct brw_reg payload
,
2410 unsigned binding_table_index
,
2411 unsigned msg_length
,
2412 unsigned response_length
,
2415 const struct gen_device_info
*devinfo
= p
->devinfo
;
2416 assert(devinfo
->gen
>= 9);
2417 const unsigned msg_subtype
=
2418 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2419 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2421 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2422 brw_set_dest(p
, insn
, dst
);
2423 brw_set_src0(p
, insn
, payload
);
2426 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2427 brw_dp_read_desc(devinfo
, binding_table_index
,
2428 per_sample
<< 5 | msg_subtype
,
2429 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2430 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2431 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2437 * Texture sample instruction.
2438 * Note: the msg_type plus msg_length values determine exactly what kind
2439 * of sampling operation is performed. See volume 4, page 161 of docs.
2441 void brw_SAMPLE(struct brw_codegen
*p
,
2442 struct brw_reg dest
,
2443 unsigned msg_reg_nr
,
2444 struct brw_reg src0
,
2445 unsigned binding_table_index
,
2448 unsigned response_length
,
2449 unsigned msg_length
,
2450 unsigned header_present
,
2452 unsigned return_format
)
2454 const struct gen_device_info
*devinfo
= p
->devinfo
;
2457 if (msg_reg_nr
!= -1)
2458 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2460 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2461 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2462 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2464 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2466 * "Instruction compression is not allowed for this instruction (that
2467 * is, send). The hardware behavior is undefined if this instruction is
2468 * set as compressed. However, compress control can be set to "SecHalf"
2469 * to affect the EMask generation."
2471 * No similar wording is found in later PRMs, but there are examples
2472 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2473 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2474 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2476 brw_inst_set_compression(devinfo
, insn
, false);
2478 if (devinfo
->gen
< 6)
2479 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2481 brw_set_dest(p
, insn
, dest
);
2482 brw_set_src0(p
, insn
, src0
);
2483 brw_set_desc(p
, insn
,
2484 brw_message_desc(devinfo
, msg_length
, response_length
,
2486 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2487 msg_type
, simd_mode
, return_format
));
2490 /* Adjust the message header's sampler state pointer to
2491 * select the correct group of 16 samplers.
2493 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2494 struct brw_reg header
,
2495 struct brw_reg sampler_index
)
2497 /* The "Sampler Index" field can only store values between 0 and 15.
2498 * However, we can add an offset to the "Sampler State Pointer"
2499 * field, effectively selecting a different set of 16 samplers.
2501 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2502 * offset, and each sampler state is only 16-bytes, so we can't
2503 * exclusively use the offset - we have to use both.
2506 const struct gen_device_info
*devinfo
= p
->devinfo
;
2508 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2509 const int sampler_state_size
= 16; /* 16 bytes */
2510 uint32_t sampler
= sampler_index
.ud
;
2512 if (sampler
>= 16) {
2513 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2515 get_element_ud(header
, 3),
2516 get_element_ud(brw_vec8_grf(0, 0), 3),
2517 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2520 /* Non-const sampler array indexing case */
2521 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2525 struct brw_reg temp
= get_element_ud(header
, 3);
2527 brw_push_insn_state(p
);
2528 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2529 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
2530 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2532 get_element_ud(header
, 3),
2533 get_element_ud(brw_vec8_grf(0, 0), 3),
2535 brw_pop_insn_state(p
);
2539 /* All these variables are pretty confusing - we might be better off
2540 * using bitmasks and macros for this, in the old style. Or perhaps
2541 * just having the caller instantiate the fields in dword3 itself.
2543 void brw_urb_WRITE(struct brw_codegen
*p
,
2544 struct brw_reg dest
,
2545 unsigned msg_reg_nr
,
2546 struct brw_reg src0
,
2547 enum brw_urb_write_flags flags
,
2548 unsigned msg_length
,
2549 unsigned response_length
,
2553 const struct gen_device_info
*devinfo
= p
->devinfo
;
2556 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2558 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2559 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2560 brw_push_insn_state(p
);
2561 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2562 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2563 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2564 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2565 BRW_REGISTER_TYPE_UD
),
2566 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2567 brw_imm_ud(0xff00));
2568 brw_pop_insn_state(p
);
2571 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2573 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2575 brw_set_dest(p
, insn
, dest
);
2576 brw_set_src0(p
, insn
, src0
);
2577 brw_set_src1(p
, insn
, brw_imm_d(0));
2579 if (devinfo
->gen
< 6)
2580 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2582 brw_set_urb_message(p
,
2592 brw_send_indirect_message(struct brw_codegen
*p
,
2595 struct brw_reg payload
,
2596 struct brw_reg desc
,
2600 const struct gen_device_info
*devinfo
= p
->devinfo
;
2601 struct brw_inst
*send
;
2603 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2605 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2607 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2608 send
= next_insn(p
, BRW_OPCODE_SEND
);
2609 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2610 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2612 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2613 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2615 brw_push_insn_state(p
);
2616 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2617 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2618 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2619 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2620 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2622 /* Load the indirect descriptor to an address register using OR so the
2623 * caller can specify additional descriptor bits with the desc_imm
2626 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2628 brw_pop_insn_state(p
);
2630 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2631 send
= next_insn(p
, BRW_OPCODE_SEND
);
2632 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2634 if (devinfo
->gen
>= 12)
2635 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, true);
2637 brw_set_src1(p
, send
, addr
);
2640 brw_set_dest(p
, send
, dst
);
2641 brw_inst_set_sfid(devinfo
, send
, sfid
);
2642 brw_inst_set_eot(devinfo
, send
, eot
);
2646 brw_send_indirect_split_message(struct brw_codegen
*p
,
2649 struct brw_reg payload0
,
2650 struct brw_reg payload1
,
2651 struct brw_reg desc
,
2653 struct brw_reg ex_desc
,
2654 unsigned ex_desc_imm
,
2657 const struct gen_device_info
*devinfo
= p
->devinfo
;
2658 struct brw_inst
*send
;
2660 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2662 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2664 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2665 desc
.ud
|= desc_imm
;
2667 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2668 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2670 brw_push_insn_state(p
);
2671 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2672 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2673 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2674 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2675 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2677 /* Load the indirect descriptor to an address register using OR so the
2678 * caller can specify additional descriptor bits with the desc_imm
2681 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2683 brw_pop_insn_state(p
);
2686 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2689 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
&&
2690 (devinfo
->gen
>= 12 || (ex_desc
.ud
& INTEL_MASK(15, 12)) == 0)) {
2691 ex_desc
.ud
|= ex_desc_imm
;
2693 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2694 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2696 brw_push_insn_state(p
);
2697 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2698 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2699 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2700 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2701 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2703 /* Load the indirect extended descriptor to an address register using OR
2704 * so the caller can specify additional descriptor bits with the
2705 * desc_imm immediate.
2707 * Even though the instruction dispatcher always pulls the SFID and EOT
2708 * fields from the instruction itself, actual external unit which
2709 * processes the message gets the SFID and EOT from the extended
2710 * descriptor which comes from the address register. If we don't OR
2711 * those two bits in, the external unit may get confused and hang.
2713 unsigned imm_part
= ex_desc_imm
| sfid
| eot
<< 5;
2715 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2716 /* ex_desc bits 15:12 don't exist in the instruction encoding prior
2717 * to Gen12, so we may have fallen back to an indirect extended
2720 brw_MOV(p
, addr
, brw_imm_ud(ex_desc
.ud
| imm_part
));
2722 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(imm_part
));
2725 brw_pop_insn_state(p
);
2728 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2731 send
= next_insn(p
, devinfo
->gen
>= 12 ? BRW_OPCODE_SEND
: BRW_OPCODE_SENDS
);
2732 brw_set_dest(p
, send
, dst
);
2733 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2734 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2736 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2737 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2738 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2740 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2741 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2742 assert(desc
.subnr
== 0);
2743 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2746 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2747 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2748 brw_inst_set_sends_ex_desc(devinfo
, send
, ex_desc
.ud
);
2750 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2751 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2752 assert((ex_desc
.subnr
& 0x3) == 0);
2753 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2754 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2757 brw_inst_set_sfid(devinfo
, send
, sfid
);
2758 brw_inst_set_eot(devinfo
, send
, eot
);
2762 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2765 struct brw_reg payload
,
2766 struct brw_reg surface
,
2769 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2770 const struct tgl_swsb swsb
= brw_get_default_swsb(p
);
2771 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2773 brw_push_insn_state(p
);
2774 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2775 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2776 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2777 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2778 brw_set_default_swsb(p
, tgl_swsb_src_dep(swsb
));
2780 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2781 * some surface array is accessed out of bounds.
2784 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2785 BRW_GET_SWZ(surface
.swizzle
, 0)),
2788 brw_pop_insn_state(p
);
2791 brw_set_default_swsb(p
, tgl_swsb_dst_dep(swsb
, 1));
2794 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2798 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2799 brw_inst
*insn
, int while_offset
, int start_offset
)
2801 int scale
= 16 / brw_jump_scale(devinfo
);
2802 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2803 : brw_inst_jip(devinfo
, insn
);
2805 return while_offset
+ jip
* scale
<= start_offset
;
2810 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2813 void *store
= p
->store
;
2814 const struct gen_device_info
*devinfo
= p
->devinfo
;
2818 for (offset
= next_offset(devinfo
, store
, start_offset
);
2819 offset
< p
->next_insn_offset
;
2820 offset
= next_offset(devinfo
, store
, offset
)) {
2821 brw_inst
*insn
= store
+ offset
;
2823 switch (brw_inst_opcode(devinfo
, insn
)) {
2827 case BRW_OPCODE_ENDIF
:
2832 case BRW_OPCODE_WHILE
:
2833 /* If the while doesn't jump before our instruction, it's the end
2834 * of a sibling do...while loop. Ignore it.
2836 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2839 case BRW_OPCODE_ELSE
:
2840 case BRW_OPCODE_HALT
:
2851 /* There is no DO instruction on gen6, so to find the end of the loop
2852 * we have to see if the loop is jumping back before our start
2856 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2858 const struct gen_device_info
*devinfo
= p
->devinfo
;
2860 void *store
= p
->store
;
2862 assert(devinfo
->gen
>= 6);
2864 /* Always start after the instruction (such as a WHILE) we're trying to fix
2867 for (offset
= next_offset(devinfo
, store
, start_offset
);
2868 offset
< p
->next_insn_offset
;
2869 offset
= next_offset(devinfo
, store
, offset
)) {
2870 brw_inst
*insn
= store
+ offset
;
2872 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2873 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2877 assert(!"not reached");
2878 return start_offset
;
2881 /* After program generation, go back and update the UIP and JIP of
2882 * BREAK, CONT, and HALT instructions to their correct locations.
2885 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2887 const struct gen_device_info
*devinfo
= p
->devinfo
;
2889 int br
= brw_jump_scale(devinfo
);
2890 int scale
= 16 / br
;
2891 void *store
= p
->store
;
2893 if (devinfo
->gen
< 6)
2896 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2897 brw_inst
*insn
= store
+ offset
;
2898 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2900 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2901 switch (brw_inst_opcode(devinfo
, insn
)) {
2902 case BRW_OPCODE_BREAK
:
2903 assert(block_end_offset
!= 0);
2904 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2905 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2906 brw_inst_set_uip(devinfo
, insn
,
2907 (brw_find_loop_end(p
, offset
) - offset
+
2908 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2910 case BRW_OPCODE_CONTINUE
:
2911 assert(block_end_offset
!= 0);
2912 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2913 brw_inst_set_uip(devinfo
, insn
,
2914 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2916 assert(brw_inst_uip(devinfo
, insn
) != 0);
2917 assert(brw_inst_jip(devinfo
, insn
) != 0);
2920 case BRW_OPCODE_ENDIF
: {
2921 int32_t jump
= (block_end_offset
== 0) ?
2922 1 * br
: (block_end_offset
- offset
) / scale
;
2923 if (devinfo
->gen
>= 7)
2924 brw_inst_set_jip(devinfo
, insn
, jump
);
2926 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2930 case BRW_OPCODE_HALT
:
2931 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2933 * "In case of the halt instruction not inside any conditional
2934 * code block, the value of <JIP> and <UIP> should be the
2935 * same. In case of the halt instruction inside conditional code
2936 * block, the <UIP> should be the end of the program, and the
2937 * <JIP> should be end of the most inner conditional code block."
2939 * The uip will have already been set by whoever set up the
2942 if (block_end_offset
== 0) {
2943 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2945 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2947 assert(brw_inst_uip(devinfo
, insn
) != 0);
2948 assert(brw_inst_jip(devinfo
, insn
) != 0);
2957 void brw_ff_sync(struct brw_codegen
*p
,
2958 struct brw_reg dest
,
2959 unsigned msg_reg_nr
,
2960 struct brw_reg src0
,
2962 unsigned response_length
,
2965 const struct gen_device_info
*devinfo
= p
->devinfo
;
2968 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2970 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2971 brw_set_dest(p
, insn
, dest
);
2972 brw_set_src0(p
, insn
, src0
);
2973 brw_set_src1(p
, insn
, brw_imm_d(0));
2975 if (devinfo
->gen
< 6)
2976 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2978 brw_set_ff_sync_message(p
,
2986 * Emit the SEND instruction necessary to generate stream output data on Gen6
2987 * (for transform feedback).
2989 * If send_commit_msg is true, this is the last piece of stream output data
2990 * from this thread, so send the data as a committed write. According to the
2991 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2993 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2994 * writes are complete by sending the final write as a committed write."
2997 brw_svb_write(struct brw_codegen
*p
,
2998 struct brw_reg dest
,
2999 unsigned msg_reg_nr
,
3000 struct brw_reg src0
,
3001 unsigned binding_table_index
,
3002 bool send_commit_msg
)
3004 const struct gen_device_info
*devinfo
= p
->devinfo
;
3005 const unsigned target_cache
=
3006 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
3007 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
3008 BRW_SFID_DATAPORT_WRITE
);
3011 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
3013 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3014 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
3015 brw_set_dest(p
, insn
, dest
);
3016 brw_set_src0(p
, insn
, src0
);
3017 brw_set_desc(p
, insn
,
3018 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
3019 brw_dp_write_desc(devinfo
, binding_table_index
,
3020 0, /* msg_control: ignored */
3021 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
3022 0, /* last_render_target: ignored */
3023 send_commit_msg
)); /* send_commit_msg */
3027 brw_surface_payload_size(struct brw_codegen
*p
,
3028 unsigned num_channels
,
3029 unsigned exec_size
/**< 0 for SIMD4x2 */)
3032 return 1; /* SIMD4x2 */
3033 else if (exec_size
<= 8)
3034 return num_channels
;
3036 return 2 * num_channels
;
3040 brw_untyped_atomic(struct brw_codegen
*p
,
3042 struct brw_reg payload
,
3043 struct brw_reg surface
,
3045 unsigned msg_length
,
3046 bool response_expected
,
3047 bool header_present
)
3049 const struct gen_device_info
*devinfo
= p
->devinfo
;
3050 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3051 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3052 GEN7_SFID_DATAPORT_DATA_CACHE
);
3053 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3054 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
3055 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3056 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3057 has_simd4x2
? 0 : 8;
3058 const unsigned response_length
=
3059 brw_surface_payload_size(p
, response_expected
, exec_size
);
3060 const unsigned desc
=
3061 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3062 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
3064 /* Mask out unused components -- This is especially important in Align16
3065 * mode on generations that don't have native support for SIMD4x2 atomics,
3066 * because unused but enabled components will cause the dataport to perform
3067 * additional atomic operations on the addresses that happen to be in the
3068 * uninitialized Y, Z and W coordinates of the payload.
3070 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3072 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
3073 payload
, surface
, desc
);
3077 brw_untyped_surface_read(struct brw_codegen
*p
,
3079 struct brw_reg payload
,
3080 struct brw_reg surface
,
3081 unsigned msg_length
,
3082 unsigned num_channels
)
3084 const struct gen_device_info
*devinfo
= p
->devinfo
;
3085 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3086 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3087 GEN7_SFID_DATAPORT_DATA_CACHE
);
3088 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3089 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
3090 const unsigned response_length
=
3091 brw_surface_payload_size(p
, num_channels
, exec_size
);
3092 const unsigned desc
=
3093 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3094 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
3096 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
3100 brw_untyped_surface_write(struct brw_codegen
*p
,
3101 struct brw_reg payload
,
3102 struct brw_reg surface
,
3103 unsigned msg_length
,
3104 unsigned num_channels
,
3105 bool header_present
)
3107 const struct gen_device_info
*devinfo
= p
->devinfo
;
3108 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3109 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3110 GEN7_SFID_DATAPORT_DATA_CACHE
);
3111 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3112 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
3113 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3114 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3115 has_simd4x2
? 0 : 8;
3116 const unsigned desc
=
3117 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3118 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
3119 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3120 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
3122 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3123 payload
, surface
, desc
);
3127 brw_set_memory_fence_message(struct brw_codegen
*p
,
3128 struct brw_inst
*insn
,
3129 enum brw_message_target sfid
,
3133 const struct gen_device_info
*devinfo
= p
->devinfo
;
3135 brw_set_desc(p
, insn
, brw_message_desc(
3136 devinfo
, 1, (commit_enable
? 1 : 0), true));
3138 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3141 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3142 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3144 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3145 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3148 unreachable("Not reached");
3152 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3154 assert(devinfo
->gen
>= 11 || bti
== 0);
3155 brw_inst_set_binding_table_index(devinfo
, insn
, bti
);
3159 brw_memory_fence(struct brw_codegen
*p
,
3162 enum opcode send_op
,
3163 enum brw_message_target sfid
,
3167 const struct gen_device_info
*devinfo
= p
->devinfo
;
3169 dst
= retype(vec1(dst
), BRW_REGISTER_TYPE_UW
);
3170 src
= retype(vec1(src
), BRW_REGISTER_TYPE_UD
);
3172 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3173 * message doesn't write anything back.
3175 struct brw_inst
*insn
= next_insn(p
, send_op
);
3176 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3177 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3178 brw_set_dest(p
, insn
, dst
);
3179 brw_set_src0(p
, insn
, src
);
3180 brw_set_memory_fence_message(p
, insn
, sfid
, commit_enable
, bti
);
3184 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3185 struct brw_reg dest
,
3189 struct brw_reg data
,
3190 unsigned msg_length
,
3191 unsigned response_length
)
3193 const struct gen_device_info
*devinfo
= p
->devinfo
;
3194 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3195 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3196 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3197 const unsigned desc
=
3198 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3199 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3202 /* brw_send_indirect_message will automatically use a direct send message
3203 * if data is actually immediate.
3205 brw_send_indirect_message(p
,
3206 GEN7_SFID_PIXEL_INTERPOLATOR
,
3215 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3216 struct brw_reg mask
)
3218 const struct gen_device_info
*devinfo
= p
->devinfo
;
3219 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3220 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3223 assert(devinfo
->gen
>= 7);
3224 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3226 brw_push_insn_state(p
);
3228 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3229 * unnecessary bits in the instruction words, get the information we need
3230 * and reset the default flag register. This allows more instructions to be
3233 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3234 brw_set_default_flag_reg(p
, 0, 0);
3236 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3237 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3239 if (devinfo
->gen
>= 8) {
3240 /* Getting the first active channel index is easy on Gen8: Just find
3241 * the first bit set in the execution mask. The register exists on
3242 * HSW already but it reads back as all ones when the current
3243 * instruction has execution masking disabled, so it's kind of
3246 struct brw_reg exec_mask
=
3247 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3249 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3250 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3251 /* Unfortunately, ce0 does not take into account the thread
3252 * dispatch mask, which may be a problem in cases where it's not
3253 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3254 * some n). Combine ce0 with the given dispatch (or vector) mask
3255 * to mask off those channels which were never dispatched by the
3258 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3259 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3260 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3261 exec_mask
= vec1(dst
);
3264 /* Quarter control has the effect of magically shifting the value of
3265 * ce0 so you'll get the first active channel relative to the
3266 * specified quarter control as result.
3268 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3270 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3272 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3273 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3275 /* Run enough instructions returning zero with execution masking and
3276 * a conditional modifier enabled in order to get the full execution
3277 * mask in f1.0. We could use a single 32-wide move here if it
3278 * weren't because of the hardware bug that causes channel enables to
3279 * be applied incorrectly to the second half of 32-wide instructions
3282 const unsigned lower_size
= MIN2(16, exec_size
);
3283 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3284 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3286 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3287 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3288 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3289 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3290 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3291 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3294 /* Find the first bit set in the exec_size-wide portion of the flag
3295 * register that was updated by the last sequence of MOV
3298 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3299 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3300 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3303 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3305 if (devinfo
->gen
>= 8 &&
3306 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3307 /* In SIMD4x2 mode the first active channel index is just the
3308 * negation of the first bit of the mask register. Note that ce0
3309 * doesn't take into account the dispatch mask, so the Gen7 path
3310 * should be used instead unless you have the guarantee that the
3311 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3314 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3315 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3319 /* Overwrite the destination without and with execution masking to
3320 * find out which of the channels is active.
3322 brw_push_insn_state(p
);
3323 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3324 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3327 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3329 brw_pop_insn_state(p
);
3330 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3334 brw_pop_insn_state(p
);
3338 brw_broadcast(struct brw_codegen
*p
,
3343 const struct gen_device_info
*devinfo
= p
->devinfo
;
3344 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3347 brw_push_insn_state(p
);
3348 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3349 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3351 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3352 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3353 assert(!src
.abs
&& !src
.negate
);
3354 assert(src
.type
== dst
.type
);
3356 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3357 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3358 /* Trivial, the source is already uniform or the index is a constant.
3359 * We will typically not get here if the optimizer is doing its job, but
3360 * asserting would be mean.
3362 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3364 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3365 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3367 /* From the Haswell PRM section "Register Region Restrictions":
3369 * "The lower bits of the AddressImmediate must not overflow to
3370 * change the register address. The lower 5 bits of Address
3371 * Immediate when added to lower 5 bits of address register gives
3372 * the sub-register offset. The upper bits of Address Immediate
3373 * when added to upper bits of address register gives the register
3374 * address. Any overflow from sub-register offset is dropped."
3376 * Fortunately, for broadcast, we never have a sub-register offset so
3377 * this isn't an issue.
3379 assert(src
.subnr
== 0);
3382 const struct brw_reg addr
=
3383 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3384 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3385 /* Limit in bytes of the signed indirect addressing immediate. */
3386 const unsigned limit
= 512;
3388 brw_push_insn_state(p
);
3389 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3390 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3392 /* Take into account the component size and horizontal stride. */
3393 assert(src
.vstride
== src
.hstride
+ src
.width
);
3394 brw_SHL(p
, addr
, vec1(idx
),
3395 brw_imm_ud(util_logbase2(type_sz(src
.type
)) +
3398 /* We can only address up to limit bytes using the indirect
3399 * addressing immediate, account for the difference if the source
3400 * register is above this limit.
3402 if (offset
>= limit
) {
3403 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3404 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3405 offset
= offset
% limit
;
3408 brw_pop_insn_state(p
);
3410 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3412 /* Use indirect addressing to fetch the specified component. */
3413 if (type_sz(src
.type
) > 4 &&
3414 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3415 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3417 * "When source or destination datatype is 64b or operation is
3418 * integer DWord multiply, indirect addressing must not be
3421 * To work around both of this issue, we do two integer MOVs
3422 * insead of one 64-bit MOV. Because no double value should ever
3423 * cross a register boundary, it's safe to use the immediate
3424 * offset in the indirect here to handle adding 4 bytes to the
3425 * offset and avoid the extra ADD to the register file.
3427 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3428 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3429 BRW_REGISTER_TYPE_D
));
3430 brw_set_default_swsb(p
, tgl_swsb_null());
3431 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3432 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3433 BRW_REGISTER_TYPE_D
));
3436 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3439 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3440 * to all bits of a flag register,
3444 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3445 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3446 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3447 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3449 /* and use predicated SEL to pick the right channel. */
3450 inst
= brw_SEL(p
, dst
,
3451 stride(suboffset(src
, 4), 4, 4, 1),
3452 stride(src
, 4, 4, 1));
3453 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3454 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3458 brw_pop_insn_state(p
);
3462 * This instruction is generated as a single-channel align1 instruction by
3463 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3465 * We can't use the typed atomic op in the FS because that has the execution
3466 * mask ANDed with the pixel mask, but we just want to write the one dword for
3469 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3470 * one u32. So we use the same untyped atomic write message as the pixel
3473 * The untyped atomic operation requires a BUFFER surface type with RAW
3474 * format, and is only accessible through the legacy DATA_CACHE dataport
3477 void brw_shader_time_add(struct brw_codegen
*p
,
3478 struct brw_reg payload
,
3479 uint32_t surf_index
)
3481 const struct gen_device_info
*devinfo
= p
->devinfo
;
3482 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3483 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3484 GEN7_SFID_DATAPORT_DATA_CACHE
);
3485 assert(devinfo
->gen
>= 7);
3487 brw_push_insn_state(p
);
3488 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3489 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3490 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3491 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3493 /* We use brw_vec1_reg and unmasked because we want to increment the given
3496 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3498 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3500 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3501 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3504 brw_inst_set_sfid(devinfo
, send
, sfid
);
3505 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3507 brw_pop_insn_state(p
);
3512 * Emit the SEND message for a barrier
3515 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3517 const struct gen_device_info
*devinfo
= p
->devinfo
;
3518 struct brw_inst
*inst
;
3520 assert(devinfo
->gen
>= 7);
3522 brw_push_insn_state(p
);
3523 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3524 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3525 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3526 brw_set_src0(p
, inst
, src
);
3527 brw_set_src1(p
, inst
, brw_null_reg());
3528 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3530 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3531 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3532 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3534 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3535 brw_pop_insn_state(p
);
3540 * Emit the wait instruction for a barrier
3543 brw_WAIT(struct brw_codegen
*p
)
3545 const struct gen_device_info
*devinfo
= p
->devinfo
;
3546 struct brw_inst
*insn
;
3548 struct brw_reg src
= brw_notification_reg();
3550 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3551 brw_set_dest(p
, insn
, src
);
3552 brw_set_src0(p
, insn
, src
);
3553 brw_set_src1(p
, insn
, brw_null_reg());
3555 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3556 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3560 brw_float_controls_mode(struct brw_codegen
*p
,
3561 unsigned mode
, unsigned mask
)
3563 /* From the Skylake PRM, Volume 7, page 760:
3564 * "Implementation Restriction on Register Access: When the control
3565 * register is used as an explicit source and/or destination, hardware
3566 * does not ensure execution pipeline coherency. Software must set the
3567 * thread control field to ‘switch’ for an instruction that uses
3568 * control register as an explicit operand."
3570 * On Gen12+ this is implemented in terms of SWSB annotations instead.
3572 brw_set_default_swsb(p
, tgl_swsb_regdist(1));
3574 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3576 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3577 if (p
->devinfo
->gen
< 12)
3578 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3581 brw_inst
*inst_or
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3583 brw_inst_set_exec_size(p
->devinfo
, inst_or
, BRW_EXECUTE_1
);
3584 if (p
->devinfo
->gen
< 12)
3585 brw_inst_set_thread_control(p
->devinfo
, inst_or
, BRW_THREAD_SWITCH
);
3588 if (p
->devinfo
->gen
>= 12)
3589 brw_SYNC(p
, TGL_SYNC_NOP
);