2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
90 const struct gen_device_info
*devinfo
= p
->devinfo
;
92 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
93 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
94 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
95 assert(dest
.nr
< 128);
97 gen7_convert_mrf_to_grf(p
, &dest
);
99 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
100 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
101 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
102 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
103 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
104 assert(dest
.subnr
% 16 == 0);
105 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
106 dest
.vstride
== dest
.width
+ 1);
107 assert(!dest
.negate
&& !dest
.abs
);
108 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
109 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
110 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
112 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
113 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
115 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
116 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
118 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
119 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
120 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
121 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
122 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
124 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
125 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
126 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
127 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
128 assert(dest
.writemask
!= 0);
130 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
131 * Although Dst.HorzStride is a don't care for Align16, HW needs
132 * this to be programmed as "01".
134 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
137 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
139 /* These are different sizes in align1 vs align16:
141 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
142 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
143 dest
.indirect_offset
);
144 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
145 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
146 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
148 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
149 dest
.indirect_offset
);
150 /* even ignored in da16, still need to set as '01' */
151 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
156 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
157 * or 16 (SIMD16), as that's normally correct. However, when dealing with
158 * small registers, it can be useful for us to automatically reduce it to
159 * match the register size.
161 if (p
->automatic_exec_sizes
) {
163 * In platforms that support fp64 we can emit instructions with a width
164 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
165 * these cases we need to make sure that these instructions have their
166 * exec sizes set properly when they are emitted and we can't rely on
167 * this code to fix it.
170 if (devinfo
->gen
>= 6)
171 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
173 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
176 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
181 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
183 const struct gen_device_info
*devinfo
= p
->devinfo
;
185 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
186 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
187 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
188 assert(reg
.nr
< 128);
190 gen7_convert_mrf_to_grf(p
, ®
);
192 if (devinfo
->gen
>= 6 &&
193 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
194 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
195 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
196 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
197 /* Any source modifiers or regions will be ignored, since this just
198 * identifies the MRF/GRF to start reading the message contents from.
199 * Check for some likely failures.
203 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
206 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
207 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
208 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
209 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
210 assert(reg
.subnr
% 16 == 0);
211 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
212 reg
.vstride
== reg
.width
+ 1);
213 assert(!reg
.negate
&& !reg
.abs
);
214 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
215 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
217 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
218 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
219 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
220 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
222 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
223 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
224 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
225 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
226 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
227 reg
.type
== BRW_REGISTER_TYPE_Q
)
228 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
230 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
232 if (type_sz(reg
.type
) < 8) {
233 brw_inst_set_src1_reg_file(devinfo
, inst
,
234 BRW_ARCHITECTURE_REGISTER_FILE
);
235 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
236 brw_inst_src0_reg_hw_type(devinfo
, inst
));
239 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
240 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
241 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
242 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
244 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
247 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
249 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
250 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
252 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
256 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
257 if (reg
.width
== BRW_WIDTH_1
&&
258 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
259 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
260 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
261 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
263 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
264 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
265 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
268 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
269 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
270 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
271 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
272 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
273 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
274 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
275 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
277 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
278 /* This is an oddity of the fact we're using the same
279 * descriptions for registers in align_16 as align_1:
281 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
282 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
283 reg
.type
== BRW_REGISTER_TYPE_DF
&&
284 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
287 * "For Align16 access mode, only encodings of 0000 and 0011
288 * are allowed. Other codes are reserved."
290 * Presumably the DevSNB behavior applies to IVB as well.
292 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
294 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
303 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
305 const struct gen_device_info
*devinfo
= p
->devinfo
;
307 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
308 assert(reg
.nr
< 128);
310 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
311 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
312 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
313 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
314 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
315 assert(reg
.subnr
== 0);
316 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
317 reg
.vstride
== reg
.width
+ 1);
318 assert(!reg
.negate
&& !reg
.abs
);
319 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
320 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
322 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
324 * "Accumulator registers may be accessed explicitly as src0
327 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
328 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
330 gen7_convert_mrf_to_grf(p
, ®
);
331 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
333 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
334 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
335 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
337 /* Only src1 can be immediate in two-argument instructions.
339 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
341 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
342 /* two-argument instructions can only use 32-bit immediates */
343 assert(type_sz(reg
.type
) < 8);
344 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
346 /* This is a hardware restriction, which may or may not be lifted
349 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
350 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
352 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
353 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
354 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
356 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
359 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
360 if (reg
.width
== BRW_WIDTH_1
&&
361 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
362 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
363 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
364 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
366 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
367 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
368 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
371 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
372 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
373 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
374 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
375 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
376 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
377 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
378 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
380 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
381 /* This is an oddity of the fact we're using the same
382 * descriptions for registers in align_16 as align_1:
384 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
385 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
386 reg
.type
== BRW_REGISTER_TYPE_DF
&&
387 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
390 * "For Align16 access mode, only encodings of 0000 and 0011
391 * are allowed. Other codes are reserved."
393 * Presumably the DevSNB behavior applies to IVB as well.
395 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
397 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
405 * Specify the descriptor and extended descriptor immediate for a SEND(C)
406 * message instruction.
409 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
410 unsigned desc
, unsigned ex_desc
)
412 const struct gen_device_info
*devinfo
= p
->devinfo
;
413 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
414 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
415 brw_inst_set_src1_file_type(devinfo
, inst
,
416 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
417 brw_inst_set_send_desc(devinfo
, inst
, desc
);
418 if (devinfo
->gen
>= 9)
419 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
422 static void brw_set_math_message( struct brw_codegen
*p
,
425 unsigned integer_type
,
429 const struct gen_device_info
*devinfo
= p
->devinfo
;
431 unsigned response_length
;
433 /* Infer message length from the function */
435 case BRW_MATH_FUNCTION_POW
:
436 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
437 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
438 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
446 /* Infer response length from the function */
448 case BRW_MATH_FUNCTION_SINCOS
:
449 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
457 brw_set_desc(p
, inst
, brw_message_desc(
458 devinfo
, msg_length
, response_length
, false));
460 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
461 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
462 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
463 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
464 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
465 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
466 brw_inst_set_saturate(devinfo
, inst
, 0);
470 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
473 unsigned response_length
,
476 const struct gen_device_info
*devinfo
= p
->devinfo
;
478 brw_set_desc(p
, insn
, brw_message_desc(
479 devinfo
, 1, response_length
, true));
481 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
482 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
483 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
484 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
485 /* The following fields are not used by FF_SYNC: */
486 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
487 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
488 brw_inst_set_urb_used(devinfo
, insn
, 0);
489 brw_inst_set_urb_complete(devinfo
, insn
, 0);
492 static void brw_set_urb_message( struct brw_codegen
*p
,
494 enum brw_urb_write_flags flags
,
496 unsigned response_length
,
498 unsigned swizzle_control
)
500 const struct gen_device_info
*devinfo
= p
->devinfo
;
502 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
503 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
504 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
506 brw_set_desc(p
, insn
, brw_message_desc(
507 devinfo
, msg_length
, response_length
, true));
509 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
510 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
512 if (flags
& BRW_URB_WRITE_OWORD
) {
513 assert(msg_length
== 2); /* header + one OWORD of data */
514 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
516 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
519 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
520 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
522 if (devinfo
->gen
< 8) {
523 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
526 if (devinfo
->gen
< 7) {
527 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
528 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
530 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
531 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
536 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
540 bool invalidate_after_read
,
542 unsigned addr_offset
,
547 const struct gen_device_info
*devinfo
= p
->devinfo
;
548 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
549 (devinfo
->gen
>= 8 && num_regs
== 8));
550 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
553 brw_set_desc(p
, inst
, brw_message_desc(
554 devinfo
, mlen
, rlen
, header_present
));
556 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
557 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
558 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
559 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
560 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
561 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
562 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
566 brw_inst_set_state(const struct gen_device_info
*devinfo
,
568 const struct brw_insn_state
*state
)
570 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
571 brw_inst_set_group(devinfo
, insn
, state
->group
);
572 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
573 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
574 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
575 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
576 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
577 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
579 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
580 state
->access_mode
== BRW_ALIGN_16
) {
581 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
582 if (devinfo
->gen
>= 7)
583 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
585 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
586 if (devinfo
->gen
>= 7)
587 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
590 if (devinfo
->gen
>= 6)
591 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
594 #define next_insn brw_next_insn
596 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
598 const struct gen_device_info
*devinfo
= p
->devinfo
;
601 if (p
->nr_insn
+ 1 > p
->store_size
) {
603 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
606 p
->next_insn_offset
+= 16;
607 insn
= &p
->store
[p
->nr_insn
++];
609 memset(insn
, 0, sizeof(*insn
));
610 brw_inst_set_opcode(devinfo
, insn
, opcode
);
612 /* Apply the default instruction state */
613 brw_inst_set_state(devinfo
, insn
, p
->current
);
619 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
620 struct brw_reg dest
, struct brw_reg src
)
622 brw_inst
*insn
= next_insn(p
, opcode
);
623 brw_set_dest(p
, insn
, dest
);
624 brw_set_src0(p
, insn
, src
);
629 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
630 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
632 /* 64-bit immediates are only supported on 1-src instructions */
633 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
634 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
636 brw_inst
*insn
= next_insn(p
, opcode
);
637 brw_set_dest(p
, insn
, dest
);
638 brw_set_src0(p
, insn
, src0
);
639 brw_set_src1(p
, insn
, src1
);
644 get_3src_subreg_nr(struct brw_reg reg
)
646 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
647 * use 32-bit units (components 0..7). Since they only support F/D/UD
648 * types, this doesn't lose any flexibility, but uses fewer bits.
650 return reg
.subnr
/ 4;
653 static enum gen10_align1_3src_vertical_stride
654 to_3src_align1_vstride(enum brw_vertical_stride vstride
)
657 case BRW_VERTICAL_STRIDE_0
:
658 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
659 case BRW_VERTICAL_STRIDE_2
:
660 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
661 case BRW_VERTICAL_STRIDE_4
:
662 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
663 case BRW_VERTICAL_STRIDE_8
:
664 case BRW_VERTICAL_STRIDE_16
:
665 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
667 unreachable("invalid vstride");
672 static enum gen10_align1_3src_src_horizontal_stride
673 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
676 case BRW_HORIZONTAL_STRIDE_0
:
677 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
678 case BRW_HORIZONTAL_STRIDE_1
:
679 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
680 case BRW_HORIZONTAL_STRIDE_2
:
681 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
682 case BRW_HORIZONTAL_STRIDE_4
:
683 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
685 unreachable("invalid hstride");
690 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
691 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
693 const struct gen_device_info
*devinfo
= p
->devinfo
;
694 brw_inst
*inst
= next_insn(p
, opcode
);
696 gen7_convert_mrf_to_grf(p
, &dest
);
698 assert(dest
.nr
< 128);
699 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
700 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| src1
.nr
< 128);
701 assert(src2
.file
!= BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
702 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
703 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
704 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
705 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
707 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
708 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
709 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
711 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
712 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
713 BRW_ALIGN1_3SRC_ACCUMULATOR
);
714 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
716 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
717 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
718 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
720 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
722 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
724 if (brw_reg_type_is_floating_point(dest
.type
)) {
725 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
726 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
728 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
729 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
732 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
733 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
734 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
735 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
737 brw_inst_set_3src_a1_src0_vstride(devinfo
, inst
,
738 to_3src_align1_vstride(src0
.vstride
));
739 brw_inst_set_3src_a1_src1_vstride(devinfo
, inst
,
740 to_3src_align1_vstride(src1
.vstride
));
741 /* no vstride on src2 */
743 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
744 to_3src_align1_hstride(src0
.hstride
));
745 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
746 to_3src_align1_hstride(src1
.hstride
));
747 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
748 to_3src_align1_hstride(src2
.hstride
));
750 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
751 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
752 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
754 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
756 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
757 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
759 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
760 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
761 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
763 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
765 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
766 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
768 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
769 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
770 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
771 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
773 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
774 src0
.file
== BRW_IMMEDIATE_VALUE
||
775 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
776 src0
.type
== BRW_REGISTER_TYPE_NF
));
777 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
778 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
779 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
780 src2
.file
== BRW_IMMEDIATE_VALUE
);
782 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
783 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
784 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
785 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
786 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
787 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
788 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
789 BRW_ALIGN1_3SRC_ACCUMULATOR
);
790 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
791 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
792 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
793 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
795 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
796 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
797 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
798 dest
.type
== BRW_REGISTER_TYPE_DF
||
799 dest
.type
== BRW_REGISTER_TYPE_D
||
800 dest
.type
== BRW_REGISTER_TYPE_UD
||
801 (dest
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 8));
802 if (devinfo
->gen
== 6) {
803 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
804 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
806 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
807 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
808 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
810 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
811 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
812 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
813 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
814 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
815 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
816 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
817 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
819 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
820 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
821 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
822 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
823 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
824 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
825 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
826 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
828 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
829 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
830 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
831 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
832 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
833 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
834 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
835 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
837 if (devinfo
->gen
>= 7) {
838 /* Set both the source and destination types based on dest.type,
839 * ignoring the source register types. The MAD and LRP emitters ensure
840 * that all four types are float. The BFE and BFI2 emitters, however,
841 * may send us mixed D and UD types and want us to ignore that and use
842 * the destination type.
844 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
845 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
847 /* From the Bspec, 3D Media GPGPU, Instruction fields, srcType:
849 * "Three source instructions can use operands with mixed-mode
850 * precision. When SrcType field is set to :f or :hf it defines
851 * precision for source 0 only, and fields Src1Type and Src2Type
852 * define precision for other source operands:
854 * 0b = :f. Single precision Float (32-bit).
855 * 1b = :hf. Half precision Float (16-bit)."
857 if (src1
.type
== BRW_REGISTER_TYPE_HF
)
858 brw_inst_set_3src_a16_src1_type(devinfo
, inst
, 1);
860 if (src2
.type
== BRW_REGISTER_TYPE_HF
)
861 brw_inst_set_3src_a16_src2_type(devinfo
, inst
, 1);
869 /***********************************************************************
870 * Convenience routines.
873 brw_inst *brw_##OP(struct brw_codegen *p, \
874 struct brw_reg dest, \
875 struct brw_reg src0) \
877 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
881 brw_inst *brw_##OP(struct brw_codegen *p, \
882 struct brw_reg dest, \
883 struct brw_reg src0, \
884 struct brw_reg src1) \
886 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
890 brw_inst *brw_##OP(struct brw_codegen *p, \
891 struct brw_reg dest, \
892 struct brw_reg src0, \
893 struct brw_reg src1, \
894 struct brw_reg src2) \
896 if (p->current->access_mode == BRW_ALIGN_16) { \
897 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
898 src0.swizzle = BRW_SWIZZLE_XXXX; \
899 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
900 src1.swizzle = BRW_SWIZZLE_XXXX; \
901 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
902 src2.swizzle = BRW_SWIZZLE_XXXX; \
904 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
908 brw_inst *brw_##OP(struct brw_codegen *p, \
909 struct brw_reg dest, \
910 struct brw_reg src0, \
911 struct brw_reg src1, \
912 struct brw_reg src2) \
914 assert(dest.type == BRW_REGISTER_TYPE_F || \
915 dest.type == BRW_REGISTER_TYPE_DF); \
916 if (dest.type == BRW_REGISTER_TYPE_F) { \
917 assert(src0.type == BRW_REGISTER_TYPE_F); \
918 assert(src1.type == BRW_REGISTER_TYPE_F); \
919 assert(src2.type == BRW_REGISTER_TYPE_F); \
920 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
921 assert(src0.type == BRW_REGISTER_TYPE_DF); \
922 assert(src1.type == BRW_REGISTER_TYPE_DF); \
923 assert(src2.type == BRW_REGISTER_TYPE_DF); \
926 if (p->current->access_mode == BRW_ALIGN_16) { \
927 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
928 src0.swizzle = BRW_SWIZZLE_XXXX; \
929 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
930 src1.swizzle = BRW_SWIZZLE_XXXX; \
931 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
932 src2.swizzle = BRW_SWIZZLE_XXXX; \
934 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
937 /* Rounding operations (other than RNDD) require two instructions - the first
938 * stores a rounded value (possibly the wrong way) in the dest register, but
939 * also sets a per-channel "increment bit" in the flag register. A predicated
940 * add of 1.0 fixes dest to contain the desired result.
942 * Sandybridge and later appear to round correctly without an ADD.
945 void brw_##OP(struct brw_codegen *p, \
946 struct brw_reg dest, \
947 struct brw_reg src) \
949 const struct gen_device_info *devinfo = p->devinfo; \
950 brw_inst *rnd, *add; \
951 rnd = next_insn(p, BRW_OPCODE_##OP); \
952 brw_set_dest(p, rnd, dest); \
953 brw_set_src0(p, rnd, src); \
955 if (devinfo->gen < 6) { \
956 /* turn on round-increments */ \
957 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
958 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
959 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
999 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
1001 const struct gen_device_info
*devinfo
= p
->devinfo
;
1003 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
1004 * To avoid the problems that causes, we use an <X,2,0> source region to
1005 * read each element twice.
1007 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
1008 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
1009 dest
.type
== BRW_REGISTER_TYPE_DF
&&
1010 (src0
.type
== BRW_REGISTER_TYPE_F
||
1011 src0
.type
== BRW_REGISTER_TYPE_D
||
1012 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
1013 !has_scalar_region(src0
)) {
1014 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
1015 src0
.vstride
= src0
.hstride
;
1016 src0
.width
= BRW_WIDTH_2
;
1017 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1020 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1024 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1025 struct brw_reg src0
, struct brw_reg src1
)
1028 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1029 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1030 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1031 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1032 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1035 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1036 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1037 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1038 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1039 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1042 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1046 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1047 struct brw_reg src0
, struct brw_reg src1
)
1049 assert(dest
.type
== src0
.type
);
1050 assert(src0
.type
== src1
.type
);
1051 switch (src0
.type
) {
1052 case BRW_REGISTER_TYPE_B
:
1053 case BRW_REGISTER_TYPE_UB
:
1054 case BRW_REGISTER_TYPE_W
:
1055 case BRW_REGISTER_TYPE_UW
:
1056 case BRW_REGISTER_TYPE_D
:
1057 case BRW_REGISTER_TYPE_UD
:
1060 unreachable("Bad type for brw_AVG");
1063 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1067 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1068 struct brw_reg src0
, struct brw_reg src1
)
1071 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1072 src0
.type
== BRW_REGISTER_TYPE_UD
||
1073 src1
.type
== BRW_REGISTER_TYPE_D
||
1074 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1075 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1078 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1079 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1080 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1081 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1082 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1085 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1086 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1087 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1088 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1089 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1092 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1093 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1094 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1095 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1097 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1101 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1102 struct brw_reg src0
, struct brw_reg src1
)
1104 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1105 src0
.width
= BRW_WIDTH_1
;
1106 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1107 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1111 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1112 struct brw_reg src0
, struct brw_reg src1
)
1114 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1115 src0
.width
= BRW_WIDTH_1
;
1116 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1117 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1118 src1
.width
= BRW_WIDTH_8
;
1119 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1120 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1124 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1126 const struct gen_device_info
*devinfo
= p
->devinfo
;
1127 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1128 /* The F32TO16 instruction doesn't support 32-bit destination types in
1129 * Align1 mode, and neither does the Gen8 implementation in terms of a
1130 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1131 * an undocumented feature.
1133 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1134 (!align16
|| devinfo
->gen
>= 8));
1138 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1140 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1141 dst
.type
== BRW_REGISTER_TYPE_W
||
1142 dst
.type
== BRW_REGISTER_TYPE_UW
||
1143 dst
.type
== BRW_REGISTER_TYPE_HF
);
1146 brw_push_insn_state(p
);
1148 if (needs_zero_fill
) {
1149 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1150 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1153 if (devinfo
->gen
>= 8) {
1154 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1156 assert(devinfo
->gen
== 7);
1157 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1160 if (needs_zero_fill
) {
1161 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1162 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1163 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1166 brw_pop_insn_state(p
);
1171 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1173 const struct gen_device_info
*devinfo
= p
->devinfo
;
1174 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1177 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1179 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1181 * Because this instruction does not have a 16-bit floating-point
1182 * type, the source data type must be Word (W). The destination type
1183 * must be F (Float).
1185 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1186 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1188 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1189 src
.type
== BRW_REGISTER_TYPE_UW
||
1190 src
.type
== BRW_REGISTER_TYPE_HF
);
1193 if (devinfo
->gen
>= 8) {
1194 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1196 assert(devinfo
->gen
== 7);
1197 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1202 void brw_NOP(struct brw_codegen
*p
)
1204 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1205 memset(insn
, 0, sizeof(*insn
));
1206 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1213 /***********************************************************************
1214 * Comparisons, if/else/endif
1218 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1219 unsigned predicate_control
)
1221 const struct gen_device_info
*devinfo
= p
->devinfo
;
1222 struct brw_reg ip
= brw_ip_reg();
1223 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1225 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1226 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1227 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1228 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1234 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1236 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1238 p
->if_stack_depth
++;
1239 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1240 p
->if_stack_array_size
*= 2;
1241 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1242 p
->if_stack_array_size
);
1247 pop_if_stack(struct brw_codegen
*p
)
1249 p
->if_stack_depth
--;
1250 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1254 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1256 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1257 p
->loop_stack_array_size
*= 2;
1258 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1259 p
->loop_stack_array_size
);
1260 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1261 p
->loop_stack_array_size
);
1264 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1265 p
->loop_stack_depth
++;
1266 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1270 get_inner_do_insn(struct brw_codegen
*p
)
1272 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1275 /* EU takes the value from the flag register and pushes it onto some
1276 * sort of a stack (presumably merging with any flag value already on
1277 * the stack). Within an if block, the flags at the top of the stack
1278 * control execution on each channel of the unit, eg. on each of the
1279 * 16 pixel values in our wm programs.
1281 * When the matching 'else' instruction is reached (presumably by
1282 * countdown of the instruction count patched in by our ELSE/ENDIF
1283 * functions), the relevant flags are inverted.
1285 * When the matching 'endif' instruction is reached, the flags are
1286 * popped off. If the stack is now empty, normal execution resumes.
1289 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1291 const struct gen_device_info
*devinfo
= p
->devinfo
;
1294 insn
= next_insn(p
, BRW_OPCODE_IF
);
1296 /* Override the defaults for this instruction:
1298 if (devinfo
->gen
< 6) {
1299 brw_set_dest(p
, insn
, brw_ip_reg());
1300 brw_set_src0(p
, insn
, brw_ip_reg());
1301 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1302 } else if (devinfo
->gen
== 6) {
1303 brw_set_dest(p
, insn
, brw_imm_w(0));
1304 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1305 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1306 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1307 } else if (devinfo
->gen
== 7) {
1308 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1309 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1310 brw_set_src1(p
, insn
, brw_imm_w(0));
1311 brw_inst_set_jip(devinfo
, insn
, 0);
1312 brw_inst_set_uip(devinfo
, insn
, 0);
1314 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1315 brw_set_src0(p
, insn
, brw_imm_d(0));
1316 brw_inst_set_jip(devinfo
, insn
, 0);
1317 brw_inst_set_uip(devinfo
, insn
, 0);
1320 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1321 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1322 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1323 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1324 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1325 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1327 push_if_stack(p
, insn
);
1328 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1332 /* This function is only used for gen6-style IF instructions with an
1333 * embedded comparison (conditional modifier). It is not used on gen7.
1336 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1337 struct brw_reg src0
, struct brw_reg src1
)
1339 const struct gen_device_info
*devinfo
= p
->devinfo
;
1342 insn
= next_insn(p
, BRW_OPCODE_IF
);
1344 brw_set_dest(p
, insn
, brw_imm_w(0));
1345 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1346 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1347 brw_set_src0(p
, insn
, src0
);
1348 brw_set_src1(p
, insn
, src1
);
1350 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1351 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1352 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1354 push_if_stack(p
, insn
);
1359 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1362 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1363 brw_inst
*if_inst
, brw_inst
*else_inst
)
1365 const struct gen_device_info
*devinfo
= p
->devinfo
;
1367 /* The next instruction (where the ENDIF would be, if it existed) */
1368 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1370 assert(p
->single_program_flow
);
1371 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1372 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1373 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1375 /* Convert IF to an ADD instruction that moves the instruction pointer
1376 * to the first instruction of the ELSE block. If there is no ELSE
1377 * block, point to where ENDIF would be. Reverse the predicate.
1379 * There's no need to execute an ENDIF since we don't need to do any
1380 * stack operations, and if we're currently executing, we just want to
1381 * continue normally.
1383 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1384 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1386 if (else_inst
!= NULL
) {
1387 /* Convert ELSE to an ADD instruction that points where the ENDIF
1390 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1392 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1393 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1395 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1400 * Patch IF and ELSE instructions with appropriate jump targets.
1403 patch_IF_ELSE(struct brw_codegen
*p
,
1404 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1406 const struct gen_device_info
*devinfo
= p
->devinfo
;
1408 /* We shouldn't be patching IF and ELSE instructions in single program flow
1409 * mode when gen < 6, because in single program flow mode on those
1410 * platforms, we convert flow control instructions to conditional ADDs that
1411 * operate on IP (see brw_ENDIF).
1413 * However, on Gen6, writing to IP doesn't work in single program flow mode
1414 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1415 * not be updated by non-flow control instructions."). And on later
1416 * platforms, there is no significant benefit to converting control flow
1417 * instructions to conditional ADDs. So we do patch IF and ELSE
1418 * instructions in single program flow mode on those platforms.
1420 if (devinfo
->gen
< 6)
1421 assert(!p
->single_program_flow
);
1423 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1424 assert(endif_inst
!= NULL
);
1425 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1427 unsigned br
= brw_jump_scale(devinfo
);
1429 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1430 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1432 if (else_inst
== NULL
) {
1433 /* Patch IF -> ENDIF */
1434 if (devinfo
->gen
< 6) {
1435 /* Turn it into an IFF, which means no mask stack operations for
1436 * all-false and jumping past the ENDIF.
1438 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1439 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1440 br
* (endif_inst
- if_inst
+ 1));
1441 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1442 } else if (devinfo
->gen
== 6) {
1443 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1444 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1446 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1447 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1450 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1452 /* Patch IF -> ELSE */
1453 if (devinfo
->gen
< 6) {
1454 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1455 br
* (else_inst
- if_inst
));
1456 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1457 } else if (devinfo
->gen
== 6) {
1458 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1459 br
* (else_inst
- if_inst
+ 1));
1462 /* Patch ELSE -> ENDIF */
1463 if (devinfo
->gen
< 6) {
1464 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1467 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1468 br
* (endif_inst
- else_inst
+ 1));
1469 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1470 } else if (devinfo
->gen
== 6) {
1471 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1472 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1473 br
* (endif_inst
- else_inst
));
1475 /* The IF instruction's JIP should point just past the ELSE */
1476 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1477 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1478 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1479 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1480 if (devinfo
->gen
>= 8) {
1481 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1482 * should point to ENDIF.
1484 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1491 brw_ELSE(struct brw_codegen
*p
)
1493 const struct gen_device_info
*devinfo
= p
->devinfo
;
1496 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1498 if (devinfo
->gen
< 6) {
1499 brw_set_dest(p
, insn
, brw_ip_reg());
1500 brw_set_src0(p
, insn
, brw_ip_reg());
1501 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1502 } else if (devinfo
->gen
== 6) {
1503 brw_set_dest(p
, insn
, brw_imm_w(0));
1504 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1505 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1506 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1507 } else if (devinfo
->gen
== 7) {
1508 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1509 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1510 brw_set_src1(p
, insn
, brw_imm_w(0));
1511 brw_inst_set_jip(devinfo
, insn
, 0);
1512 brw_inst_set_uip(devinfo
, insn
, 0);
1514 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1515 brw_set_src0(p
, insn
, brw_imm_d(0));
1516 brw_inst_set_jip(devinfo
, insn
, 0);
1517 brw_inst_set_uip(devinfo
, insn
, 0);
1520 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1521 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1522 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1523 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1525 push_if_stack(p
, insn
);
1529 brw_ENDIF(struct brw_codegen
*p
)
1531 const struct gen_device_info
*devinfo
= p
->devinfo
;
1532 brw_inst
*insn
= NULL
;
1533 brw_inst
*else_inst
= NULL
;
1534 brw_inst
*if_inst
= NULL
;
1536 bool emit_endif
= true;
1538 /* In single program flow mode, we can express IF and ELSE instructions
1539 * equivalently as ADD instructions that operate on IP. On platforms prior
1540 * to Gen6, flow control instructions cause an implied thread switch, so
1541 * this is a significant savings.
1543 * However, on Gen6, writing to IP doesn't work in single program flow mode
1544 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1545 * not be updated by non-flow control instructions."). And on later
1546 * platforms, there is no significant benefit to converting control flow
1547 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1550 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1554 * A single next_insn() may change the base address of instruction store
1555 * memory(p->store), so call it first before referencing the instruction
1556 * store pointer from an index
1559 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1561 /* Pop the IF and (optional) ELSE instructions from the stack */
1562 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1563 tmp
= pop_if_stack(p
);
1564 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1566 tmp
= pop_if_stack(p
);
1571 /* ENDIF is useless; don't bother emitting it. */
1572 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1576 if (devinfo
->gen
< 6) {
1577 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1578 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1579 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1580 } else if (devinfo
->gen
== 6) {
1581 brw_set_dest(p
, insn
, brw_imm_w(0));
1582 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1583 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1584 } else if (devinfo
->gen
== 7) {
1585 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1586 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1587 brw_set_src1(p
, insn
, brw_imm_w(0));
1589 brw_set_src0(p
, insn
, brw_imm_d(0));
1592 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1593 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1594 if (devinfo
->gen
< 6)
1595 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1597 /* Also pop item off the stack in the endif instruction: */
1598 if (devinfo
->gen
< 6) {
1599 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1600 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1601 } else if (devinfo
->gen
== 6) {
1602 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1604 brw_inst_set_jip(devinfo
, insn
, 2);
1606 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1610 brw_BREAK(struct brw_codegen
*p
)
1612 const struct gen_device_info
*devinfo
= p
->devinfo
;
1615 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1616 if (devinfo
->gen
>= 8) {
1617 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1618 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1619 } else if (devinfo
->gen
>= 6) {
1620 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1621 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1622 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1624 brw_set_dest(p
, insn
, brw_ip_reg());
1625 brw_set_src0(p
, insn
, brw_ip_reg());
1626 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1627 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1628 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1630 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1631 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1637 brw_CONT(struct brw_codegen
*p
)
1639 const struct gen_device_info
*devinfo
= p
->devinfo
;
1642 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1643 brw_set_dest(p
, insn
, brw_ip_reg());
1644 if (devinfo
->gen
>= 8) {
1645 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1647 brw_set_src0(p
, insn
, brw_ip_reg());
1648 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1651 if (devinfo
->gen
< 6) {
1652 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1653 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1655 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1656 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1661 gen6_HALT(struct brw_codegen
*p
)
1663 const struct gen_device_info
*devinfo
= p
->devinfo
;
1666 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1667 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1668 if (devinfo
->gen
>= 8) {
1669 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1671 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1672 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1675 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1676 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1682 * The DO/WHILE is just an unterminated loop -- break or continue are
1683 * used for control within the loop. We have a few ways they can be
1686 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1687 * jip and no DO instruction.
1689 * For non-uniform control flow pre-gen6, there's a DO instruction to
1690 * push the mask, and a WHILE to jump back, and BREAK to get out and
1693 * For gen6, there's no more mask stack, so no need for DO. WHILE
1694 * just points back to the first instruction of the loop.
1697 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1699 const struct gen_device_info
*devinfo
= p
->devinfo
;
1701 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1702 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1703 return &p
->store
[p
->nr_insn
];
1705 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1707 push_loop_stack(p
, insn
);
1709 /* Override the defaults for this instruction:
1711 brw_set_dest(p
, insn
, brw_null_reg());
1712 brw_set_src0(p
, insn
, brw_null_reg());
1713 brw_set_src1(p
, insn
, brw_null_reg());
1715 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1716 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1717 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1724 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1727 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1728 * nesting, since it can always just point to the end of the block/current loop.
1731 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1733 const struct gen_device_info
*devinfo
= p
->devinfo
;
1734 brw_inst
*do_inst
= get_inner_do_insn(p
);
1736 unsigned br
= brw_jump_scale(devinfo
);
1738 assert(devinfo
->gen
< 6);
1740 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1741 /* If the jump count is != 0, that means that this instruction has already
1742 * been patched because it's part of a loop inside of the one we're
1745 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1746 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1747 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1748 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1749 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1750 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1756 brw_WHILE(struct brw_codegen
*p
)
1758 const struct gen_device_info
*devinfo
= p
->devinfo
;
1759 brw_inst
*insn
, *do_insn
;
1760 unsigned br
= brw_jump_scale(devinfo
);
1762 if (devinfo
->gen
>= 6) {
1763 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1764 do_insn
= get_inner_do_insn(p
);
1766 if (devinfo
->gen
>= 8) {
1767 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1768 brw_set_src0(p
, insn
, brw_imm_d(0));
1769 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1770 } else if (devinfo
->gen
== 7) {
1771 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1772 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1773 brw_set_src1(p
, insn
, brw_imm_w(0));
1774 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1776 brw_set_dest(p
, insn
, brw_imm_w(0));
1777 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1778 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1779 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1782 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1785 if (p
->single_program_flow
) {
1786 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1787 do_insn
= get_inner_do_insn(p
);
1789 brw_set_dest(p
, insn
, brw_ip_reg());
1790 brw_set_src0(p
, insn
, brw_ip_reg());
1791 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1792 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1794 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1795 do_insn
= get_inner_do_insn(p
);
1797 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1799 brw_set_dest(p
, insn
, brw_ip_reg());
1800 brw_set_src0(p
, insn
, brw_ip_reg());
1801 brw_set_src1(p
, insn
, brw_imm_d(0));
1803 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1804 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1805 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1807 brw_patch_break_cont(p
, insn
);
1810 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1812 p
->loop_stack_depth
--;
1819 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1821 const struct gen_device_info
*devinfo
= p
->devinfo
;
1822 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1825 if (devinfo
->gen
>= 5)
1828 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1829 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1831 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1832 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1835 /* To integrate with the above, it makes sense that the comparison
1836 * instruction should populate the flag register. It might be simpler
1837 * just to use the flag reg for most WM tasks?
1839 void brw_CMP(struct brw_codegen
*p
,
1840 struct brw_reg dest
,
1841 unsigned conditional
,
1842 struct brw_reg src0
,
1843 struct brw_reg src1
)
1845 const struct gen_device_info
*devinfo
= p
->devinfo
;
1846 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1848 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1849 brw_set_dest(p
, insn
, dest
);
1850 brw_set_src0(p
, insn
, src0
);
1851 brw_set_src1(p
, insn
, src1
);
1853 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1855 * "Any CMP instruction with a null destination must use a {switch}."
1857 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1858 * mentioned on their work-arounds pages.
1860 if (devinfo
->gen
== 7) {
1861 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1862 dest
.nr
== BRW_ARF_NULL
) {
1863 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1868 /***********************************************************************
1869 * Helpers for the various SEND message types:
1872 /** Extended math function, float[8].
1874 void gen4_math(struct brw_codegen
*p
,
1875 struct brw_reg dest
,
1877 unsigned msg_reg_nr
,
1879 unsigned precision
)
1881 const struct gen_device_info
*devinfo
= p
->devinfo
;
1882 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1884 if (has_scalar_region(src
)) {
1885 data_type
= BRW_MATH_DATA_SCALAR
;
1887 data_type
= BRW_MATH_DATA_VECTOR
;
1890 assert(devinfo
->gen
< 6);
1892 /* Example code doesn't set predicate_control for send
1895 brw_inst_set_pred_control(devinfo
, insn
, 0);
1896 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1898 brw_set_dest(p
, insn
, dest
);
1899 brw_set_src0(p
, insn
, src
);
1900 brw_set_math_message(p
,
1903 src
.type
== BRW_REGISTER_TYPE_D
,
1908 void gen6_math(struct brw_codegen
*p
,
1909 struct brw_reg dest
,
1911 struct brw_reg src0
,
1912 struct brw_reg src1
)
1914 const struct gen_device_info
*devinfo
= p
->devinfo
;
1915 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1917 assert(devinfo
->gen
>= 6);
1919 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1920 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1922 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1923 if (devinfo
->gen
== 6) {
1924 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1925 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1928 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1929 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1930 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1931 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1932 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1933 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1934 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1936 assert(src0
.type
== BRW_REGISTER_TYPE_F
||
1937 (src0
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
1938 assert(src1
.type
== BRW_REGISTER_TYPE_F
||
1939 (src1
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
1942 /* Source modifiers are ignored for extended math instructions on Gen6. */
1943 if (devinfo
->gen
== 6) {
1944 assert(!src0
.negate
);
1946 assert(!src1
.negate
);
1950 brw_inst_set_math_function(devinfo
, insn
, function
);
1952 brw_set_dest(p
, insn
, dest
);
1953 brw_set_src0(p
, insn
, src0
);
1954 brw_set_src1(p
, insn
, src1
);
1958 * Return the right surface index to access the thread scratch space using
1959 * stateless dataport messages.
1962 brw_scratch_surface_idx(const struct brw_codegen
*p
)
1964 /* The scratch space is thread-local so IA coherency is unnecessary. */
1965 if (p
->devinfo
->gen
>= 8)
1966 return GEN8_BTI_STATELESS_NON_COHERENT
;
1968 return BRW_BTI_STATELESS
;
1972 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1973 * using a constant offset per channel.
1975 * The offset must be aligned to oword size (16 bytes). Used for
1976 * register spilling.
1978 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1983 const struct gen_device_info
*devinfo
= p
->devinfo
;
1984 const unsigned target_cache
=
1985 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
1986 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
1987 BRW_SFID_DATAPORT_WRITE
);
1990 if (devinfo
->gen
>= 6)
1993 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1995 const unsigned mlen
= 1 + num_regs
;
1997 /* Set up the message header. This is g0, with g0.2 filled with
1998 * the offset. We don't want to leave our offset around in g0 or
1999 * it'll screw up texture samples, so set it up inside the message
2003 brw_push_insn_state(p
);
2004 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2005 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2006 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2008 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2010 /* set message header global offset field (reg 0, element 2) */
2011 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2013 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2015 2), BRW_REGISTER_TYPE_UD
),
2016 brw_imm_ud(offset
));
2018 brw_pop_insn_state(p
);
2022 struct brw_reg dest
;
2023 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2024 int send_commit_msg
;
2025 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2026 BRW_REGISTER_TYPE_UW
);
2028 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2029 brw_inst_set_compression(devinfo
, insn
, false);
2031 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2032 src_header
= vec16(src_header
);
2034 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2035 if (devinfo
->gen
< 6)
2036 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2038 /* Until gen6, writes followed by reads from the same location
2039 * are not guaranteed to be ordered unless write_commit is set.
2040 * If set, then a no-op write is issued to the destination
2041 * register to set a dependency, and a read from the destination
2042 * can be used to ensure the ordering.
2044 * For gen6, only writes between different threads need ordering
2045 * protection. Our use of DP writes is all about register
2046 * spilling within a thread.
2048 if (devinfo
->gen
>= 6) {
2049 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2050 send_commit_msg
= 0;
2053 send_commit_msg
= 1;
2056 brw_set_dest(p
, insn
, dest
);
2057 if (devinfo
->gen
>= 6) {
2058 brw_set_src0(p
, insn
, mrf
);
2060 brw_set_src0(p
, insn
, brw_null_reg());
2063 if (devinfo
->gen
>= 6)
2064 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2066 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2068 brw_set_desc(p
, insn
,
2069 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2070 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2071 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2072 msg_type
, 0, /* not a render target */
2079 * Read a block of owords (half a GRF each) from the scratch buffer
2080 * using a constant index per channel.
2082 * Offset must be aligned to oword size (16 bytes). Used for register
2086 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2087 struct brw_reg dest
,
2092 const struct gen_device_info
*devinfo
= p
->devinfo
;
2094 if (devinfo
->gen
>= 6)
2097 if (p
->devinfo
->gen
>= 7) {
2098 /* On gen 7 and above, we no longer have message registers and we can
2099 * send from any register we want. By using the destination register
2100 * for the message, we guarantee that the implied message write won't
2101 * accidentally overwrite anything. This has been a problem because
2102 * the MRF registers and source for the final FB write are both fixed
2105 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2107 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2109 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2111 const unsigned rlen
= num_regs
;
2112 const unsigned target_cache
=
2113 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2114 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2115 BRW_SFID_DATAPORT_READ
);
2118 brw_push_insn_state(p
);
2119 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2120 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2121 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2123 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2125 /* set message header global offset field (reg 0, element 2) */
2126 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2127 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2129 brw_pop_insn_state(p
);
2133 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2135 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2136 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2137 brw_inst_set_compression(devinfo
, insn
, false);
2139 brw_set_dest(p
, insn
, dest
); /* UW? */
2140 if (devinfo
->gen
>= 6) {
2141 brw_set_src0(p
, insn
, mrf
);
2143 brw_set_src0(p
, insn
, brw_null_reg());
2144 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2147 brw_set_desc(p
, insn
,
2148 brw_message_desc(devinfo
, 1, rlen
, true) |
2149 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2150 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2151 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2152 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2157 gen7_block_read_scratch(struct brw_codegen
*p
,
2158 struct brw_reg dest
,
2162 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2163 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2165 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2167 /* The HW requires that the header is present; this is to get the g0.5
2170 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2172 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2173 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2174 * is 32 bytes, which happens to be the size of a register.
2177 assert(offset
< (1 << 12));
2179 gen7_set_dp_scratch_message(p
, insn
,
2180 false, /* scratch read */
2182 false, /* invalidate after read */
2185 1, /* mlen: just g0 */
2186 num_regs
, /* rlen */
2187 true); /* header present */
2191 * Read float[4] vectors from the data port constant cache.
2192 * Location (in buffer) should be a multiple of 16.
2193 * Used for fetching shader constants.
2195 void brw_oword_block_read(struct brw_codegen
*p
,
2196 struct brw_reg dest
,
2199 uint32_t bind_table_index
)
2201 const struct gen_device_info
*devinfo
= p
->devinfo
;
2202 const unsigned target_cache
=
2203 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2204 BRW_SFID_DATAPORT_READ
);
2205 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2207 /* On newer hardware, offset is in units of owords. */
2208 if (devinfo
->gen
>= 6)
2211 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2213 brw_push_insn_state(p
);
2214 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2215 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2216 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2218 brw_push_insn_state(p
);
2219 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2220 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2222 /* set message header global offset field (reg 0, element 2) */
2223 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2225 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2227 2), BRW_REGISTER_TYPE_UD
),
2228 brw_imm_ud(offset
));
2229 brw_pop_insn_state(p
);
2231 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2233 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2235 /* cast dest to a uword[8] vector */
2236 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2238 brw_set_dest(p
, insn
, dest
);
2239 if (devinfo
->gen
>= 6) {
2240 brw_set_src0(p
, insn
, mrf
);
2242 brw_set_src0(p
, insn
, brw_null_reg());
2243 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2246 brw_set_desc(p
, insn
,
2247 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2248 brw_dp_read_desc(devinfo
, bind_table_index
,
2249 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2250 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2251 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2253 brw_pop_insn_state(p
);
2257 brw_fb_WRITE(struct brw_codegen
*p
,
2258 struct brw_reg payload
,
2259 struct brw_reg implied_header
,
2260 unsigned msg_control
,
2261 unsigned binding_table_index
,
2262 unsigned msg_length
,
2263 unsigned response_length
,
2265 bool last_render_target
,
2266 bool header_present
)
2268 const struct gen_device_info
*devinfo
= p
->devinfo
;
2269 const unsigned target_cache
=
2270 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2271 BRW_SFID_DATAPORT_WRITE
);
2274 struct brw_reg dest
, src0
;
2276 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2277 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2279 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2281 if (devinfo
->gen
>= 6) {
2282 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2284 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2286 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2287 brw_inst_set_compression(devinfo
, insn
, false);
2289 if (devinfo
->gen
>= 6) {
2290 /* headerless version, just submit color payload */
2293 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2295 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2296 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2297 src0
= implied_header
;
2299 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2302 brw_set_dest(p
, insn
, dest
);
2303 brw_set_src0(p
, insn
, src0
);
2304 brw_set_desc(p
, insn
,
2305 brw_message_desc(devinfo
, msg_length
, response_length
,
2307 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2308 msg_type
, last_render_target
,
2309 0 /* send_commit_msg */));
2310 brw_inst_set_eot(devinfo
, insn
, eot
);
2316 gen9_fb_READ(struct brw_codegen
*p
,
2318 struct brw_reg payload
,
2319 unsigned binding_table_index
,
2320 unsigned msg_length
,
2321 unsigned response_length
,
2324 const struct gen_device_info
*devinfo
= p
->devinfo
;
2325 assert(devinfo
->gen
>= 9);
2326 const unsigned msg_subtype
=
2327 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2328 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2330 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2331 brw_set_dest(p
, insn
, dst
);
2332 brw_set_src0(p
, insn
, payload
);
2335 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2336 brw_dp_read_desc(devinfo
, binding_table_index
,
2337 per_sample
<< 5 | msg_subtype
,
2338 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2339 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2340 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2346 * Texture sample instruction.
2347 * Note: the msg_type plus msg_length values determine exactly what kind
2348 * of sampling operation is performed. See volume 4, page 161 of docs.
2350 void brw_SAMPLE(struct brw_codegen
*p
,
2351 struct brw_reg dest
,
2352 unsigned msg_reg_nr
,
2353 struct brw_reg src0
,
2354 unsigned binding_table_index
,
2357 unsigned response_length
,
2358 unsigned msg_length
,
2359 unsigned header_present
,
2361 unsigned return_format
)
2363 const struct gen_device_info
*devinfo
= p
->devinfo
;
2366 if (msg_reg_nr
!= -1)
2367 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2369 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2370 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2371 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2373 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2375 * "Instruction compression is not allowed for this instruction (that
2376 * is, send). The hardware behavior is undefined if this instruction is
2377 * set as compressed. However, compress control can be set to "SecHalf"
2378 * to affect the EMask generation."
2380 * No similar wording is found in later PRMs, but there are examples
2381 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2382 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2383 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2385 brw_inst_set_compression(devinfo
, insn
, false);
2387 if (devinfo
->gen
< 6)
2388 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2390 brw_set_dest(p
, insn
, dest
);
2391 brw_set_src0(p
, insn
, src0
);
2392 brw_set_desc(p
, insn
,
2393 brw_message_desc(devinfo
, msg_length
, response_length
,
2395 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2396 msg_type
, simd_mode
, return_format
));
2399 /* Adjust the message header's sampler state pointer to
2400 * select the correct group of 16 samplers.
2402 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2403 struct brw_reg header
,
2404 struct brw_reg sampler_index
)
2406 /* The "Sampler Index" field can only store values between 0 and 15.
2407 * However, we can add an offset to the "Sampler State Pointer"
2408 * field, effectively selecting a different set of 16 samplers.
2410 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2411 * offset, and each sampler state is only 16-bytes, so we can't
2412 * exclusively use the offset - we have to use both.
2415 const struct gen_device_info
*devinfo
= p
->devinfo
;
2417 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2418 const int sampler_state_size
= 16; /* 16 bytes */
2419 uint32_t sampler
= sampler_index
.ud
;
2421 if (sampler
>= 16) {
2422 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2424 get_element_ud(header
, 3),
2425 get_element_ud(brw_vec8_grf(0, 0), 3),
2426 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2429 /* Non-const sampler array indexing case */
2430 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2434 struct brw_reg temp
= get_element_ud(header
, 3);
2436 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2437 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2439 get_element_ud(header
, 3),
2440 get_element_ud(brw_vec8_grf(0, 0), 3),
2445 /* All these variables are pretty confusing - we might be better off
2446 * using bitmasks and macros for this, in the old style. Or perhaps
2447 * just having the caller instantiate the fields in dword3 itself.
2449 void brw_urb_WRITE(struct brw_codegen
*p
,
2450 struct brw_reg dest
,
2451 unsigned msg_reg_nr
,
2452 struct brw_reg src0
,
2453 enum brw_urb_write_flags flags
,
2454 unsigned msg_length
,
2455 unsigned response_length
,
2459 const struct gen_device_info
*devinfo
= p
->devinfo
;
2462 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2464 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2465 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2466 brw_push_insn_state(p
);
2467 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2468 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2469 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2470 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2471 BRW_REGISTER_TYPE_UD
),
2472 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2473 brw_imm_ud(0xff00));
2474 brw_pop_insn_state(p
);
2477 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2479 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2481 brw_set_dest(p
, insn
, dest
);
2482 brw_set_src0(p
, insn
, src0
);
2483 brw_set_src1(p
, insn
, brw_imm_d(0));
2485 if (devinfo
->gen
< 6)
2486 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2488 brw_set_urb_message(p
,
2498 brw_send_indirect_message(struct brw_codegen
*p
,
2501 struct brw_reg payload
,
2502 struct brw_reg desc
,
2506 const struct gen_device_info
*devinfo
= p
->devinfo
;
2507 struct brw_inst
*send
;
2509 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2511 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2513 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2514 send
= next_insn(p
, BRW_OPCODE_SEND
);
2515 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2518 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2520 brw_push_insn_state(p
);
2521 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2522 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2523 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2524 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2526 /* Load the indirect descriptor to an address register using OR so the
2527 * caller can specify additional descriptor bits with the desc_imm
2530 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2532 brw_pop_insn_state(p
);
2534 send
= next_insn(p
, BRW_OPCODE_SEND
);
2535 brw_set_src1(p
, send
, addr
);
2538 brw_set_dest(p
, send
, dst
);
2539 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2540 brw_inst_set_sfid(devinfo
, send
, sfid
);
2541 brw_inst_set_eot(devinfo
, send
, eot
);
2545 brw_send_indirect_split_message(struct brw_codegen
*p
,
2548 struct brw_reg payload0
,
2549 struct brw_reg payload1
,
2550 struct brw_reg desc
,
2552 struct brw_reg ex_desc
,
2553 unsigned ex_desc_imm
,
2556 const struct gen_device_info
*devinfo
= p
->devinfo
;
2557 struct brw_inst
*send
;
2559 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2561 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2563 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2564 desc
.ud
|= desc_imm
;
2566 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2568 brw_push_insn_state(p
);
2569 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2570 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2571 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2572 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2574 /* Load the indirect descriptor to an address register using OR so the
2575 * caller can specify additional descriptor bits with the desc_imm
2578 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2580 brw_pop_insn_state(p
);
2584 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2585 ex_desc
.ud
|= ex_desc_imm
;
2587 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2589 brw_push_insn_state(p
);
2590 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2591 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2592 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2593 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2595 /* Load the indirect extended descriptor to an address register using OR
2596 * so the caller can specify additional descriptor bits with the
2597 * desc_imm immediate.
2599 * Even though the instruction dispatcher always pulls the SFID and EOT
2600 * fields from the instruction itself, actual external unit which
2601 * processes the message gets the SFID and EOT from the extended
2602 * descriptor which comes from the address register. If we don't OR
2603 * those two bits in, the external unit may get confused and hang.
2605 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(ex_desc_imm
| sfid
| eot
<< 5));
2607 brw_pop_insn_state(p
);
2611 send
= next_insn(p
, BRW_OPCODE_SENDS
);
2612 brw_set_dest(p
, send
, dst
);
2613 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2614 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2616 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2617 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2618 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2620 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2621 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2622 assert(desc
.subnr
== 0);
2623 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2626 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2627 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2628 brw_inst_set_send_ex_desc(devinfo
, send
, ex_desc
.ud
);
2630 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2631 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2632 assert((ex_desc
.subnr
& 0x3) == 0);
2633 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2634 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2637 brw_inst_set_sfid(devinfo
, send
, sfid
);
2638 brw_inst_set_eot(devinfo
, send
, eot
);
2642 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2645 struct brw_reg payload
,
2646 struct brw_reg surface
,
2649 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2650 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2652 brw_push_insn_state(p
);
2653 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2654 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2655 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2656 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2658 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2659 * some surface array is accessed out of bounds.
2662 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2663 BRW_GET_SWZ(surface
.swizzle
, 0)),
2666 brw_pop_insn_state(p
);
2671 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2675 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2676 brw_inst
*insn
, int while_offset
, int start_offset
)
2678 int scale
= 16 / brw_jump_scale(devinfo
);
2679 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2680 : brw_inst_jip(devinfo
, insn
);
2682 return while_offset
+ jip
* scale
<= start_offset
;
2687 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2690 void *store
= p
->store
;
2691 const struct gen_device_info
*devinfo
= p
->devinfo
;
2695 for (offset
= next_offset(devinfo
, store
, start_offset
);
2696 offset
< p
->next_insn_offset
;
2697 offset
= next_offset(devinfo
, store
, offset
)) {
2698 brw_inst
*insn
= store
+ offset
;
2700 switch (brw_inst_opcode(devinfo
, insn
)) {
2704 case BRW_OPCODE_ENDIF
:
2709 case BRW_OPCODE_WHILE
:
2710 /* If the while doesn't jump before our instruction, it's the end
2711 * of a sibling do...while loop. Ignore it.
2713 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2716 case BRW_OPCODE_ELSE
:
2717 case BRW_OPCODE_HALT
:
2726 /* There is no DO instruction on gen6, so to find the end of the loop
2727 * we have to see if the loop is jumping back before our start
2731 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2733 const struct gen_device_info
*devinfo
= p
->devinfo
;
2735 void *store
= p
->store
;
2737 assert(devinfo
->gen
>= 6);
2739 /* Always start after the instruction (such as a WHILE) we're trying to fix
2742 for (offset
= next_offset(devinfo
, store
, start_offset
);
2743 offset
< p
->next_insn_offset
;
2744 offset
= next_offset(devinfo
, store
, offset
)) {
2745 brw_inst
*insn
= store
+ offset
;
2747 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2748 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2752 assert(!"not reached");
2753 return start_offset
;
2756 /* After program generation, go back and update the UIP and JIP of
2757 * BREAK, CONT, and HALT instructions to their correct locations.
2760 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2762 const struct gen_device_info
*devinfo
= p
->devinfo
;
2764 int br
= brw_jump_scale(devinfo
);
2765 int scale
= 16 / br
;
2766 void *store
= p
->store
;
2768 if (devinfo
->gen
< 6)
2771 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2772 brw_inst
*insn
= store
+ offset
;
2773 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2775 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2776 switch (brw_inst_opcode(devinfo
, insn
)) {
2777 case BRW_OPCODE_BREAK
:
2778 assert(block_end_offset
!= 0);
2779 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2780 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2781 brw_inst_set_uip(devinfo
, insn
,
2782 (brw_find_loop_end(p
, offset
) - offset
+
2783 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2785 case BRW_OPCODE_CONTINUE
:
2786 assert(block_end_offset
!= 0);
2787 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2788 brw_inst_set_uip(devinfo
, insn
,
2789 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2791 assert(brw_inst_uip(devinfo
, insn
) != 0);
2792 assert(brw_inst_jip(devinfo
, insn
) != 0);
2795 case BRW_OPCODE_ENDIF
: {
2796 int32_t jump
= (block_end_offset
== 0) ?
2797 1 * br
: (block_end_offset
- offset
) / scale
;
2798 if (devinfo
->gen
>= 7)
2799 brw_inst_set_jip(devinfo
, insn
, jump
);
2801 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2805 case BRW_OPCODE_HALT
:
2806 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2808 * "In case of the halt instruction not inside any conditional
2809 * code block, the value of <JIP> and <UIP> should be the
2810 * same. In case of the halt instruction inside conditional code
2811 * block, the <UIP> should be the end of the program, and the
2812 * <JIP> should be end of the most inner conditional code block."
2814 * The uip will have already been set by whoever set up the
2817 if (block_end_offset
== 0) {
2818 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2820 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2822 assert(brw_inst_uip(devinfo
, insn
) != 0);
2823 assert(brw_inst_jip(devinfo
, insn
) != 0);
2829 void brw_ff_sync(struct brw_codegen
*p
,
2830 struct brw_reg dest
,
2831 unsigned msg_reg_nr
,
2832 struct brw_reg src0
,
2834 unsigned response_length
,
2837 const struct gen_device_info
*devinfo
= p
->devinfo
;
2840 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2842 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2843 brw_set_dest(p
, insn
, dest
);
2844 brw_set_src0(p
, insn
, src0
);
2845 brw_set_src1(p
, insn
, brw_imm_d(0));
2847 if (devinfo
->gen
< 6)
2848 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2850 brw_set_ff_sync_message(p
,
2858 * Emit the SEND instruction necessary to generate stream output data on Gen6
2859 * (for transform feedback).
2861 * If send_commit_msg is true, this is the last piece of stream output data
2862 * from this thread, so send the data as a committed write. According to the
2863 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2865 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2866 * writes are complete by sending the final write as a committed write."
2869 brw_svb_write(struct brw_codegen
*p
,
2870 struct brw_reg dest
,
2871 unsigned msg_reg_nr
,
2872 struct brw_reg src0
,
2873 unsigned binding_table_index
,
2874 bool send_commit_msg
)
2876 const struct gen_device_info
*devinfo
= p
->devinfo
;
2877 const unsigned target_cache
=
2878 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2879 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2880 BRW_SFID_DATAPORT_WRITE
);
2883 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2885 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2886 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2887 brw_set_dest(p
, insn
, dest
);
2888 brw_set_src0(p
, insn
, src0
);
2889 brw_set_desc(p
, insn
,
2890 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
2891 brw_dp_write_desc(devinfo
, binding_table_index
,
2892 0, /* msg_control: ignored */
2893 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2894 0, /* last_render_target: ignored */
2895 send_commit_msg
)); /* send_commit_msg */
2899 brw_surface_payload_size(struct brw_codegen
*p
,
2900 unsigned num_channels
,
2901 unsigned exec_size
/**< 0 for SIMD4x2 */)
2904 return 1; /* SIMD4x2 */
2905 else if (exec_size
<= 8)
2906 return num_channels
;
2908 return 2 * num_channels
;
2912 brw_untyped_atomic(struct brw_codegen
*p
,
2914 struct brw_reg payload
,
2915 struct brw_reg surface
,
2917 unsigned msg_length
,
2918 bool response_expected
,
2919 bool header_present
)
2921 const struct gen_device_info
*devinfo
= p
->devinfo
;
2922 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2923 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2924 GEN7_SFID_DATAPORT_DATA_CACHE
);
2925 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2926 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
2927 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2928 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2929 has_simd4x2
? 0 : 8;
2930 const unsigned response_length
=
2931 brw_surface_payload_size(p
, response_expected
, exec_size
);
2932 const unsigned desc
=
2933 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
2934 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
2936 /* Mask out unused components -- This is especially important in Align16
2937 * mode on generations that don't have native support for SIMD4x2 atomics,
2938 * because unused but enabled components will cause the dataport to perform
2939 * additional atomic operations on the addresses that happen to be in the
2940 * uninitialized Y, Z and W coordinates of the payload.
2942 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2944 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
2945 payload
, surface
, desc
);
2949 brw_untyped_surface_read(struct brw_codegen
*p
,
2951 struct brw_reg payload
,
2952 struct brw_reg surface
,
2953 unsigned msg_length
,
2954 unsigned num_channels
)
2956 const struct gen_device_info
*devinfo
= p
->devinfo
;
2957 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2958 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2959 GEN7_SFID_DATAPORT_DATA_CACHE
);
2960 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2961 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
2962 const unsigned response_length
=
2963 brw_surface_payload_size(p
, num_channels
, exec_size
);
2964 const unsigned desc
=
2965 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
2966 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
2968 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
2972 brw_untyped_surface_write(struct brw_codegen
*p
,
2973 struct brw_reg payload
,
2974 struct brw_reg surface
,
2975 unsigned msg_length
,
2976 unsigned num_channels
,
2977 bool header_present
)
2979 const struct gen_device_info
*devinfo
= p
->devinfo
;
2980 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2981 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2982 GEN7_SFID_DATAPORT_DATA_CACHE
);
2983 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2984 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
2985 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2986 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2987 has_simd4x2
? 0 : 8;
2988 const unsigned desc
=
2989 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
2990 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
2991 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2992 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
2994 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2995 payload
, surface
, desc
);
2999 brw_set_memory_fence_message(struct brw_codegen
*p
,
3000 struct brw_inst
*insn
,
3001 enum brw_message_target sfid
,
3004 const struct gen_device_info
*devinfo
= p
->devinfo
;
3006 brw_set_desc(p
, insn
, brw_message_desc(
3007 devinfo
, 1, (commit_enable
? 1 : 0), true));
3009 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3012 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3013 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3015 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3016 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3019 unreachable("Not reached");
3023 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3027 brw_memory_fence(struct brw_codegen
*p
,
3029 enum opcode send_op
)
3031 const struct gen_device_info
*devinfo
= p
->devinfo
;
3032 const bool commit_enable
=
3033 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3034 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3035 struct brw_inst
*insn
;
3037 brw_push_insn_state(p
);
3038 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3039 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3042 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3043 * message doesn't write anything back.
3045 insn
= next_insn(p
, send_op
);
3046 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3047 brw_set_dest(p
, insn
, dst
);
3048 brw_set_src0(p
, insn
, dst
);
3049 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3052 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3053 /* IVB does typed surface access through the render cache, so we need to
3054 * flush it too. Use a different register so both flushes can be
3055 * pipelined by the hardware.
3057 insn
= next_insn(p
, send_op
);
3058 brw_set_dest(p
, insn
, offset(dst
, 1));
3059 brw_set_src0(p
, insn
, offset(dst
, 1));
3060 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3063 /* Now write the response of the second message into the response of the
3064 * first to trigger a pipeline stall -- This way future render and data
3065 * cache messages will be properly ordered with respect to past data and
3066 * render cache messages.
3068 brw_MOV(p
, dst
, offset(dst
, 1));
3071 brw_pop_insn_state(p
);
3075 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3076 struct brw_reg dest
,
3080 struct brw_reg data
,
3081 unsigned msg_length
,
3082 unsigned response_length
)
3084 const struct gen_device_info
*devinfo
= p
->devinfo
;
3085 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3086 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3087 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3088 const unsigned desc
=
3089 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3090 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3093 /* brw_send_indirect_message will automatically use a direct send message
3094 * if data is actually immediate.
3096 brw_send_indirect_message(p
,
3097 GEN7_SFID_PIXEL_INTERPOLATOR
,
3106 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3107 struct brw_reg mask
)
3109 const struct gen_device_info
*devinfo
= p
->devinfo
;
3110 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3111 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3114 assert(devinfo
->gen
>= 7);
3115 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3117 brw_push_insn_state(p
);
3119 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3120 * unnecessary bits in the instruction words, get the information we need
3121 * and reset the default flag register. This allows more instructions to be
3124 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3125 brw_set_default_flag_reg(p
, 0, 0);
3127 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3128 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3130 if (devinfo
->gen
>= 8) {
3131 /* Getting the first active channel index is easy on Gen8: Just find
3132 * the first bit set in the execution mask. The register exists on
3133 * HSW already but it reads back as all ones when the current
3134 * instruction has execution masking disabled, so it's kind of
3137 struct brw_reg exec_mask
=
3138 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3140 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3141 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3142 /* Unfortunately, ce0 does not take into account the thread
3143 * dispatch mask, which may be a problem in cases where it's not
3144 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3145 * some n). Combine ce0 with the given dispatch (or vector) mask
3146 * to mask off those channels which were never dispatched by the
3149 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3150 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3151 exec_mask
= vec1(dst
);
3154 /* Quarter control has the effect of magically shifting the value of
3155 * ce0 so you'll get the first active channel relative to the
3156 * specified quarter control as result.
3158 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3160 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3162 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3163 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3165 /* Run enough instructions returning zero with execution masking and
3166 * a conditional modifier enabled in order to get the full execution
3167 * mask in f1.0. We could use a single 32-wide move here if it
3168 * weren't because of the hardware bug that causes channel enables to
3169 * be applied incorrectly to the second half of 32-wide instructions
3172 const unsigned lower_size
= MIN2(16, exec_size
);
3173 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3174 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3176 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3177 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3178 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3179 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3180 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3181 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3184 /* Find the first bit set in the exec_size-wide portion of the flag
3185 * register that was updated by the last sequence of MOV
3188 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3189 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3190 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3193 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3195 if (devinfo
->gen
>= 8 &&
3196 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3197 /* In SIMD4x2 mode the first active channel index is just the
3198 * negation of the first bit of the mask register. Note that ce0
3199 * doesn't take into account the dispatch mask, so the Gen7 path
3200 * should be used instead unless you have the guarantee that the
3201 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3204 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3205 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3209 /* Overwrite the destination without and with execution masking to
3210 * find out which of the channels is active.
3212 brw_push_insn_state(p
);
3213 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3214 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3217 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3219 brw_pop_insn_state(p
);
3220 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3224 brw_pop_insn_state(p
);
3228 brw_broadcast(struct brw_codegen
*p
,
3233 const struct gen_device_info
*devinfo
= p
->devinfo
;
3234 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3237 brw_push_insn_state(p
);
3238 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3239 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3241 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3242 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3243 assert(!src
.abs
&& !src
.negate
);
3244 assert(src
.type
== dst
.type
);
3246 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3247 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3248 /* Trivial, the source is already uniform or the index is a constant.
3249 * We will typically not get here if the optimizer is doing its job, but
3250 * asserting would be mean.
3252 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3254 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3255 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3257 /* From the Haswell PRM section "Register Region Restrictions":
3259 * "The lower bits of the AddressImmediate must not overflow to
3260 * change the register address. The lower 5 bits of Address
3261 * Immediate when added to lower 5 bits of address register gives
3262 * the sub-register offset. The upper bits of Address Immediate
3263 * when added to upper bits of address register gives the register
3264 * address. Any overflow from sub-register offset is dropped."
3266 * Fortunately, for broadcast, we never have a sub-register offset so
3267 * this isn't an issue.
3269 assert(src
.subnr
== 0);
3272 const struct brw_reg addr
=
3273 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3274 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3275 /* Limit in bytes of the signed indirect addressing immediate. */
3276 const unsigned limit
= 512;
3278 brw_push_insn_state(p
);
3279 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3280 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3282 /* Take into account the component size and horizontal stride. */
3283 assert(src
.vstride
== src
.hstride
+ src
.width
);
3284 brw_SHL(p
, addr
, vec1(idx
),
3285 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3288 /* We can only address up to limit bytes using the indirect
3289 * addressing immediate, account for the difference if the source
3290 * register is above this limit.
3292 if (offset
>= limit
) {
3293 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3294 offset
= offset
% limit
;
3297 brw_pop_insn_state(p
);
3299 /* Use indirect addressing to fetch the specified component. */
3300 if (type_sz(src
.type
) > 4 &&
3301 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3302 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3304 * "When source or destination datatype is 64b or operation is
3305 * integer DWord multiply, indirect addressing must not be
3308 * To work around both of this issue, we do two integer MOVs
3309 * insead of one 64-bit MOV. Because no double value should ever
3310 * cross a register boundary, it's safe to use the immediate
3311 * offset in the indirect here to handle adding 4 bytes to the
3312 * offset and avoid the extra ADD to the register file.
3314 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3315 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3316 BRW_REGISTER_TYPE_D
));
3317 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3318 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3319 BRW_REGISTER_TYPE_D
));
3322 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3325 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3326 * to all bits of a flag register,
3330 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3331 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3332 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3333 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3335 /* and use predicated SEL to pick the right channel. */
3336 inst
= brw_SEL(p
, dst
,
3337 stride(suboffset(src
, 4), 4, 4, 1),
3338 stride(src
, 4, 4, 1));
3339 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3340 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3344 brw_pop_insn_state(p
);
3348 * This instruction is generated as a single-channel align1 instruction by
3349 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3351 * We can't use the typed atomic op in the FS because that has the execution
3352 * mask ANDed with the pixel mask, but we just want to write the one dword for
3355 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3356 * one u32. So we use the same untyped atomic write message as the pixel
3359 * The untyped atomic operation requires a BUFFER surface type with RAW
3360 * format, and is only accessible through the legacy DATA_CACHE dataport
3363 void brw_shader_time_add(struct brw_codegen
*p
,
3364 struct brw_reg payload
,
3365 uint32_t surf_index
)
3367 const struct gen_device_info
*devinfo
= p
->devinfo
;
3368 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3369 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3370 GEN7_SFID_DATAPORT_DATA_CACHE
);
3371 assert(devinfo
->gen
>= 7);
3373 brw_push_insn_state(p
);
3374 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3375 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3376 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3377 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3379 /* We use brw_vec1_reg and unmasked because we want to increment the given
3382 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3384 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3386 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3387 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3390 brw_inst_set_sfid(devinfo
, send
, sfid
);
3391 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3393 brw_pop_insn_state(p
);
3398 * Emit the SEND message for a barrier
3401 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3403 const struct gen_device_info
*devinfo
= p
->devinfo
;
3404 struct brw_inst
*inst
;
3406 assert(devinfo
->gen
>= 7);
3408 brw_push_insn_state(p
);
3409 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3410 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3411 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3412 brw_set_src0(p
, inst
, src
);
3413 brw_set_src1(p
, inst
, brw_null_reg());
3414 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3416 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3417 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3418 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3419 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3421 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3422 brw_pop_insn_state(p
);
3427 * Emit the wait instruction for a barrier
3430 brw_WAIT(struct brw_codegen
*p
)
3432 const struct gen_device_info
*devinfo
= p
->devinfo
;
3433 struct brw_inst
*insn
;
3435 struct brw_reg src
= brw_notification_reg();
3437 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3438 brw_set_dest(p
, insn
, src
);
3439 brw_set_src0(p
, insn
, src
);
3440 brw_set_src1(p
, insn
, brw_null_reg());
3442 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3443 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3447 * Changes the floating point rounding mode updating the control register
3448 * field defined at cr0.0[5-6] bits. This function supports the changes to
3449 * RTNE (00), RU (01), RD (10) and RTZ (11) rounding using bitwise operations.
3450 * Only RTNE and RTZ rounding are enabled at nir.
3453 brw_rounding_mode(struct brw_codegen
*p
,
3454 enum brw_rnd_mode mode
)
3456 const unsigned bits
= mode
<< BRW_CR0_RND_MODE_SHIFT
;
3458 if (bits
!= BRW_CR0_RND_MODE_MASK
) {
3459 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3460 brw_imm_ud(~BRW_CR0_RND_MODE_MASK
));
3461 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3463 /* From the Skylake PRM, Volume 7, page 760:
3464 * "Implementation Restriction on Register Access: When the control
3465 * register is used as an explicit source and/or destination, hardware
3466 * does not ensure execution pipeline coherency. Software must set the
3467 * thread control field to ‘switch’ for an instruction that uses
3468 * control register as an explicit operand."
3470 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3474 brw_inst
*inst
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3476 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3477 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);