2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
90 const struct gen_device_info
*devinfo
= p
->devinfo
;
92 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
93 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
94 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
95 assert(dest
.nr
< 128);
97 gen7_convert_mrf_to_grf(p
, &dest
);
99 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
100 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
101 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
102 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
103 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
104 assert(dest
.subnr
% 16 == 0);
105 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
106 dest
.vstride
== dest
.width
+ 1);
107 assert(!dest
.negate
&& !dest
.abs
);
108 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
109 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
110 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
112 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
113 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
115 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
116 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
118 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
119 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
120 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
121 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
122 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
124 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
125 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
126 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
127 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
128 assert(dest
.writemask
!= 0);
130 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
131 * Although Dst.HorzStride is a don't care for Align16, HW needs
132 * this to be programmed as "01".
134 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
137 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
139 /* These are different sizes in align1 vs align16:
141 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
142 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
143 dest
.indirect_offset
);
144 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
145 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
146 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
148 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
149 dest
.indirect_offset
);
150 /* even ignored in da16, still need to set as '01' */
151 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
156 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
157 * or 16 (SIMD16), as that's normally correct. However, when dealing with
158 * small registers, it can be useful for us to automatically reduce it to
159 * match the register size.
161 if (p
->automatic_exec_sizes
) {
163 * In platforms that support fp64 we can emit instructions with a width
164 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
165 * these cases we need to make sure that these instructions have their
166 * exec sizes set properly when they are emitted and we can't rely on
167 * this code to fix it.
170 if (devinfo
->gen
>= 6)
171 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
173 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
176 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
181 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
183 const struct gen_device_info
*devinfo
= p
->devinfo
;
185 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
186 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
187 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
188 assert(reg
.nr
< 128);
190 gen7_convert_mrf_to_grf(p
, ®
);
192 if (devinfo
->gen
>= 6 &&
193 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
194 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
195 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
196 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
197 /* Any source modifiers or regions will be ignored, since this just
198 * identifies the MRF/GRF to start reading the message contents from.
199 * Check for some likely failures.
203 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
206 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
207 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
208 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
209 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
210 assert(reg
.subnr
% 16 == 0);
211 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
212 reg
.vstride
== reg
.width
+ 1);
213 assert(!reg
.negate
&& !reg
.abs
);
214 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
215 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
217 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
218 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
219 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
220 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
222 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
223 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
224 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
225 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
226 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
227 reg
.type
== BRW_REGISTER_TYPE_Q
)
228 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
230 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
232 if (type_sz(reg
.type
) < 8) {
233 brw_inst_set_src1_reg_file(devinfo
, inst
,
234 BRW_ARCHITECTURE_REGISTER_FILE
);
235 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
236 brw_inst_src0_reg_hw_type(devinfo
, inst
));
239 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
240 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
241 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
242 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
244 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
247 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
249 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
250 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
252 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
256 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
257 if (reg
.width
== BRW_WIDTH_1
&&
258 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
259 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
260 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
261 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
263 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
264 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
265 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
268 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
269 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
270 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
271 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
272 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
273 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
274 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
275 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
277 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
278 /* This is an oddity of the fact we're using the same
279 * descriptions for registers in align_16 as align_1:
281 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
282 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
283 reg
.type
== BRW_REGISTER_TYPE_DF
&&
284 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
287 * "For Align16 access mode, only encodings of 0000 and 0011
288 * are allowed. Other codes are reserved."
290 * Presumably the DevSNB behavior applies to IVB as well.
292 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
294 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
303 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
305 const struct gen_device_info
*devinfo
= p
->devinfo
;
307 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
308 assert(reg
.nr
< 128);
310 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
311 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
312 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
313 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
314 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
315 assert(reg
.subnr
== 0);
316 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
317 reg
.vstride
== reg
.width
+ 1);
318 assert(!reg
.negate
&& !reg
.abs
);
319 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
320 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
322 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
324 * "Accumulator registers may be accessed explicitly as src0
327 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
328 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
330 gen7_convert_mrf_to_grf(p
, ®
);
331 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
333 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
334 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
335 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
337 /* Only src1 can be immediate in two-argument instructions.
339 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
341 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
342 /* two-argument instructions can only use 32-bit immediates */
343 assert(type_sz(reg
.type
) < 8);
344 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
346 /* This is a hardware restriction, which may or may not be lifted
349 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
350 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
352 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
353 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
354 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
356 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
359 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
360 if (reg
.width
== BRW_WIDTH_1
&&
361 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
362 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
363 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
364 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
366 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
367 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
368 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
371 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
372 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
373 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
374 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
375 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
376 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
377 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
378 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
380 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
381 /* This is an oddity of the fact we're using the same
382 * descriptions for registers in align_16 as align_1:
384 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
385 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
386 reg
.type
== BRW_REGISTER_TYPE_DF
&&
387 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
390 * "For Align16 access mode, only encodings of 0000 and 0011
391 * are allowed. Other codes are reserved."
393 * Presumably the DevSNB behavior applies to IVB as well.
395 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
397 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
405 * Specify the descriptor and extended descriptor immediate for a SEND(C)
406 * message instruction.
409 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
410 unsigned desc
, unsigned ex_desc
)
412 const struct gen_device_info
*devinfo
= p
->devinfo
;
413 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
414 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
415 brw_inst_set_src1_file_type(devinfo
, inst
,
416 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
417 brw_inst_set_send_desc(devinfo
, inst
, desc
);
418 if (devinfo
->gen
>= 9)
419 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
422 static void brw_set_math_message( struct brw_codegen
*p
,
425 unsigned integer_type
,
429 const struct gen_device_info
*devinfo
= p
->devinfo
;
431 unsigned response_length
;
433 /* Infer message length from the function */
435 case BRW_MATH_FUNCTION_POW
:
436 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
437 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
438 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
446 /* Infer response length from the function */
448 case BRW_MATH_FUNCTION_SINCOS
:
449 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
457 brw_set_desc(p
, inst
, brw_message_desc(
458 devinfo
, msg_length
, response_length
, false));
460 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
461 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
462 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
463 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
464 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
465 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
466 brw_inst_set_saturate(devinfo
, inst
, 0);
470 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
473 unsigned response_length
,
476 const struct gen_device_info
*devinfo
= p
->devinfo
;
478 brw_set_desc(p
, insn
, brw_message_desc(
479 devinfo
, 1, response_length
, true));
481 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
482 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
483 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
484 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
485 /* The following fields are not used by FF_SYNC: */
486 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
487 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
488 brw_inst_set_urb_used(devinfo
, insn
, 0);
489 brw_inst_set_urb_complete(devinfo
, insn
, 0);
492 static void brw_set_urb_message( struct brw_codegen
*p
,
494 enum brw_urb_write_flags flags
,
496 unsigned response_length
,
498 unsigned swizzle_control
)
500 const struct gen_device_info
*devinfo
= p
->devinfo
;
502 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
503 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
504 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
506 brw_set_desc(p
, insn
, brw_message_desc(
507 devinfo
, msg_length
, response_length
, true));
509 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
510 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
512 if (flags
& BRW_URB_WRITE_OWORD
) {
513 assert(msg_length
== 2); /* header + one OWORD of data */
514 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
516 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
519 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
520 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
522 if (devinfo
->gen
< 8) {
523 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
526 if (devinfo
->gen
< 7) {
527 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
528 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
530 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
531 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
536 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
540 bool invalidate_after_read
,
542 unsigned addr_offset
,
547 const struct gen_device_info
*devinfo
= p
->devinfo
;
548 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
549 (devinfo
->gen
>= 8 && num_regs
== 8));
550 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
553 brw_set_desc(p
, inst
, brw_message_desc(
554 devinfo
, mlen
, rlen
, header_present
));
556 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
557 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
558 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
559 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
560 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
561 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
562 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
566 brw_inst_set_state(const struct gen_device_info
*devinfo
,
568 const struct brw_insn_state
*state
)
570 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
571 brw_inst_set_group(devinfo
, insn
, state
->group
);
572 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
573 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
574 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
575 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
576 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
577 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
579 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
580 state
->access_mode
== BRW_ALIGN_16
) {
581 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
582 if (devinfo
->gen
>= 7)
583 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
585 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
586 if (devinfo
->gen
>= 7)
587 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
590 if (devinfo
->gen
>= 6)
591 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
594 #define next_insn brw_next_insn
596 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
598 const struct gen_device_info
*devinfo
= p
->devinfo
;
601 if (p
->nr_insn
+ 1 > p
->store_size
) {
603 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
606 p
->next_insn_offset
+= 16;
607 insn
= &p
->store
[p
->nr_insn
++];
609 memset(insn
, 0, sizeof(*insn
));
610 brw_inst_set_opcode(devinfo
, insn
, opcode
);
612 /* Apply the default instruction state */
613 brw_inst_set_state(devinfo
, insn
, p
->current
);
619 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
620 struct brw_reg dest
, struct brw_reg src
)
622 brw_inst
*insn
= next_insn(p
, opcode
);
623 brw_set_dest(p
, insn
, dest
);
624 brw_set_src0(p
, insn
, src
);
629 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
630 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
632 /* 64-bit immediates are only supported on 1-src instructions */
633 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
634 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
636 brw_inst
*insn
= next_insn(p
, opcode
);
637 brw_set_dest(p
, insn
, dest
);
638 brw_set_src0(p
, insn
, src0
);
639 brw_set_src1(p
, insn
, src1
);
644 get_3src_subreg_nr(struct brw_reg reg
)
646 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
647 * use 32-bit units (components 0..7). Since they only support F/D/UD
648 * types, this doesn't lose any flexibility, but uses fewer bits.
650 return reg
.subnr
/ 4;
653 static enum gen10_align1_3src_vertical_stride
654 to_3src_align1_vstride(enum brw_vertical_stride vstride
)
657 case BRW_VERTICAL_STRIDE_0
:
658 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
659 case BRW_VERTICAL_STRIDE_2
:
660 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
661 case BRW_VERTICAL_STRIDE_4
:
662 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
663 case BRW_VERTICAL_STRIDE_8
:
664 case BRW_VERTICAL_STRIDE_16
:
665 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
667 unreachable("invalid vstride");
672 static enum gen10_align1_3src_src_horizontal_stride
673 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
676 case BRW_HORIZONTAL_STRIDE_0
:
677 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
678 case BRW_HORIZONTAL_STRIDE_1
:
679 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
680 case BRW_HORIZONTAL_STRIDE_2
:
681 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
682 case BRW_HORIZONTAL_STRIDE_4
:
683 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
685 unreachable("invalid hstride");
690 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
691 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
693 const struct gen_device_info
*devinfo
= p
->devinfo
;
694 brw_inst
*inst
= next_insn(p
, opcode
);
696 gen7_convert_mrf_to_grf(p
, &dest
);
698 assert(dest
.nr
< 128);
699 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
700 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| src1
.nr
< 128);
701 assert(src2
.file
!= BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
702 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
703 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
704 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
705 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
707 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
708 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
709 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
711 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
712 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
713 BRW_ALIGN1_3SRC_ACCUMULATOR
);
714 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
716 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
717 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
718 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
720 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
722 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
724 if (brw_reg_type_is_floating_point(dest
.type
)) {
725 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
726 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
728 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
729 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
732 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
733 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
734 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
735 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
737 brw_inst_set_3src_a1_src0_vstride(devinfo
, inst
,
738 to_3src_align1_vstride(src0
.vstride
));
739 brw_inst_set_3src_a1_src1_vstride(devinfo
, inst
,
740 to_3src_align1_vstride(src1
.vstride
));
741 /* no vstride on src2 */
743 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
744 to_3src_align1_hstride(src0
.hstride
));
745 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
746 to_3src_align1_hstride(src1
.hstride
));
747 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
748 to_3src_align1_hstride(src2
.hstride
));
750 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
751 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
752 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
754 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
756 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
757 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
759 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
760 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
761 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
763 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
765 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
766 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
768 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
769 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
770 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
771 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
773 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
774 src0
.file
== BRW_IMMEDIATE_VALUE
||
775 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
776 src0
.type
== BRW_REGISTER_TYPE_NF
));
777 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
778 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
779 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
780 src2
.file
== BRW_IMMEDIATE_VALUE
);
782 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
783 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
784 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
785 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
786 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
787 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
788 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
789 BRW_ALIGN1_3SRC_ACCUMULATOR
);
790 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
791 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
792 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
793 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
795 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
796 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
797 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
798 dest
.type
== BRW_REGISTER_TYPE_DF
||
799 dest
.type
== BRW_REGISTER_TYPE_D
||
800 dest
.type
== BRW_REGISTER_TYPE_UD
);
801 if (devinfo
->gen
== 6) {
802 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
803 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
805 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
806 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
807 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
809 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
810 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
811 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
812 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
813 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
814 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
815 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
816 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
818 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
819 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
820 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
821 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
822 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
823 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
824 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
825 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
827 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
828 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
829 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
830 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
831 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
832 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
833 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
834 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
836 if (devinfo
->gen
>= 7) {
837 /* Set both the source and destination types based on dest.type,
838 * ignoring the source register types. The MAD and LRP emitters ensure
839 * that all four types are float. The BFE and BFI2 emitters, however,
840 * may send us mixed D and UD types and want us to ignore that and use
841 * the destination type.
843 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
844 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
852 /***********************************************************************
853 * Convenience routines.
856 brw_inst *brw_##OP(struct brw_codegen *p, \
857 struct brw_reg dest, \
858 struct brw_reg src0) \
860 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
864 brw_inst *brw_##OP(struct brw_codegen *p, \
865 struct brw_reg dest, \
866 struct brw_reg src0, \
867 struct brw_reg src1) \
869 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
873 brw_inst *brw_##OP(struct brw_codegen *p, \
874 struct brw_reg dest, \
875 struct brw_reg src0, \
876 struct brw_reg src1, \
877 struct brw_reg src2) \
879 if (p->current->access_mode == BRW_ALIGN_16) { \
880 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
881 src0.swizzle = BRW_SWIZZLE_XXXX; \
882 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
883 src1.swizzle = BRW_SWIZZLE_XXXX; \
884 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
885 src2.swizzle = BRW_SWIZZLE_XXXX; \
887 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
891 brw_inst *brw_##OP(struct brw_codegen *p, \
892 struct brw_reg dest, \
893 struct brw_reg src0, \
894 struct brw_reg src1, \
895 struct brw_reg src2) \
897 assert(dest.type == BRW_REGISTER_TYPE_F || \
898 dest.type == BRW_REGISTER_TYPE_DF); \
899 if (dest.type == BRW_REGISTER_TYPE_F) { \
900 assert(src0.type == BRW_REGISTER_TYPE_F); \
901 assert(src1.type == BRW_REGISTER_TYPE_F); \
902 assert(src2.type == BRW_REGISTER_TYPE_F); \
903 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
904 assert(src0.type == BRW_REGISTER_TYPE_DF); \
905 assert(src1.type == BRW_REGISTER_TYPE_DF); \
906 assert(src2.type == BRW_REGISTER_TYPE_DF); \
909 if (p->current->access_mode == BRW_ALIGN_16) { \
910 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
911 src0.swizzle = BRW_SWIZZLE_XXXX; \
912 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
913 src1.swizzle = BRW_SWIZZLE_XXXX; \
914 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
915 src2.swizzle = BRW_SWIZZLE_XXXX; \
917 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
920 /* Rounding operations (other than RNDD) require two instructions - the first
921 * stores a rounded value (possibly the wrong way) in the dest register, but
922 * also sets a per-channel "increment bit" in the flag register. A predicated
923 * add of 1.0 fixes dest to contain the desired result.
925 * Sandybridge and later appear to round correctly without an ADD.
928 void brw_##OP(struct brw_codegen *p, \
929 struct brw_reg dest, \
930 struct brw_reg src) \
932 const struct gen_device_info *devinfo = p->devinfo; \
933 brw_inst *rnd, *add; \
934 rnd = next_insn(p, BRW_OPCODE_##OP); \
935 brw_set_dest(p, rnd, dest); \
936 brw_set_src0(p, rnd, src); \
938 if (devinfo->gen < 6) { \
939 /* turn on round-increments */ \
940 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
941 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
942 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
982 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
984 const struct gen_device_info
*devinfo
= p
->devinfo
;
986 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
987 * To avoid the problems that causes, we use an <X,2,0> source region to
988 * read each element twice.
990 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
991 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
992 dest
.type
== BRW_REGISTER_TYPE_DF
&&
993 (src0
.type
== BRW_REGISTER_TYPE_F
||
994 src0
.type
== BRW_REGISTER_TYPE_D
||
995 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
996 !has_scalar_region(src0
)) {
997 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
998 src0
.vstride
= src0
.hstride
;
999 src0
.width
= BRW_WIDTH_2
;
1000 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1003 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1007 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1008 struct brw_reg src0
, struct brw_reg src1
)
1011 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1012 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1013 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1014 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1015 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1018 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1019 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1020 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1021 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1022 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1025 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1029 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1030 struct brw_reg src0
, struct brw_reg src1
)
1032 assert(dest
.type
== src0
.type
);
1033 assert(src0
.type
== src1
.type
);
1034 switch (src0
.type
) {
1035 case BRW_REGISTER_TYPE_B
:
1036 case BRW_REGISTER_TYPE_UB
:
1037 case BRW_REGISTER_TYPE_W
:
1038 case BRW_REGISTER_TYPE_UW
:
1039 case BRW_REGISTER_TYPE_D
:
1040 case BRW_REGISTER_TYPE_UD
:
1043 unreachable("Bad type for brw_AVG");
1046 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1050 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1051 struct brw_reg src0
, struct brw_reg src1
)
1054 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1055 src0
.type
== BRW_REGISTER_TYPE_UD
||
1056 src1
.type
== BRW_REGISTER_TYPE_D
||
1057 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1058 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1061 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1062 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1063 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1064 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1065 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1068 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1069 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1070 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1071 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1072 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1075 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1076 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1077 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1078 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1080 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1084 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1085 struct brw_reg src0
, struct brw_reg src1
)
1087 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1088 src0
.width
= BRW_WIDTH_1
;
1089 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1090 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1094 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1095 struct brw_reg src0
, struct brw_reg src1
)
1097 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1098 src0
.width
= BRW_WIDTH_1
;
1099 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1100 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1101 src1
.width
= BRW_WIDTH_8
;
1102 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1103 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1107 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1109 const struct gen_device_info
*devinfo
= p
->devinfo
;
1110 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1111 /* The F32TO16 instruction doesn't support 32-bit destination types in
1112 * Align1 mode, and neither does the Gen8 implementation in terms of a
1113 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1114 * an undocumented feature.
1116 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1117 (!align16
|| devinfo
->gen
>= 8));
1121 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1123 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1124 dst
.type
== BRW_REGISTER_TYPE_W
||
1125 dst
.type
== BRW_REGISTER_TYPE_UW
||
1126 dst
.type
== BRW_REGISTER_TYPE_HF
);
1129 brw_push_insn_state(p
);
1131 if (needs_zero_fill
) {
1132 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1133 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1136 if (devinfo
->gen
>= 8) {
1137 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1139 assert(devinfo
->gen
== 7);
1140 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1143 if (needs_zero_fill
) {
1144 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1145 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1146 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1149 brw_pop_insn_state(p
);
1154 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1156 const struct gen_device_info
*devinfo
= p
->devinfo
;
1157 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1160 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1162 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1164 * Because this instruction does not have a 16-bit floating-point
1165 * type, the source data type must be Word (W). The destination type
1166 * must be F (Float).
1168 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1169 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1171 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1172 src
.type
== BRW_REGISTER_TYPE_UW
||
1173 src
.type
== BRW_REGISTER_TYPE_HF
);
1176 if (devinfo
->gen
>= 8) {
1177 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1179 assert(devinfo
->gen
== 7);
1180 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1185 void brw_NOP(struct brw_codegen
*p
)
1187 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1188 memset(insn
, 0, sizeof(*insn
));
1189 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1196 /***********************************************************************
1197 * Comparisons, if/else/endif
1201 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1202 unsigned predicate_control
)
1204 const struct gen_device_info
*devinfo
= p
->devinfo
;
1205 struct brw_reg ip
= brw_ip_reg();
1206 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1208 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1209 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1210 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1211 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1217 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1219 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1221 p
->if_stack_depth
++;
1222 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1223 p
->if_stack_array_size
*= 2;
1224 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1225 p
->if_stack_array_size
);
1230 pop_if_stack(struct brw_codegen
*p
)
1232 p
->if_stack_depth
--;
1233 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1237 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1239 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1240 p
->loop_stack_array_size
*= 2;
1241 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1242 p
->loop_stack_array_size
);
1243 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1244 p
->loop_stack_array_size
);
1247 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1248 p
->loop_stack_depth
++;
1249 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1253 get_inner_do_insn(struct brw_codegen
*p
)
1255 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1258 /* EU takes the value from the flag register and pushes it onto some
1259 * sort of a stack (presumably merging with any flag value already on
1260 * the stack). Within an if block, the flags at the top of the stack
1261 * control execution on each channel of the unit, eg. on each of the
1262 * 16 pixel values in our wm programs.
1264 * When the matching 'else' instruction is reached (presumably by
1265 * countdown of the instruction count patched in by our ELSE/ENDIF
1266 * functions), the relevant flags are inverted.
1268 * When the matching 'endif' instruction is reached, the flags are
1269 * popped off. If the stack is now empty, normal execution resumes.
1272 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1274 const struct gen_device_info
*devinfo
= p
->devinfo
;
1277 insn
= next_insn(p
, BRW_OPCODE_IF
);
1279 /* Override the defaults for this instruction:
1281 if (devinfo
->gen
< 6) {
1282 brw_set_dest(p
, insn
, brw_ip_reg());
1283 brw_set_src0(p
, insn
, brw_ip_reg());
1284 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1285 } else if (devinfo
->gen
== 6) {
1286 brw_set_dest(p
, insn
, brw_imm_w(0));
1287 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1288 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1289 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1290 } else if (devinfo
->gen
== 7) {
1291 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1292 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1293 brw_set_src1(p
, insn
, brw_imm_w(0));
1294 brw_inst_set_jip(devinfo
, insn
, 0);
1295 brw_inst_set_uip(devinfo
, insn
, 0);
1297 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1298 brw_set_src0(p
, insn
, brw_imm_d(0));
1299 brw_inst_set_jip(devinfo
, insn
, 0);
1300 brw_inst_set_uip(devinfo
, insn
, 0);
1303 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1304 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1305 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1306 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1307 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1308 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1310 push_if_stack(p
, insn
);
1311 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1315 /* This function is only used for gen6-style IF instructions with an
1316 * embedded comparison (conditional modifier). It is not used on gen7.
1319 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1320 struct brw_reg src0
, struct brw_reg src1
)
1322 const struct gen_device_info
*devinfo
= p
->devinfo
;
1325 insn
= next_insn(p
, BRW_OPCODE_IF
);
1327 brw_set_dest(p
, insn
, brw_imm_w(0));
1328 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1329 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1330 brw_set_src0(p
, insn
, src0
);
1331 brw_set_src1(p
, insn
, src1
);
1333 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1334 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1335 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1337 push_if_stack(p
, insn
);
1342 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1345 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1346 brw_inst
*if_inst
, brw_inst
*else_inst
)
1348 const struct gen_device_info
*devinfo
= p
->devinfo
;
1350 /* The next instruction (where the ENDIF would be, if it existed) */
1351 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1353 assert(p
->single_program_flow
);
1354 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1355 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1356 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1358 /* Convert IF to an ADD instruction that moves the instruction pointer
1359 * to the first instruction of the ELSE block. If there is no ELSE
1360 * block, point to where ENDIF would be. Reverse the predicate.
1362 * There's no need to execute an ENDIF since we don't need to do any
1363 * stack operations, and if we're currently executing, we just want to
1364 * continue normally.
1366 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1367 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1369 if (else_inst
!= NULL
) {
1370 /* Convert ELSE to an ADD instruction that points where the ENDIF
1373 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1375 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1376 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1378 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1383 * Patch IF and ELSE instructions with appropriate jump targets.
1386 patch_IF_ELSE(struct brw_codegen
*p
,
1387 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1389 const struct gen_device_info
*devinfo
= p
->devinfo
;
1391 /* We shouldn't be patching IF and ELSE instructions in single program flow
1392 * mode when gen < 6, because in single program flow mode on those
1393 * platforms, we convert flow control instructions to conditional ADDs that
1394 * operate on IP (see brw_ENDIF).
1396 * However, on Gen6, writing to IP doesn't work in single program flow mode
1397 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1398 * not be updated by non-flow control instructions."). And on later
1399 * platforms, there is no significant benefit to converting control flow
1400 * instructions to conditional ADDs. So we do patch IF and ELSE
1401 * instructions in single program flow mode on those platforms.
1403 if (devinfo
->gen
< 6)
1404 assert(!p
->single_program_flow
);
1406 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1407 assert(endif_inst
!= NULL
);
1408 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1410 unsigned br
= brw_jump_scale(devinfo
);
1412 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1413 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1415 if (else_inst
== NULL
) {
1416 /* Patch IF -> ENDIF */
1417 if (devinfo
->gen
< 6) {
1418 /* Turn it into an IFF, which means no mask stack operations for
1419 * all-false and jumping past the ENDIF.
1421 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1422 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1423 br
* (endif_inst
- if_inst
+ 1));
1424 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1425 } else if (devinfo
->gen
== 6) {
1426 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1427 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1429 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1430 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1433 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1435 /* Patch IF -> ELSE */
1436 if (devinfo
->gen
< 6) {
1437 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1438 br
* (else_inst
- if_inst
));
1439 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1440 } else if (devinfo
->gen
== 6) {
1441 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1442 br
* (else_inst
- if_inst
+ 1));
1445 /* Patch ELSE -> ENDIF */
1446 if (devinfo
->gen
< 6) {
1447 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1450 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1451 br
* (endif_inst
- else_inst
+ 1));
1452 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1453 } else if (devinfo
->gen
== 6) {
1454 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1455 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1456 br
* (endif_inst
- else_inst
));
1458 /* The IF instruction's JIP should point just past the ELSE */
1459 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1460 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1461 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1462 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1463 if (devinfo
->gen
>= 8) {
1464 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1465 * should point to ENDIF.
1467 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1474 brw_ELSE(struct brw_codegen
*p
)
1476 const struct gen_device_info
*devinfo
= p
->devinfo
;
1479 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1481 if (devinfo
->gen
< 6) {
1482 brw_set_dest(p
, insn
, brw_ip_reg());
1483 brw_set_src0(p
, insn
, brw_ip_reg());
1484 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1485 } else if (devinfo
->gen
== 6) {
1486 brw_set_dest(p
, insn
, brw_imm_w(0));
1487 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1488 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1489 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1490 } else if (devinfo
->gen
== 7) {
1491 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1492 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1493 brw_set_src1(p
, insn
, brw_imm_w(0));
1494 brw_inst_set_jip(devinfo
, insn
, 0);
1495 brw_inst_set_uip(devinfo
, insn
, 0);
1497 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1498 brw_set_src0(p
, insn
, brw_imm_d(0));
1499 brw_inst_set_jip(devinfo
, insn
, 0);
1500 brw_inst_set_uip(devinfo
, insn
, 0);
1503 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1504 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1505 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1506 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1508 push_if_stack(p
, insn
);
1512 brw_ENDIF(struct brw_codegen
*p
)
1514 const struct gen_device_info
*devinfo
= p
->devinfo
;
1515 brw_inst
*insn
= NULL
;
1516 brw_inst
*else_inst
= NULL
;
1517 brw_inst
*if_inst
= NULL
;
1519 bool emit_endif
= true;
1521 /* In single program flow mode, we can express IF and ELSE instructions
1522 * equivalently as ADD instructions that operate on IP. On platforms prior
1523 * to Gen6, flow control instructions cause an implied thread switch, so
1524 * this is a significant savings.
1526 * However, on Gen6, writing to IP doesn't work in single program flow mode
1527 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1528 * not be updated by non-flow control instructions."). And on later
1529 * platforms, there is no significant benefit to converting control flow
1530 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1533 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1537 * A single next_insn() may change the base address of instruction store
1538 * memory(p->store), so call it first before referencing the instruction
1539 * store pointer from an index
1542 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1544 /* Pop the IF and (optional) ELSE instructions from the stack */
1545 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1546 tmp
= pop_if_stack(p
);
1547 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1549 tmp
= pop_if_stack(p
);
1554 /* ENDIF is useless; don't bother emitting it. */
1555 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1559 if (devinfo
->gen
< 6) {
1560 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1561 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1562 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1563 } else if (devinfo
->gen
== 6) {
1564 brw_set_dest(p
, insn
, brw_imm_w(0));
1565 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1566 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1567 } else if (devinfo
->gen
== 7) {
1568 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1569 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1570 brw_set_src1(p
, insn
, brw_imm_w(0));
1572 brw_set_src0(p
, insn
, brw_imm_d(0));
1575 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1576 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1577 if (devinfo
->gen
< 6)
1578 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1580 /* Also pop item off the stack in the endif instruction: */
1581 if (devinfo
->gen
< 6) {
1582 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1583 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1584 } else if (devinfo
->gen
== 6) {
1585 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1587 brw_inst_set_jip(devinfo
, insn
, 2);
1589 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1593 brw_BREAK(struct brw_codegen
*p
)
1595 const struct gen_device_info
*devinfo
= p
->devinfo
;
1598 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1599 if (devinfo
->gen
>= 8) {
1600 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1601 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1602 } else if (devinfo
->gen
>= 6) {
1603 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1604 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1605 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1607 brw_set_dest(p
, insn
, brw_ip_reg());
1608 brw_set_src0(p
, insn
, brw_ip_reg());
1609 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1610 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1611 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1613 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1614 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1620 brw_CONT(struct brw_codegen
*p
)
1622 const struct gen_device_info
*devinfo
= p
->devinfo
;
1625 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1626 brw_set_dest(p
, insn
, brw_ip_reg());
1627 if (devinfo
->gen
>= 8) {
1628 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1630 brw_set_src0(p
, insn
, brw_ip_reg());
1631 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1634 if (devinfo
->gen
< 6) {
1635 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1636 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1638 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1639 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1644 gen6_HALT(struct brw_codegen
*p
)
1646 const struct gen_device_info
*devinfo
= p
->devinfo
;
1649 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1650 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1651 if (devinfo
->gen
>= 8) {
1652 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1654 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1655 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1658 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1659 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1665 * The DO/WHILE is just an unterminated loop -- break or continue are
1666 * used for control within the loop. We have a few ways they can be
1669 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1670 * jip and no DO instruction.
1672 * For non-uniform control flow pre-gen6, there's a DO instruction to
1673 * push the mask, and a WHILE to jump back, and BREAK to get out and
1676 * For gen6, there's no more mask stack, so no need for DO. WHILE
1677 * just points back to the first instruction of the loop.
1680 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1682 const struct gen_device_info
*devinfo
= p
->devinfo
;
1684 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1685 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1686 return &p
->store
[p
->nr_insn
];
1688 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1690 push_loop_stack(p
, insn
);
1692 /* Override the defaults for this instruction:
1694 brw_set_dest(p
, insn
, brw_null_reg());
1695 brw_set_src0(p
, insn
, brw_null_reg());
1696 brw_set_src1(p
, insn
, brw_null_reg());
1698 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1699 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1700 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1707 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1710 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1711 * nesting, since it can always just point to the end of the block/current loop.
1714 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1716 const struct gen_device_info
*devinfo
= p
->devinfo
;
1717 brw_inst
*do_inst
= get_inner_do_insn(p
);
1719 unsigned br
= brw_jump_scale(devinfo
);
1721 assert(devinfo
->gen
< 6);
1723 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1724 /* If the jump count is != 0, that means that this instruction has already
1725 * been patched because it's part of a loop inside of the one we're
1728 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1729 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1730 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1731 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1732 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1733 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1739 brw_WHILE(struct brw_codegen
*p
)
1741 const struct gen_device_info
*devinfo
= p
->devinfo
;
1742 brw_inst
*insn
, *do_insn
;
1743 unsigned br
= brw_jump_scale(devinfo
);
1745 if (devinfo
->gen
>= 6) {
1746 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1747 do_insn
= get_inner_do_insn(p
);
1749 if (devinfo
->gen
>= 8) {
1750 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1751 brw_set_src0(p
, insn
, brw_imm_d(0));
1752 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1753 } else if (devinfo
->gen
== 7) {
1754 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1755 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1756 brw_set_src1(p
, insn
, brw_imm_w(0));
1757 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1759 brw_set_dest(p
, insn
, brw_imm_w(0));
1760 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1761 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1762 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1765 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1768 if (p
->single_program_flow
) {
1769 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1770 do_insn
= get_inner_do_insn(p
);
1772 brw_set_dest(p
, insn
, brw_ip_reg());
1773 brw_set_src0(p
, insn
, brw_ip_reg());
1774 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1775 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1777 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1778 do_insn
= get_inner_do_insn(p
);
1780 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1782 brw_set_dest(p
, insn
, brw_ip_reg());
1783 brw_set_src0(p
, insn
, brw_ip_reg());
1784 brw_set_src1(p
, insn
, brw_imm_d(0));
1786 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1787 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1788 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1790 brw_patch_break_cont(p
, insn
);
1793 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1795 p
->loop_stack_depth
--;
1802 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1804 const struct gen_device_info
*devinfo
= p
->devinfo
;
1805 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1808 if (devinfo
->gen
>= 5)
1811 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1812 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1814 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1815 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1818 /* To integrate with the above, it makes sense that the comparison
1819 * instruction should populate the flag register. It might be simpler
1820 * just to use the flag reg for most WM tasks?
1822 void brw_CMP(struct brw_codegen
*p
,
1823 struct brw_reg dest
,
1824 unsigned conditional
,
1825 struct brw_reg src0
,
1826 struct brw_reg src1
)
1828 const struct gen_device_info
*devinfo
= p
->devinfo
;
1829 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1831 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1832 brw_set_dest(p
, insn
, dest
);
1833 brw_set_src0(p
, insn
, src0
);
1834 brw_set_src1(p
, insn
, src1
);
1836 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1838 * "Any CMP instruction with a null destination must use a {switch}."
1840 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1841 * mentioned on their work-arounds pages.
1843 if (devinfo
->gen
== 7) {
1844 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1845 dest
.nr
== BRW_ARF_NULL
) {
1846 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1851 /***********************************************************************
1852 * Helpers for the various SEND message types:
1855 /** Extended math function, float[8].
1857 void gen4_math(struct brw_codegen
*p
,
1858 struct brw_reg dest
,
1860 unsigned msg_reg_nr
,
1862 unsigned precision
)
1864 const struct gen_device_info
*devinfo
= p
->devinfo
;
1865 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1867 if (has_scalar_region(src
)) {
1868 data_type
= BRW_MATH_DATA_SCALAR
;
1870 data_type
= BRW_MATH_DATA_VECTOR
;
1873 assert(devinfo
->gen
< 6);
1875 /* Example code doesn't set predicate_control for send
1878 brw_inst_set_pred_control(devinfo
, insn
, 0);
1879 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1881 brw_set_dest(p
, insn
, dest
);
1882 brw_set_src0(p
, insn
, src
);
1883 brw_set_math_message(p
,
1886 src
.type
== BRW_REGISTER_TYPE_D
,
1891 void gen6_math(struct brw_codegen
*p
,
1892 struct brw_reg dest
,
1894 struct brw_reg src0
,
1895 struct brw_reg src1
)
1897 const struct gen_device_info
*devinfo
= p
->devinfo
;
1898 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1900 assert(devinfo
->gen
>= 6);
1902 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1903 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1905 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1906 if (devinfo
->gen
== 6) {
1907 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1908 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1911 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1912 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1913 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1914 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1915 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1916 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1917 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1919 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1920 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1923 /* Source modifiers are ignored for extended math instructions on Gen6. */
1924 if (devinfo
->gen
== 6) {
1925 assert(!src0
.negate
);
1927 assert(!src1
.negate
);
1931 brw_inst_set_math_function(devinfo
, insn
, function
);
1933 brw_set_dest(p
, insn
, dest
);
1934 brw_set_src0(p
, insn
, src0
);
1935 brw_set_src1(p
, insn
, src1
);
1939 * Return the right surface index to access the thread scratch space using
1940 * stateless dataport messages.
1943 brw_scratch_surface_idx(const struct brw_codegen
*p
)
1945 /* The scratch space is thread-local so IA coherency is unnecessary. */
1946 if (p
->devinfo
->gen
>= 8)
1947 return GEN8_BTI_STATELESS_NON_COHERENT
;
1949 return BRW_BTI_STATELESS
;
1953 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1954 * using a constant offset per channel.
1956 * The offset must be aligned to oword size (16 bytes). Used for
1957 * register spilling.
1959 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1964 const struct gen_device_info
*devinfo
= p
->devinfo
;
1965 const unsigned target_cache
=
1966 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
1967 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
1968 BRW_SFID_DATAPORT_WRITE
);
1971 if (devinfo
->gen
>= 6)
1974 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1976 const unsigned mlen
= 1 + num_regs
;
1978 /* Set up the message header. This is g0, with g0.2 filled with
1979 * the offset. We don't want to leave our offset around in g0 or
1980 * it'll screw up texture samples, so set it up inside the message
1984 brw_push_insn_state(p
);
1985 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
1986 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
1987 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
1989 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
1991 /* set message header global offset field (reg 0, element 2) */
1992 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
1994 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
1996 2), BRW_REGISTER_TYPE_UD
),
1997 brw_imm_ud(offset
));
1999 brw_pop_insn_state(p
);
2003 struct brw_reg dest
;
2004 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2005 int send_commit_msg
;
2006 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2007 BRW_REGISTER_TYPE_UW
);
2009 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2010 brw_inst_set_compression(devinfo
, insn
, false);
2012 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2013 src_header
= vec16(src_header
);
2015 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2016 if (devinfo
->gen
< 6)
2017 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2019 /* Until gen6, writes followed by reads from the same location
2020 * are not guaranteed to be ordered unless write_commit is set.
2021 * If set, then a no-op write is issued to the destination
2022 * register to set a dependency, and a read from the destination
2023 * can be used to ensure the ordering.
2025 * For gen6, only writes between different threads need ordering
2026 * protection. Our use of DP writes is all about register
2027 * spilling within a thread.
2029 if (devinfo
->gen
>= 6) {
2030 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2031 send_commit_msg
= 0;
2034 send_commit_msg
= 1;
2037 brw_set_dest(p
, insn
, dest
);
2038 if (devinfo
->gen
>= 6) {
2039 brw_set_src0(p
, insn
, mrf
);
2041 brw_set_src0(p
, insn
, brw_null_reg());
2044 if (devinfo
->gen
>= 6)
2045 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2047 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2049 brw_set_desc(p
, insn
,
2050 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2051 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2052 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2053 msg_type
, 0, /* not a render target */
2060 * Read a block of owords (half a GRF each) from the scratch buffer
2061 * using a constant index per channel.
2063 * Offset must be aligned to oword size (16 bytes). Used for register
2067 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2068 struct brw_reg dest
,
2073 const struct gen_device_info
*devinfo
= p
->devinfo
;
2075 if (devinfo
->gen
>= 6)
2078 if (p
->devinfo
->gen
>= 7) {
2079 /* On gen 7 and above, we no longer have message registers and we can
2080 * send from any register we want. By using the destination register
2081 * for the message, we guarantee that the implied message write won't
2082 * accidentally overwrite anything. This has been a problem because
2083 * the MRF registers and source for the final FB write are both fixed
2086 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2088 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2090 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2092 const unsigned rlen
= num_regs
;
2093 const unsigned target_cache
=
2094 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2095 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2096 BRW_SFID_DATAPORT_READ
);
2099 brw_push_insn_state(p
);
2100 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2101 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2102 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2104 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2106 /* set message header global offset field (reg 0, element 2) */
2107 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2108 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2110 brw_pop_insn_state(p
);
2114 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2116 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2117 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2118 brw_inst_set_compression(devinfo
, insn
, false);
2120 brw_set_dest(p
, insn
, dest
); /* UW? */
2121 if (devinfo
->gen
>= 6) {
2122 brw_set_src0(p
, insn
, mrf
);
2124 brw_set_src0(p
, insn
, brw_null_reg());
2125 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2128 brw_set_desc(p
, insn
,
2129 brw_message_desc(devinfo
, 1, rlen
, true) |
2130 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2131 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2132 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2133 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2138 gen7_block_read_scratch(struct brw_codegen
*p
,
2139 struct brw_reg dest
,
2143 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2144 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2146 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2148 /* The HW requires that the header is present; this is to get the g0.5
2151 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2153 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2154 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2155 * is 32 bytes, which happens to be the size of a register.
2158 assert(offset
< (1 << 12));
2160 gen7_set_dp_scratch_message(p
, insn
,
2161 false, /* scratch read */
2163 false, /* invalidate after read */
2166 1, /* mlen: just g0 */
2167 num_regs
, /* rlen */
2168 true); /* header present */
2172 * Read float[4] vectors from the data port constant cache.
2173 * Location (in buffer) should be a multiple of 16.
2174 * Used for fetching shader constants.
2176 void brw_oword_block_read(struct brw_codegen
*p
,
2177 struct brw_reg dest
,
2180 uint32_t bind_table_index
)
2182 const struct gen_device_info
*devinfo
= p
->devinfo
;
2183 const unsigned target_cache
=
2184 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2185 BRW_SFID_DATAPORT_READ
);
2186 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2188 /* On newer hardware, offset is in units of owords. */
2189 if (devinfo
->gen
>= 6)
2192 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2194 brw_push_insn_state(p
);
2195 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2196 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2197 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2199 brw_push_insn_state(p
);
2200 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2201 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2203 /* set message header global offset field (reg 0, element 2) */
2204 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2206 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2208 2), BRW_REGISTER_TYPE_UD
),
2209 brw_imm_ud(offset
));
2210 brw_pop_insn_state(p
);
2212 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2214 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2216 /* cast dest to a uword[8] vector */
2217 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2219 brw_set_dest(p
, insn
, dest
);
2220 if (devinfo
->gen
>= 6) {
2221 brw_set_src0(p
, insn
, mrf
);
2223 brw_set_src0(p
, insn
, brw_null_reg());
2224 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2227 brw_set_desc(p
, insn
,
2228 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2229 brw_dp_read_desc(devinfo
, bind_table_index
,
2230 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2231 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2232 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2234 brw_pop_insn_state(p
);
2238 brw_fb_WRITE(struct brw_codegen
*p
,
2239 struct brw_reg payload
,
2240 struct brw_reg implied_header
,
2241 unsigned msg_control
,
2242 unsigned binding_table_index
,
2243 unsigned msg_length
,
2244 unsigned response_length
,
2246 bool last_render_target
,
2247 bool header_present
)
2249 const struct gen_device_info
*devinfo
= p
->devinfo
;
2250 const unsigned target_cache
=
2251 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2252 BRW_SFID_DATAPORT_WRITE
);
2255 struct brw_reg dest
, src0
;
2257 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2258 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2260 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2262 if (devinfo
->gen
>= 6) {
2263 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2265 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2267 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2268 brw_inst_set_compression(devinfo
, insn
, false);
2270 if (devinfo
->gen
>= 6) {
2271 /* headerless version, just submit color payload */
2274 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2276 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2277 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2278 src0
= implied_header
;
2280 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2283 brw_set_dest(p
, insn
, dest
);
2284 brw_set_src0(p
, insn
, src0
);
2285 brw_set_desc(p
, insn
,
2286 brw_message_desc(devinfo
, msg_length
, response_length
,
2288 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2289 msg_type
, last_render_target
,
2290 0 /* send_commit_msg */));
2291 brw_inst_set_eot(devinfo
, insn
, eot
);
2297 gen9_fb_READ(struct brw_codegen
*p
,
2299 struct brw_reg payload
,
2300 unsigned binding_table_index
,
2301 unsigned msg_length
,
2302 unsigned response_length
,
2305 const struct gen_device_info
*devinfo
= p
->devinfo
;
2306 assert(devinfo
->gen
>= 9);
2307 const unsigned msg_subtype
=
2308 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2309 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2311 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2312 brw_set_dest(p
, insn
, dst
);
2313 brw_set_src0(p
, insn
, payload
);
2316 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2317 brw_dp_read_desc(devinfo
, binding_table_index
,
2318 per_sample
<< 5 | msg_subtype
,
2319 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2320 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2321 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2327 * Texture sample instruction.
2328 * Note: the msg_type plus msg_length values determine exactly what kind
2329 * of sampling operation is performed. See volume 4, page 161 of docs.
2331 void brw_SAMPLE(struct brw_codegen
*p
,
2332 struct brw_reg dest
,
2333 unsigned msg_reg_nr
,
2334 struct brw_reg src0
,
2335 unsigned binding_table_index
,
2338 unsigned response_length
,
2339 unsigned msg_length
,
2340 unsigned header_present
,
2342 unsigned return_format
)
2344 const struct gen_device_info
*devinfo
= p
->devinfo
;
2347 if (msg_reg_nr
!= -1)
2348 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2350 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2351 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2352 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2354 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2356 * "Instruction compression is not allowed for this instruction (that
2357 * is, send). The hardware behavior is undefined if this instruction is
2358 * set as compressed. However, compress control can be set to "SecHalf"
2359 * to affect the EMask generation."
2361 * No similar wording is found in later PRMs, but there are examples
2362 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2363 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2364 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2366 brw_inst_set_compression(devinfo
, insn
, false);
2368 if (devinfo
->gen
< 6)
2369 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2371 brw_set_dest(p
, insn
, dest
);
2372 brw_set_src0(p
, insn
, src0
);
2373 brw_set_desc(p
, insn
,
2374 brw_message_desc(devinfo
, msg_length
, response_length
,
2376 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2377 msg_type
, simd_mode
, return_format
));
2380 /* Adjust the message header's sampler state pointer to
2381 * select the correct group of 16 samplers.
2383 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2384 struct brw_reg header
,
2385 struct brw_reg sampler_index
)
2387 /* The "Sampler Index" field can only store values between 0 and 15.
2388 * However, we can add an offset to the "Sampler State Pointer"
2389 * field, effectively selecting a different set of 16 samplers.
2391 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2392 * offset, and each sampler state is only 16-bytes, so we can't
2393 * exclusively use the offset - we have to use both.
2396 const struct gen_device_info
*devinfo
= p
->devinfo
;
2398 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2399 const int sampler_state_size
= 16; /* 16 bytes */
2400 uint32_t sampler
= sampler_index
.ud
;
2402 if (sampler
>= 16) {
2403 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2405 get_element_ud(header
, 3),
2406 get_element_ud(brw_vec8_grf(0, 0), 3),
2407 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2410 /* Non-const sampler array indexing case */
2411 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2415 struct brw_reg temp
= get_element_ud(header
, 3);
2417 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2418 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2420 get_element_ud(header
, 3),
2421 get_element_ud(brw_vec8_grf(0, 0), 3),
2426 /* All these variables are pretty confusing - we might be better off
2427 * using bitmasks and macros for this, in the old style. Or perhaps
2428 * just having the caller instantiate the fields in dword3 itself.
2430 void brw_urb_WRITE(struct brw_codegen
*p
,
2431 struct brw_reg dest
,
2432 unsigned msg_reg_nr
,
2433 struct brw_reg src0
,
2434 enum brw_urb_write_flags flags
,
2435 unsigned msg_length
,
2436 unsigned response_length
,
2440 const struct gen_device_info
*devinfo
= p
->devinfo
;
2443 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2445 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2446 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2447 brw_push_insn_state(p
);
2448 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2449 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2450 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2451 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2452 BRW_REGISTER_TYPE_UD
),
2453 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2454 brw_imm_ud(0xff00));
2455 brw_pop_insn_state(p
);
2458 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2460 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2462 brw_set_dest(p
, insn
, dest
);
2463 brw_set_src0(p
, insn
, src0
);
2464 brw_set_src1(p
, insn
, brw_imm_d(0));
2466 if (devinfo
->gen
< 6)
2467 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2469 brw_set_urb_message(p
,
2479 brw_send_indirect_message(struct brw_codegen
*p
,
2482 struct brw_reg payload
,
2483 struct brw_reg desc
,
2487 const struct gen_device_info
*devinfo
= p
->devinfo
;
2488 struct brw_inst
*send
;
2490 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2492 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2494 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2495 send
= next_insn(p
, BRW_OPCODE_SEND
);
2496 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2499 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2501 brw_push_insn_state(p
);
2502 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2503 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2504 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2505 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2507 /* Load the indirect descriptor to an address register using OR so the
2508 * caller can specify additional descriptor bits with the desc_imm
2511 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2513 brw_pop_insn_state(p
);
2515 send
= next_insn(p
, BRW_OPCODE_SEND
);
2516 brw_set_src1(p
, send
, addr
);
2519 brw_set_dest(p
, send
, dst
);
2520 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2521 brw_inst_set_sfid(devinfo
, send
, sfid
);
2522 brw_inst_set_eot(devinfo
, send
, eot
);
2526 brw_send_indirect_split_message(struct brw_codegen
*p
,
2529 struct brw_reg payload0
,
2530 struct brw_reg payload1
,
2531 struct brw_reg desc
,
2533 struct brw_reg ex_desc
,
2534 unsigned ex_desc_imm
,
2537 const struct gen_device_info
*devinfo
= p
->devinfo
;
2538 struct brw_inst
*send
;
2540 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2542 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2544 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2545 desc
.ud
|= desc_imm
;
2547 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2549 brw_push_insn_state(p
);
2550 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2551 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2552 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2553 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2555 /* Load the indirect descriptor to an address register using OR so the
2556 * caller can specify additional descriptor bits with the desc_imm
2559 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2561 brw_pop_insn_state(p
);
2565 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2566 ex_desc
.ud
|= ex_desc_imm
;
2568 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2570 brw_push_insn_state(p
);
2571 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2572 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2573 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2574 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2576 /* Load the indirect extended descriptor to an address register using OR
2577 * so the caller can specify additional descriptor bits with the
2578 * desc_imm immediate.
2580 * Even though the instruction dispatcher always pulls the SFID and EOT
2581 * fields from the instruction itself, actual external unit which
2582 * processes the message gets the SFID and EOT from the extended
2583 * descriptor which comes from the address register. If we don't OR
2584 * those two bits in, the external unit may get confused and hang.
2586 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(ex_desc_imm
| sfid
| eot
<< 5));
2588 brw_pop_insn_state(p
);
2592 send
= next_insn(p
, BRW_OPCODE_SENDS
);
2593 brw_set_dest(p
, send
, dst
);
2594 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2595 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2597 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2598 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2599 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2601 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2602 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2603 assert(desc
.subnr
== 0);
2604 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2607 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2608 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2609 brw_inst_set_send_ex_desc(devinfo
, send
, ex_desc
.ud
);
2611 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2612 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2613 assert((ex_desc
.subnr
& 0x3) == 0);
2614 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2615 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2618 brw_inst_set_sfid(devinfo
, send
, sfid
);
2619 brw_inst_set_eot(devinfo
, send
, eot
);
2623 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2626 struct brw_reg payload
,
2627 struct brw_reg surface
,
2630 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2631 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2633 brw_push_insn_state(p
);
2634 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2635 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2636 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2637 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2639 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2640 * some surface array is accessed out of bounds.
2643 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2644 BRW_GET_SWZ(surface
.swizzle
, 0)),
2647 brw_pop_insn_state(p
);
2652 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2656 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2657 brw_inst
*insn
, int while_offset
, int start_offset
)
2659 int scale
= 16 / brw_jump_scale(devinfo
);
2660 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2661 : brw_inst_jip(devinfo
, insn
);
2663 return while_offset
+ jip
* scale
<= start_offset
;
2668 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2671 void *store
= p
->store
;
2672 const struct gen_device_info
*devinfo
= p
->devinfo
;
2676 for (offset
= next_offset(devinfo
, store
, start_offset
);
2677 offset
< p
->next_insn_offset
;
2678 offset
= next_offset(devinfo
, store
, offset
)) {
2679 brw_inst
*insn
= store
+ offset
;
2681 switch (brw_inst_opcode(devinfo
, insn
)) {
2685 case BRW_OPCODE_ENDIF
:
2690 case BRW_OPCODE_WHILE
:
2691 /* If the while doesn't jump before our instruction, it's the end
2692 * of a sibling do...while loop. Ignore it.
2694 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2697 case BRW_OPCODE_ELSE
:
2698 case BRW_OPCODE_HALT
:
2707 /* There is no DO instruction on gen6, so to find the end of the loop
2708 * we have to see if the loop is jumping back before our start
2712 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2714 const struct gen_device_info
*devinfo
= p
->devinfo
;
2716 void *store
= p
->store
;
2718 assert(devinfo
->gen
>= 6);
2720 /* Always start after the instruction (such as a WHILE) we're trying to fix
2723 for (offset
= next_offset(devinfo
, store
, start_offset
);
2724 offset
< p
->next_insn_offset
;
2725 offset
= next_offset(devinfo
, store
, offset
)) {
2726 brw_inst
*insn
= store
+ offset
;
2728 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2729 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2733 assert(!"not reached");
2734 return start_offset
;
2737 /* After program generation, go back and update the UIP and JIP of
2738 * BREAK, CONT, and HALT instructions to their correct locations.
2741 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2743 const struct gen_device_info
*devinfo
= p
->devinfo
;
2745 int br
= brw_jump_scale(devinfo
);
2746 int scale
= 16 / br
;
2747 void *store
= p
->store
;
2749 if (devinfo
->gen
< 6)
2752 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2753 brw_inst
*insn
= store
+ offset
;
2754 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2756 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2757 switch (brw_inst_opcode(devinfo
, insn
)) {
2758 case BRW_OPCODE_BREAK
:
2759 assert(block_end_offset
!= 0);
2760 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2761 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2762 brw_inst_set_uip(devinfo
, insn
,
2763 (brw_find_loop_end(p
, offset
) - offset
+
2764 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2766 case BRW_OPCODE_CONTINUE
:
2767 assert(block_end_offset
!= 0);
2768 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2769 brw_inst_set_uip(devinfo
, insn
,
2770 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2772 assert(brw_inst_uip(devinfo
, insn
) != 0);
2773 assert(brw_inst_jip(devinfo
, insn
) != 0);
2776 case BRW_OPCODE_ENDIF
: {
2777 int32_t jump
= (block_end_offset
== 0) ?
2778 1 * br
: (block_end_offset
- offset
) / scale
;
2779 if (devinfo
->gen
>= 7)
2780 brw_inst_set_jip(devinfo
, insn
, jump
);
2782 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2786 case BRW_OPCODE_HALT
:
2787 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2789 * "In case of the halt instruction not inside any conditional
2790 * code block, the value of <JIP> and <UIP> should be the
2791 * same. In case of the halt instruction inside conditional code
2792 * block, the <UIP> should be the end of the program, and the
2793 * <JIP> should be end of the most inner conditional code block."
2795 * The uip will have already been set by whoever set up the
2798 if (block_end_offset
== 0) {
2799 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2801 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2803 assert(brw_inst_uip(devinfo
, insn
) != 0);
2804 assert(brw_inst_jip(devinfo
, insn
) != 0);
2810 void brw_ff_sync(struct brw_codegen
*p
,
2811 struct brw_reg dest
,
2812 unsigned msg_reg_nr
,
2813 struct brw_reg src0
,
2815 unsigned response_length
,
2818 const struct gen_device_info
*devinfo
= p
->devinfo
;
2821 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2823 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2824 brw_set_dest(p
, insn
, dest
);
2825 brw_set_src0(p
, insn
, src0
);
2826 brw_set_src1(p
, insn
, brw_imm_d(0));
2828 if (devinfo
->gen
< 6)
2829 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2831 brw_set_ff_sync_message(p
,
2839 * Emit the SEND instruction necessary to generate stream output data on Gen6
2840 * (for transform feedback).
2842 * If send_commit_msg is true, this is the last piece of stream output data
2843 * from this thread, so send the data as a committed write. According to the
2844 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2846 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2847 * writes are complete by sending the final write as a committed write."
2850 brw_svb_write(struct brw_codegen
*p
,
2851 struct brw_reg dest
,
2852 unsigned msg_reg_nr
,
2853 struct brw_reg src0
,
2854 unsigned binding_table_index
,
2855 bool send_commit_msg
)
2857 const struct gen_device_info
*devinfo
= p
->devinfo
;
2858 const unsigned target_cache
=
2859 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2860 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2861 BRW_SFID_DATAPORT_WRITE
);
2864 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2866 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2867 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2868 brw_set_dest(p
, insn
, dest
);
2869 brw_set_src0(p
, insn
, src0
);
2870 brw_set_desc(p
, insn
,
2871 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
2872 brw_dp_write_desc(devinfo
, binding_table_index
,
2873 0, /* msg_control: ignored */
2874 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2875 0, /* last_render_target: ignored */
2876 send_commit_msg
)); /* send_commit_msg */
2880 brw_surface_payload_size(struct brw_codegen
*p
,
2881 unsigned num_channels
,
2882 unsigned exec_size
/**< 0 for SIMD4x2 */)
2885 return 1; /* SIMD4x2 */
2886 else if (exec_size
<= 8)
2887 return num_channels
;
2889 return 2 * num_channels
;
2893 brw_untyped_atomic(struct brw_codegen
*p
,
2895 struct brw_reg payload
,
2896 struct brw_reg surface
,
2898 unsigned msg_length
,
2899 bool response_expected
,
2900 bool header_present
)
2902 const struct gen_device_info
*devinfo
= p
->devinfo
;
2903 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2904 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2905 GEN7_SFID_DATAPORT_DATA_CACHE
);
2906 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2907 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
2908 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2909 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2910 has_simd4x2
? 0 : 8;
2911 const unsigned response_length
=
2912 brw_surface_payload_size(p
, response_expected
, exec_size
);
2913 const unsigned desc
=
2914 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
2915 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
2917 /* Mask out unused components -- This is especially important in Align16
2918 * mode on generations that don't have native support for SIMD4x2 atomics,
2919 * because unused but enabled components will cause the dataport to perform
2920 * additional atomic operations on the addresses that happen to be in the
2921 * uninitialized Y, Z and W coordinates of the payload.
2923 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2925 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
2926 payload
, surface
, desc
);
2930 brw_untyped_surface_read(struct brw_codegen
*p
,
2932 struct brw_reg payload
,
2933 struct brw_reg surface
,
2934 unsigned msg_length
,
2935 unsigned num_channels
)
2937 const struct gen_device_info
*devinfo
= p
->devinfo
;
2938 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2939 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2940 GEN7_SFID_DATAPORT_DATA_CACHE
);
2941 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2942 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
2943 const unsigned response_length
=
2944 brw_surface_payload_size(p
, num_channels
, exec_size
);
2945 const unsigned desc
=
2946 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
2947 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
2949 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
2953 brw_untyped_surface_write(struct brw_codegen
*p
,
2954 struct brw_reg payload
,
2955 struct brw_reg surface
,
2956 unsigned msg_length
,
2957 unsigned num_channels
,
2958 bool header_present
)
2960 const struct gen_device_info
*devinfo
= p
->devinfo
;
2961 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2962 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2963 GEN7_SFID_DATAPORT_DATA_CACHE
);
2964 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2965 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
2966 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2967 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2968 has_simd4x2
? 0 : 8;
2969 const unsigned desc
=
2970 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
2971 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
2972 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2973 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
2975 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2976 payload
, surface
, desc
);
2980 brw_typed_atomic(struct brw_codegen
*p
,
2982 struct brw_reg payload
,
2983 struct brw_reg surface
,
2985 unsigned msg_length
,
2986 bool response_expected
,
2987 bool header_present
) {
2988 const struct gen_device_info
*devinfo
= p
->devinfo
;
2989 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2990 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2991 GEN6_SFID_DATAPORT_RENDER_CACHE
);
2992 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2993 /* SIMD4x2 typed atomic instructions only exist on HSW+ */
2994 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2995 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2996 has_simd4x2
? 0 : 8;
2997 /* Typed atomics don't support SIMD16 */
2998 assert(exec_size
<= 8);
2999 const unsigned response_length
=
3000 brw_surface_payload_size(p
, response_expected
, exec_size
);
3001 const unsigned desc
=
3002 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3003 brw_dp_typed_atomic_desc(devinfo
, exec_size
, brw_get_default_group(p
),
3004 atomic_op
, response_expected
);
3005 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3006 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3008 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
3009 payload
, surface
, desc
);
3013 brw_typed_surface_read(struct brw_codegen
*p
,
3015 struct brw_reg payload
,
3016 struct brw_reg surface
,
3017 unsigned msg_length
,
3018 unsigned num_channels
,
3019 bool header_present
)
3021 const struct gen_device_info
*devinfo
= p
->devinfo
;
3022 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3023 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3024 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3025 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3026 /* SIMD4x2 typed read instructions only exist on HSW+ */
3027 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3028 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3029 has_simd4x2
? 0 : 8;
3030 const unsigned response_length
=
3031 brw_surface_payload_size(p
, num_channels
, exec_size
);
3032 const unsigned desc
=
3033 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3034 brw_dp_typed_surface_rw_desc(devinfo
, exec_size
, brw_get_default_group(p
),
3035 num_channels
, false);
3037 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
3041 brw_typed_surface_write(struct brw_codegen
*p
,
3042 struct brw_reg payload
,
3043 struct brw_reg surface
,
3044 unsigned msg_length
,
3045 unsigned num_channels
,
3046 bool header_present
)
3048 const struct gen_device_info
*devinfo
= p
->devinfo
;
3049 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3050 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3051 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3052 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3053 /* SIMD4x2 typed read instructions only exist on HSW+ */
3054 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
3055 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
3056 has_simd4x2
? 0 : 8;
3057 const unsigned desc
=
3058 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3059 brw_dp_typed_surface_rw_desc(devinfo
, exec_size
, brw_get_default_group(p
),
3060 num_channels
, true);
3061 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3062 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
3064 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3065 payload
, surface
, desc
);
3069 brw_set_memory_fence_message(struct brw_codegen
*p
,
3070 struct brw_inst
*insn
,
3071 enum brw_message_target sfid
,
3074 const struct gen_device_info
*devinfo
= p
->devinfo
;
3076 brw_set_desc(p
, insn
, brw_message_desc(
3077 devinfo
, 1, (commit_enable
? 1 : 0), true));
3079 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3082 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3083 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3085 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3086 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3089 unreachable("Not reached");
3093 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3097 brw_memory_fence(struct brw_codegen
*p
,
3099 enum opcode send_op
)
3101 const struct gen_device_info
*devinfo
= p
->devinfo
;
3102 const bool commit_enable
=
3103 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3104 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3105 struct brw_inst
*insn
;
3107 brw_push_insn_state(p
);
3108 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3109 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3112 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3113 * message doesn't write anything back.
3115 insn
= next_insn(p
, send_op
);
3116 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3117 brw_set_dest(p
, insn
, dst
);
3118 brw_set_src0(p
, insn
, dst
);
3119 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3122 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3123 /* IVB does typed surface access through the render cache, so we need to
3124 * flush it too. Use a different register so both flushes can be
3125 * pipelined by the hardware.
3127 insn
= next_insn(p
, send_op
);
3128 brw_set_dest(p
, insn
, offset(dst
, 1));
3129 brw_set_src0(p
, insn
, offset(dst
, 1));
3130 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3133 /* Now write the response of the second message into the response of the
3134 * first to trigger a pipeline stall -- This way future render and data
3135 * cache messages will be properly ordered with respect to past data and
3136 * render cache messages.
3138 brw_MOV(p
, dst
, offset(dst
, 1));
3141 brw_pop_insn_state(p
);
3145 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3146 struct brw_reg dest
,
3150 struct brw_reg data
,
3151 unsigned msg_length
,
3152 unsigned response_length
)
3154 const struct gen_device_info
*devinfo
= p
->devinfo
;
3155 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3156 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3157 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3158 const unsigned desc
=
3159 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3160 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3163 /* brw_send_indirect_message will automatically use a direct send message
3164 * if data is actually immediate.
3166 brw_send_indirect_message(p
,
3167 GEN7_SFID_PIXEL_INTERPOLATOR
,
3176 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3177 struct brw_reg mask
)
3179 const struct gen_device_info
*devinfo
= p
->devinfo
;
3180 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3181 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3184 assert(devinfo
->gen
>= 7);
3185 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3187 brw_push_insn_state(p
);
3189 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3190 * unnecessary bits in the instruction words, get the information we need
3191 * and reset the default flag register. This allows more instructions to be
3194 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3195 brw_set_default_flag_reg(p
, 0, 0);
3197 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3198 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3200 if (devinfo
->gen
>= 8) {
3201 /* Getting the first active channel index is easy on Gen8: Just find
3202 * the first bit set in the execution mask. The register exists on
3203 * HSW already but it reads back as all ones when the current
3204 * instruction has execution masking disabled, so it's kind of
3207 struct brw_reg exec_mask
=
3208 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3210 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3211 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3212 /* Unfortunately, ce0 does not take into account the thread
3213 * dispatch mask, which may be a problem in cases where it's not
3214 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3215 * some n). Combine ce0 with the given dispatch (or vector) mask
3216 * to mask off those channels which were never dispatched by the
3219 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3220 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3221 exec_mask
= vec1(dst
);
3224 /* Quarter control has the effect of magically shifting the value of
3225 * ce0 so you'll get the first active channel relative to the
3226 * specified quarter control as result.
3228 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3230 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3232 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3233 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3235 /* Run enough instructions returning zero with execution masking and
3236 * a conditional modifier enabled in order to get the full execution
3237 * mask in f1.0. We could use a single 32-wide move here if it
3238 * weren't because of the hardware bug that causes channel enables to
3239 * be applied incorrectly to the second half of 32-wide instructions
3242 const unsigned lower_size
= MIN2(16, exec_size
);
3243 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3244 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3246 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3247 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3248 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3249 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3250 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3251 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3254 /* Find the first bit set in the exec_size-wide portion of the flag
3255 * register that was updated by the last sequence of MOV
3258 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3259 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3260 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3263 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3265 if (devinfo
->gen
>= 8 &&
3266 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3267 /* In SIMD4x2 mode the first active channel index is just the
3268 * negation of the first bit of the mask register. Note that ce0
3269 * doesn't take into account the dispatch mask, so the Gen7 path
3270 * should be used instead unless you have the guarantee that the
3271 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3274 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3275 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3279 /* Overwrite the destination without and with execution masking to
3280 * find out which of the channels is active.
3282 brw_push_insn_state(p
);
3283 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3284 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3287 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3289 brw_pop_insn_state(p
);
3290 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3294 brw_pop_insn_state(p
);
3298 brw_broadcast(struct brw_codegen
*p
,
3303 const struct gen_device_info
*devinfo
= p
->devinfo
;
3304 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3307 brw_push_insn_state(p
);
3308 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3309 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3311 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3312 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3313 assert(!src
.abs
&& !src
.negate
);
3314 assert(src
.type
== dst
.type
);
3316 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3317 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3318 /* Trivial, the source is already uniform or the index is a constant.
3319 * We will typically not get here if the optimizer is doing its job, but
3320 * asserting would be mean.
3322 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3324 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3325 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3327 /* From the Haswell PRM section "Register Region Restrictions":
3329 * "The lower bits of the AddressImmediate must not overflow to
3330 * change the register address. The lower 5 bits of Address
3331 * Immediate when added to lower 5 bits of address register gives
3332 * the sub-register offset. The upper bits of Address Immediate
3333 * when added to upper bits of address register gives the register
3334 * address. Any overflow from sub-register offset is dropped."
3336 * Fortunately, for broadcast, we never have a sub-register offset so
3337 * this isn't an issue.
3339 assert(src
.subnr
== 0);
3342 const struct brw_reg addr
=
3343 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3344 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3345 /* Limit in bytes of the signed indirect addressing immediate. */
3346 const unsigned limit
= 512;
3348 brw_push_insn_state(p
);
3349 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3350 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3352 /* Take into account the component size and horizontal stride. */
3353 assert(src
.vstride
== src
.hstride
+ src
.width
);
3354 brw_SHL(p
, addr
, vec1(idx
),
3355 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3358 /* We can only address up to limit bytes using the indirect
3359 * addressing immediate, account for the difference if the source
3360 * register is above this limit.
3362 if (offset
>= limit
) {
3363 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3364 offset
= offset
% limit
;
3367 brw_pop_insn_state(p
);
3369 /* Use indirect addressing to fetch the specified component. */
3370 if (type_sz(src
.type
) > 4 &&
3371 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3372 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3374 * "When source or destination datatype is 64b or operation is
3375 * integer DWord multiply, indirect addressing must not be
3378 * To work around both of this issue, we do two integer MOVs
3379 * insead of one 64-bit MOV. Because no double value should ever
3380 * cross a register boundary, it's safe to use the immediate
3381 * offset in the indirect here to handle adding 4 bytes to the
3382 * offset and avoid the extra ADD to the register file.
3384 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3385 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3386 BRW_REGISTER_TYPE_D
));
3387 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3388 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3389 BRW_REGISTER_TYPE_D
));
3392 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3395 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3396 * to all bits of a flag register,
3400 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3401 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3402 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3403 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3405 /* and use predicated SEL to pick the right channel. */
3406 inst
= brw_SEL(p
, dst
,
3407 stride(suboffset(src
, 4), 4, 4, 1),
3408 stride(src
, 4, 4, 1));
3409 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3410 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3414 brw_pop_insn_state(p
);
3418 * This instruction is generated as a single-channel align1 instruction by
3419 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3421 * We can't use the typed atomic op in the FS because that has the execution
3422 * mask ANDed with the pixel mask, but we just want to write the one dword for
3425 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3426 * one u32. So we use the same untyped atomic write message as the pixel
3429 * The untyped atomic operation requires a BUFFER surface type with RAW
3430 * format, and is only accessible through the legacy DATA_CACHE dataport
3433 void brw_shader_time_add(struct brw_codegen
*p
,
3434 struct brw_reg payload
,
3435 uint32_t surf_index
)
3437 const struct gen_device_info
*devinfo
= p
->devinfo
;
3438 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3439 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3440 GEN7_SFID_DATAPORT_DATA_CACHE
);
3441 assert(devinfo
->gen
>= 7);
3443 brw_push_insn_state(p
);
3444 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3445 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3446 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3447 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3449 /* We use brw_vec1_reg and unmasked because we want to increment the given
3452 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3454 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3456 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3457 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3460 brw_inst_set_sfid(devinfo
, send
, sfid
);
3461 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3463 brw_pop_insn_state(p
);
3468 * Emit the SEND message for a barrier
3471 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3473 const struct gen_device_info
*devinfo
= p
->devinfo
;
3474 struct brw_inst
*inst
;
3476 assert(devinfo
->gen
>= 7);
3478 brw_push_insn_state(p
);
3479 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3480 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3481 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3482 brw_set_src0(p
, inst
, src
);
3483 brw_set_src1(p
, inst
, brw_null_reg());
3484 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3486 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3487 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3488 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3489 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3491 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3492 brw_pop_insn_state(p
);
3497 * Emit the wait instruction for a barrier
3500 brw_WAIT(struct brw_codegen
*p
)
3502 const struct gen_device_info
*devinfo
= p
->devinfo
;
3503 struct brw_inst
*insn
;
3505 struct brw_reg src
= brw_notification_reg();
3507 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3508 brw_set_dest(p
, insn
, src
);
3509 brw_set_src0(p
, insn
, src
);
3510 brw_set_src1(p
, insn
, brw_null_reg());
3512 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3513 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3517 * Changes the floating point rounding mode updating the control register
3518 * field defined at cr0.0[5-6] bits. This function supports the changes to
3519 * RTNE (00), RU (01), RD (10) and RTZ (11) rounding using bitwise operations.
3520 * Only RTNE and RTZ rounding are enabled at nir.
3523 brw_rounding_mode(struct brw_codegen
*p
,
3524 enum brw_rnd_mode mode
)
3526 const unsigned bits
= mode
<< BRW_CR0_RND_MODE_SHIFT
;
3528 if (bits
!= BRW_CR0_RND_MODE_MASK
) {
3529 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3530 brw_imm_ud(~BRW_CR0_RND_MODE_MASK
));
3531 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3533 /* From the Skylake PRM, Volume 7, page 760:
3534 * "Implementation Restriction on Register Access: When the control
3535 * register is used as an explicit source and/or destination, hardware
3536 * does not ensure execution pipeline coherency. Software must set the
3537 * thread control field to ‘switch’ for an instruction that uses
3538 * control register as an explicit operand."
3540 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3544 brw_inst
*inst
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3546 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3547 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);