2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
90 const struct gen_device_info
*devinfo
= p
->devinfo
;
92 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
93 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
94 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
95 assert(dest
.nr
< 128);
97 gen7_convert_mrf_to_grf(p
, &dest
);
99 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
100 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
101 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
102 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
103 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
104 assert(dest
.subnr
% 16 == 0);
105 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
106 dest
.vstride
== dest
.width
+ 1);
107 assert(!dest
.negate
&& !dest
.abs
);
108 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
109 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
110 brw_inst_set_send_dst_reg_file(devinfo
, inst
, dest
.file
);
112 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
113 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
115 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
116 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
118 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
119 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
120 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
121 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
122 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
124 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
125 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
126 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
127 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
128 assert(dest
.writemask
!= 0);
130 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
131 * Although Dst.HorzStride is a don't care for Align16, HW needs
132 * this to be programmed as "01".
134 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
137 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
139 /* These are different sizes in align1 vs align16:
141 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
142 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
143 dest
.indirect_offset
);
144 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
145 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
146 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
148 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
149 dest
.indirect_offset
);
150 /* even ignored in da16, still need to set as '01' */
151 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
156 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
157 * or 16 (SIMD16), as that's normally correct. However, when dealing with
158 * small registers, it can be useful for us to automatically reduce it to
159 * match the register size.
161 if (p
->automatic_exec_sizes
) {
163 * In platforms that support fp64 we can emit instructions with a width
164 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
165 * these cases we need to make sure that these instructions have their
166 * exec sizes set properly when they are emitted and we can't rely on
167 * this code to fix it.
170 if (devinfo
->gen
>= 6)
171 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
173 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
176 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
181 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
183 const struct gen_device_info
*devinfo
= p
->devinfo
;
185 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
186 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
187 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
188 assert(reg
.nr
< 128);
190 gen7_convert_mrf_to_grf(p
, ®
);
192 if (devinfo
->gen
>= 6 &&
193 (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
194 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
||
195 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
196 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
)) {
197 /* Any source modifiers or regions will be ignored, since this just
198 * identifies the MRF/GRF to start reading the message contents from.
199 * Check for some likely failures.
203 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
206 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
207 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
208 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
);
209 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
210 assert(reg
.subnr
% 16 == 0);
211 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
212 reg
.vstride
== reg
.width
+ 1);
213 assert(!reg
.negate
&& !reg
.abs
);
214 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
215 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
217 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
218 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
219 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
220 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
222 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
223 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
224 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
225 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
226 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
227 reg
.type
== BRW_REGISTER_TYPE_Q
)
228 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
230 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
232 if (type_sz(reg
.type
) < 8) {
233 brw_inst_set_src1_reg_file(devinfo
, inst
,
234 BRW_ARCHITECTURE_REGISTER_FILE
);
235 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
236 brw_inst_src0_reg_hw_type(devinfo
, inst
));
239 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
240 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
241 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
242 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
244 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
247 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
249 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
250 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
252 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
256 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
257 if (reg
.width
== BRW_WIDTH_1
&&
258 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
259 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
260 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
261 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
263 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
264 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
265 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
268 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
269 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
270 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
271 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
272 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
273 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
274 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
275 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
277 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
278 /* This is an oddity of the fact we're using the same
279 * descriptions for registers in align_16 as align_1:
281 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
282 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
283 reg
.type
== BRW_REGISTER_TYPE_DF
&&
284 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
287 * "For Align16 access mode, only encodings of 0000 and 0011
288 * are allowed. Other codes are reserved."
290 * Presumably the DevSNB behavior applies to IVB as well.
292 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
294 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
303 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
305 const struct gen_device_info
*devinfo
= p
->devinfo
;
307 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
308 assert(reg
.nr
< 128);
310 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDS
||
311 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDSC
) {
312 assert(reg
.file
== BRW_GENERAL_REGISTER_FILE
||
313 reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
314 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
315 assert(reg
.subnr
== 0);
316 assert(reg
.hstride
== BRW_HORIZONTAL_STRIDE_1
&&
317 reg
.vstride
== reg
.width
+ 1);
318 assert(!reg
.negate
&& !reg
.abs
);
319 brw_inst_set_send_src1_reg_nr(devinfo
, inst
, reg
.nr
);
320 brw_inst_set_send_src1_reg_file(devinfo
, inst
, reg
.file
);
322 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
324 * "Accumulator registers may be accessed explicitly as src0
327 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
328 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
330 gen7_convert_mrf_to_grf(p
, ®
);
331 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
333 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
334 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
335 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
337 /* Only src1 can be immediate in two-argument instructions.
339 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
341 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
342 /* two-argument instructions can only use 32-bit immediates */
343 assert(type_sz(reg
.type
) < 8);
344 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
346 /* This is a hardware restriction, which may or may not be lifted
349 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
350 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
352 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
353 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
354 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
356 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
359 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
360 if (reg
.width
== BRW_WIDTH_1
&&
361 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
362 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
363 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
364 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
366 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
367 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
368 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
371 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
372 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
373 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
374 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
375 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
376 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
377 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
378 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
380 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
381 /* This is an oddity of the fact we're using the same
382 * descriptions for registers in align_16 as align_1:
384 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
385 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
386 reg
.type
== BRW_REGISTER_TYPE_DF
&&
387 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
390 * "For Align16 access mode, only encodings of 0000 and 0011
391 * are allowed. Other codes are reserved."
393 * Presumably the DevSNB behavior applies to IVB as well.
395 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
397 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
405 * Specify the descriptor and extended descriptor immediate for a SEND(C)
406 * message instruction.
409 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
410 unsigned desc
, unsigned ex_desc
)
412 const struct gen_device_info
*devinfo
= p
->devinfo
;
413 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
414 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
415 brw_inst_set_src1_file_type(devinfo
, inst
,
416 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
417 brw_inst_set_send_desc(devinfo
, inst
, desc
);
418 if (devinfo
->gen
>= 9)
419 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
422 static void brw_set_math_message( struct brw_codegen
*p
,
425 unsigned integer_type
,
429 const struct gen_device_info
*devinfo
= p
->devinfo
;
431 unsigned response_length
;
433 /* Infer message length from the function */
435 case BRW_MATH_FUNCTION_POW
:
436 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
437 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
438 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
446 /* Infer response length from the function */
448 case BRW_MATH_FUNCTION_SINCOS
:
449 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
457 brw_set_desc(p
, inst
, brw_message_desc(
458 devinfo
, msg_length
, response_length
, false));
460 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
461 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
462 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
463 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
464 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
465 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
466 brw_inst_set_saturate(devinfo
, inst
, 0);
470 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
473 unsigned response_length
,
476 const struct gen_device_info
*devinfo
= p
->devinfo
;
478 brw_set_desc(p
, insn
, brw_message_desc(
479 devinfo
, 1, response_length
, true));
481 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
482 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
483 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
484 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
485 /* The following fields are not used by FF_SYNC: */
486 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
487 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
488 brw_inst_set_urb_used(devinfo
, insn
, 0);
489 brw_inst_set_urb_complete(devinfo
, insn
, 0);
492 static void brw_set_urb_message( struct brw_codegen
*p
,
494 enum brw_urb_write_flags flags
,
496 unsigned response_length
,
498 unsigned swizzle_control
)
500 const struct gen_device_info
*devinfo
= p
->devinfo
;
502 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
503 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
504 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
506 brw_set_desc(p
, insn
, brw_message_desc(
507 devinfo
, msg_length
, response_length
, true));
509 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
510 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
512 if (flags
& BRW_URB_WRITE_OWORD
) {
513 assert(msg_length
== 2); /* header + one OWORD of data */
514 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
516 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
519 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
520 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
522 if (devinfo
->gen
< 8) {
523 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
526 if (devinfo
->gen
< 7) {
527 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
528 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
530 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
531 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
536 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
540 bool invalidate_after_read
,
542 unsigned addr_offset
,
547 const struct gen_device_info
*devinfo
= p
->devinfo
;
548 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
549 (devinfo
->gen
>= 8 && num_regs
== 8));
550 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
553 brw_set_desc(p
, inst
, brw_message_desc(
554 devinfo
, mlen
, rlen
, header_present
));
556 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
557 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
558 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
559 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
560 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
561 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
562 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
566 brw_inst_set_state(const struct gen_device_info
*devinfo
,
568 const struct brw_insn_state
*state
)
570 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
571 brw_inst_set_group(devinfo
, insn
, state
->group
);
572 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
573 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
574 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
575 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
576 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
577 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
579 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
580 state
->access_mode
== BRW_ALIGN_16
) {
581 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
582 if (devinfo
->gen
>= 7)
583 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
585 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
586 if (devinfo
->gen
>= 7)
587 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
590 if (devinfo
->gen
>= 6)
591 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
594 #define next_insn brw_next_insn
596 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
598 const struct gen_device_info
*devinfo
= p
->devinfo
;
601 if (p
->nr_insn
+ 1 > p
->store_size
) {
603 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
606 p
->next_insn_offset
+= 16;
607 insn
= &p
->store
[p
->nr_insn
++];
609 memset(insn
, 0, sizeof(*insn
));
610 brw_inst_set_opcode(devinfo
, insn
, opcode
);
612 /* Apply the default instruction state */
613 brw_inst_set_state(devinfo
, insn
, p
->current
);
619 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
620 struct brw_reg dest
, struct brw_reg src
)
622 brw_inst
*insn
= next_insn(p
, opcode
);
623 brw_set_dest(p
, insn
, dest
);
624 brw_set_src0(p
, insn
, src
);
629 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
630 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
632 /* 64-bit immediates are only supported on 1-src instructions */
633 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
634 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
636 brw_inst
*insn
= next_insn(p
, opcode
);
637 brw_set_dest(p
, insn
, dest
);
638 brw_set_src0(p
, insn
, src0
);
639 brw_set_src1(p
, insn
, src1
);
644 get_3src_subreg_nr(struct brw_reg reg
)
646 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
647 * use 32-bit units (components 0..7). Since they only support F/D/UD
648 * types, this doesn't lose any flexibility, but uses fewer bits.
650 return reg
.subnr
/ 4;
653 static enum gen10_align1_3src_vertical_stride
654 to_3src_align1_vstride(enum brw_vertical_stride vstride
)
657 case BRW_VERTICAL_STRIDE_0
:
658 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
659 case BRW_VERTICAL_STRIDE_2
:
660 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
661 case BRW_VERTICAL_STRIDE_4
:
662 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
663 case BRW_VERTICAL_STRIDE_8
:
664 case BRW_VERTICAL_STRIDE_16
:
665 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
667 unreachable("invalid vstride");
672 static enum gen10_align1_3src_src_horizontal_stride
673 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
676 case BRW_HORIZONTAL_STRIDE_0
:
677 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
678 case BRW_HORIZONTAL_STRIDE_1
:
679 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
680 case BRW_HORIZONTAL_STRIDE_2
:
681 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
682 case BRW_HORIZONTAL_STRIDE_4
:
683 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
685 unreachable("invalid hstride");
690 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
691 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
693 const struct gen_device_info
*devinfo
= p
->devinfo
;
694 brw_inst
*inst
= next_insn(p
, opcode
);
696 gen7_convert_mrf_to_grf(p
, &dest
);
698 assert(dest
.nr
< 128);
699 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
700 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| src1
.nr
< 128);
701 assert(src2
.file
!= BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
702 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
703 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
704 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
705 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
707 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
708 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
709 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
711 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
712 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
713 BRW_ALIGN1_3SRC_ACCUMULATOR
);
714 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
716 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
717 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
718 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
720 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
722 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
724 if (brw_reg_type_is_floating_point(dest
.type
)) {
725 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
726 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
728 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
729 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
732 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
733 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
734 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
735 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
737 brw_inst_set_3src_a1_src0_vstride(devinfo
, inst
,
738 to_3src_align1_vstride(src0
.vstride
));
739 brw_inst_set_3src_a1_src1_vstride(devinfo
, inst
,
740 to_3src_align1_vstride(src1
.vstride
));
741 /* no vstride on src2 */
743 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
744 to_3src_align1_hstride(src0
.hstride
));
745 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
746 to_3src_align1_hstride(src1
.hstride
));
747 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
748 to_3src_align1_hstride(src2
.hstride
));
750 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
751 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
752 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
754 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
756 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
757 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
759 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
760 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
761 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
763 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
765 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
766 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
768 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
769 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
770 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
771 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
773 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
774 src0
.file
== BRW_IMMEDIATE_VALUE
||
775 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
776 src0
.type
== BRW_REGISTER_TYPE_NF
));
777 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
778 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
779 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
780 src2
.file
== BRW_IMMEDIATE_VALUE
);
782 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
783 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
784 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
785 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
786 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
787 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
788 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
789 BRW_ALIGN1_3SRC_ACCUMULATOR
);
790 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
791 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
792 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
793 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
795 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
796 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
797 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
798 dest
.type
== BRW_REGISTER_TYPE_DF
||
799 dest
.type
== BRW_REGISTER_TYPE_D
||
800 dest
.type
== BRW_REGISTER_TYPE_UD
||
801 (dest
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 8));
802 if (devinfo
->gen
== 6) {
803 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
804 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
806 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
807 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
808 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
810 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
811 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
812 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
813 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
814 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
815 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
816 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
817 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
819 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
820 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
821 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
822 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
823 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
824 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
825 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
826 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
828 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
829 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
830 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
831 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
832 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
833 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
834 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
835 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
837 if (devinfo
->gen
>= 7) {
838 /* Set both the source and destination types based on dest.type,
839 * ignoring the source register types. The MAD and LRP emitters ensure
840 * that all four types are float. The BFE and BFI2 emitters, however,
841 * may send us mixed D and UD types and want us to ignore that and use
842 * the destination type.
844 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
845 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
853 /***********************************************************************
854 * Convenience routines.
857 brw_inst *brw_##OP(struct brw_codegen *p, \
858 struct brw_reg dest, \
859 struct brw_reg src0) \
861 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
865 brw_inst *brw_##OP(struct brw_codegen *p, \
866 struct brw_reg dest, \
867 struct brw_reg src0, \
868 struct brw_reg src1) \
870 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
874 brw_inst *brw_##OP(struct brw_codegen *p, \
875 struct brw_reg dest, \
876 struct brw_reg src0, \
877 struct brw_reg src1, \
878 struct brw_reg src2) \
880 if (p->current->access_mode == BRW_ALIGN_16) { \
881 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
882 src0.swizzle = BRW_SWIZZLE_XXXX; \
883 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
884 src1.swizzle = BRW_SWIZZLE_XXXX; \
885 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
886 src2.swizzle = BRW_SWIZZLE_XXXX; \
888 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
892 brw_inst *brw_##OP(struct brw_codegen *p, \
893 struct brw_reg dest, \
894 struct brw_reg src0, \
895 struct brw_reg src1, \
896 struct brw_reg src2) \
898 assert(dest.type == BRW_REGISTER_TYPE_F || \
899 dest.type == BRW_REGISTER_TYPE_DF); \
900 if (dest.type == BRW_REGISTER_TYPE_F) { \
901 assert(src0.type == BRW_REGISTER_TYPE_F); \
902 assert(src1.type == BRW_REGISTER_TYPE_F); \
903 assert(src2.type == BRW_REGISTER_TYPE_F); \
904 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
905 assert(src0.type == BRW_REGISTER_TYPE_DF); \
906 assert(src1.type == BRW_REGISTER_TYPE_DF); \
907 assert(src2.type == BRW_REGISTER_TYPE_DF); \
910 if (p->current->access_mode == BRW_ALIGN_16) { \
911 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
912 src0.swizzle = BRW_SWIZZLE_XXXX; \
913 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
914 src1.swizzle = BRW_SWIZZLE_XXXX; \
915 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
916 src2.swizzle = BRW_SWIZZLE_XXXX; \
918 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
921 /* Rounding operations (other than RNDD) require two instructions - the first
922 * stores a rounded value (possibly the wrong way) in the dest register, but
923 * also sets a per-channel "increment bit" in the flag register. A predicated
924 * add of 1.0 fixes dest to contain the desired result.
926 * Sandybridge and later appear to round correctly without an ADD.
929 void brw_##OP(struct brw_codegen *p, \
930 struct brw_reg dest, \
931 struct brw_reg src) \
933 const struct gen_device_info *devinfo = p->devinfo; \
934 brw_inst *rnd, *add; \
935 rnd = next_insn(p, BRW_OPCODE_##OP); \
936 brw_set_dest(p, rnd, dest); \
937 brw_set_src0(p, rnd, src); \
939 if (devinfo->gen < 6) { \
940 /* turn on round-increments */ \
941 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
942 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
943 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
983 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
985 const struct gen_device_info
*devinfo
= p
->devinfo
;
987 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
988 * To avoid the problems that causes, we use an <X,2,0> source region to
989 * read each element twice.
991 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
992 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
993 dest
.type
== BRW_REGISTER_TYPE_DF
&&
994 (src0
.type
== BRW_REGISTER_TYPE_F
||
995 src0
.type
== BRW_REGISTER_TYPE_D
||
996 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
997 !has_scalar_region(src0
)) {
998 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
999 src0
.vstride
= src0
.hstride
;
1000 src0
.width
= BRW_WIDTH_2
;
1001 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1004 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
1008 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1009 struct brw_reg src0
, struct brw_reg src1
)
1012 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1013 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1014 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1015 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1016 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1019 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1020 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1021 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1022 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1023 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1026 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1030 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1031 struct brw_reg src0
, struct brw_reg src1
)
1033 assert(dest
.type
== src0
.type
);
1034 assert(src0
.type
== src1
.type
);
1035 switch (src0
.type
) {
1036 case BRW_REGISTER_TYPE_B
:
1037 case BRW_REGISTER_TYPE_UB
:
1038 case BRW_REGISTER_TYPE_W
:
1039 case BRW_REGISTER_TYPE_UW
:
1040 case BRW_REGISTER_TYPE_D
:
1041 case BRW_REGISTER_TYPE_UD
:
1044 unreachable("Bad type for brw_AVG");
1047 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1051 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1052 struct brw_reg src0
, struct brw_reg src1
)
1055 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1056 src0
.type
== BRW_REGISTER_TYPE_UD
||
1057 src1
.type
== BRW_REGISTER_TYPE_D
||
1058 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1059 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1062 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1063 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1064 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1065 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1066 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1069 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1070 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1071 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1072 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1073 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1076 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1077 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1078 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1079 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1081 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1085 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1086 struct brw_reg src0
, struct brw_reg src1
)
1088 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1089 src0
.width
= BRW_WIDTH_1
;
1090 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1091 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1095 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1096 struct brw_reg src0
, struct brw_reg src1
)
1098 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1099 src0
.width
= BRW_WIDTH_1
;
1100 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1101 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1102 src1
.width
= BRW_WIDTH_8
;
1103 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1104 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1108 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1110 const struct gen_device_info
*devinfo
= p
->devinfo
;
1111 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1112 /* The F32TO16 instruction doesn't support 32-bit destination types in
1113 * Align1 mode, and neither does the Gen8 implementation in terms of a
1114 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1115 * an undocumented feature.
1117 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1118 (!align16
|| devinfo
->gen
>= 8));
1122 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1124 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1125 dst
.type
== BRW_REGISTER_TYPE_W
||
1126 dst
.type
== BRW_REGISTER_TYPE_UW
||
1127 dst
.type
== BRW_REGISTER_TYPE_HF
);
1130 brw_push_insn_state(p
);
1132 if (needs_zero_fill
) {
1133 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1134 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1137 if (devinfo
->gen
>= 8) {
1138 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1140 assert(devinfo
->gen
== 7);
1141 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1144 if (needs_zero_fill
) {
1145 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1146 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1147 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1150 brw_pop_insn_state(p
);
1155 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1157 const struct gen_device_info
*devinfo
= p
->devinfo
;
1158 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1161 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1163 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1165 * Because this instruction does not have a 16-bit floating-point
1166 * type, the source data type must be Word (W). The destination type
1167 * must be F (Float).
1169 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1170 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1172 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1173 src
.type
== BRW_REGISTER_TYPE_UW
||
1174 src
.type
== BRW_REGISTER_TYPE_HF
);
1177 if (devinfo
->gen
>= 8) {
1178 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1180 assert(devinfo
->gen
== 7);
1181 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1186 void brw_NOP(struct brw_codegen
*p
)
1188 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1189 memset(insn
, 0, sizeof(*insn
));
1190 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1197 /***********************************************************************
1198 * Comparisons, if/else/endif
1202 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1203 unsigned predicate_control
)
1205 const struct gen_device_info
*devinfo
= p
->devinfo
;
1206 struct brw_reg ip
= brw_ip_reg();
1207 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1209 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1210 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1211 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1212 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1218 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1220 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1222 p
->if_stack_depth
++;
1223 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1224 p
->if_stack_array_size
*= 2;
1225 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1226 p
->if_stack_array_size
);
1231 pop_if_stack(struct brw_codegen
*p
)
1233 p
->if_stack_depth
--;
1234 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1238 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1240 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1241 p
->loop_stack_array_size
*= 2;
1242 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1243 p
->loop_stack_array_size
);
1244 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1245 p
->loop_stack_array_size
);
1248 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1249 p
->loop_stack_depth
++;
1250 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1254 get_inner_do_insn(struct brw_codegen
*p
)
1256 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1259 /* EU takes the value from the flag register and pushes it onto some
1260 * sort of a stack (presumably merging with any flag value already on
1261 * the stack). Within an if block, the flags at the top of the stack
1262 * control execution on each channel of the unit, eg. on each of the
1263 * 16 pixel values in our wm programs.
1265 * When the matching 'else' instruction is reached (presumably by
1266 * countdown of the instruction count patched in by our ELSE/ENDIF
1267 * functions), the relevant flags are inverted.
1269 * When the matching 'endif' instruction is reached, the flags are
1270 * popped off. If the stack is now empty, normal execution resumes.
1273 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1275 const struct gen_device_info
*devinfo
= p
->devinfo
;
1278 insn
= next_insn(p
, BRW_OPCODE_IF
);
1280 /* Override the defaults for this instruction:
1282 if (devinfo
->gen
< 6) {
1283 brw_set_dest(p
, insn
, brw_ip_reg());
1284 brw_set_src0(p
, insn
, brw_ip_reg());
1285 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1286 } else if (devinfo
->gen
== 6) {
1287 brw_set_dest(p
, insn
, brw_imm_w(0));
1288 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1289 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1290 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1291 } else if (devinfo
->gen
== 7) {
1292 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1293 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1294 brw_set_src1(p
, insn
, brw_imm_w(0));
1295 brw_inst_set_jip(devinfo
, insn
, 0);
1296 brw_inst_set_uip(devinfo
, insn
, 0);
1298 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1299 brw_set_src0(p
, insn
, brw_imm_d(0));
1300 brw_inst_set_jip(devinfo
, insn
, 0);
1301 brw_inst_set_uip(devinfo
, insn
, 0);
1304 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1305 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1306 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1307 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1308 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1309 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1311 push_if_stack(p
, insn
);
1312 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1316 /* This function is only used for gen6-style IF instructions with an
1317 * embedded comparison (conditional modifier). It is not used on gen7.
1320 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1321 struct brw_reg src0
, struct brw_reg src1
)
1323 const struct gen_device_info
*devinfo
= p
->devinfo
;
1326 insn
= next_insn(p
, BRW_OPCODE_IF
);
1328 brw_set_dest(p
, insn
, brw_imm_w(0));
1329 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1330 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1331 brw_set_src0(p
, insn
, src0
);
1332 brw_set_src1(p
, insn
, src1
);
1334 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1335 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1336 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1338 push_if_stack(p
, insn
);
1343 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1346 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1347 brw_inst
*if_inst
, brw_inst
*else_inst
)
1349 const struct gen_device_info
*devinfo
= p
->devinfo
;
1351 /* The next instruction (where the ENDIF would be, if it existed) */
1352 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1354 assert(p
->single_program_flow
);
1355 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1356 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1357 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1359 /* Convert IF to an ADD instruction that moves the instruction pointer
1360 * to the first instruction of the ELSE block. If there is no ELSE
1361 * block, point to where ENDIF would be. Reverse the predicate.
1363 * There's no need to execute an ENDIF since we don't need to do any
1364 * stack operations, and if we're currently executing, we just want to
1365 * continue normally.
1367 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1368 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1370 if (else_inst
!= NULL
) {
1371 /* Convert ELSE to an ADD instruction that points where the ENDIF
1374 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1376 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1377 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1379 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1384 * Patch IF and ELSE instructions with appropriate jump targets.
1387 patch_IF_ELSE(struct brw_codegen
*p
,
1388 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1390 const struct gen_device_info
*devinfo
= p
->devinfo
;
1392 /* We shouldn't be patching IF and ELSE instructions in single program flow
1393 * mode when gen < 6, because in single program flow mode on those
1394 * platforms, we convert flow control instructions to conditional ADDs that
1395 * operate on IP (see brw_ENDIF).
1397 * However, on Gen6, writing to IP doesn't work in single program flow mode
1398 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1399 * not be updated by non-flow control instructions."). And on later
1400 * platforms, there is no significant benefit to converting control flow
1401 * instructions to conditional ADDs. So we do patch IF and ELSE
1402 * instructions in single program flow mode on those platforms.
1404 if (devinfo
->gen
< 6)
1405 assert(!p
->single_program_flow
);
1407 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1408 assert(endif_inst
!= NULL
);
1409 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1411 unsigned br
= brw_jump_scale(devinfo
);
1413 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1414 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1416 if (else_inst
== NULL
) {
1417 /* Patch IF -> ENDIF */
1418 if (devinfo
->gen
< 6) {
1419 /* Turn it into an IFF, which means no mask stack operations for
1420 * all-false and jumping past the ENDIF.
1422 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1423 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1424 br
* (endif_inst
- if_inst
+ 1));
1425 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1426 } else if (devinfo
->gen
== 6) {
1427 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1428 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1430 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1431 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1434 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1436 /* Patch IF -> ELSE */
1437 if (devinfo
->gen
< 6) {
1438 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1439 br
* (else_inst
- if_inst
));
1440 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1441 } else if (devinfo
->gen
== 6) {
1442 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1443 br
* (else_inst
- if_inst
+ 1));
1446 /* Patch ELSE -> ENDIF */
1447 if (devinfo
->gen
< 6) {
1448 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1451 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1452 br
* (endif_inst
- else_inst
+ 1));
1453 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1454 } else if (devinfo
->gen
== 6) {
1455 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1456 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1457 br
* (endif_inst
- else_inst
));
1459 /* The IF instruction's JIP should point just past the ELSE */
1460 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1461 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1462 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1463 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1464 if (devinfo
->gen
>= 8) {
1465 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1466 * should point to ENDIF.
1468 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1475 brw_ELSE(struct brw_codegen
*p
)
1477 const struct gen_device_info
*devinfo
= p
->devinfo
;
1480 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1482 if (devinfo
->gen
< 6) {
1483 brw_set_dest(p
, insn
, brw_ip_reg());
1484 brw_set_src0(p
, insn
, brw_ip_reg());
1485 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1486 } else if (devinfo
->gen
== 6) {
1487 brw_set_dest(p
, insn
, brw_imm_w(0));
1488 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1489 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1490 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1491 } else if (devinfo
->gen
== 7) {
1492 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1493 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1494 brw_set_src1(p
, insn
, brw_imm_w(0));
1495 brw_inst_set_jip(devinfo
, insn
, 0);
1496 brw_inst_set_uip(devinfo
, insn
, 0);
1498 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1499 brw_set_src0(p
, insn
, brw_imm_d(0));
1500 brw_inst_set_jip(devinfo
, insn
, 0);
1501 brw_inst_set_uip(devinfo
, insn
, 0);
1504 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1505 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1506 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1507 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1509 push_if_stack(p
, insn
);
1513 brw_ENDIF(struct brw_codegen
*p
)
1515 const struct gen_device_info
*devinfo
= p
->devinfo
;
1516 brw_inst
*insn
= NULL
;
1517 brw_inst
*else_inst
= NULL
;
1518 brw_inst
*if_inst
= NULL
;
1520 bool emit_endif
= true;
1522 /* In single program flow mode, we can express IF and ELSE instructions
1523 * equivalently as ADD instructions that operate on IP. On platforms prior
1524 * to Gen6, flow control instructions cause an implied thread switch, so
1525 * this is a significant savings.
1527 * However, on Gen6, writing to IP doesn't work in single program flow mode
1528 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1529 * not be updated by non-flow control instructions."). And on later
1530 * platforms, there is no significant benefit to converting control flow
1531 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1534 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1538 * A single next_insn() may change the base address of instruction store
1539 * memory(p->store), so call it first before referencing the instruction
1540 * store pointer from an index
1543 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1545 /* Pop the IF and (optional) ELSE instructions from the stack */
1546 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1547 tmp
= pop_if_stack(p
);
1548 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1550 tmp
= pop_if_stack(p
);
1555 /* ENDIF is useless; don't bother emitting it. */
1556 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1560 if (devinfo
->gen
< 6) {
1561 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1562 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1563 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1564 } else if (devinfo
->gen
== 6) {
1565 brw_set_dest(p
, insn
, brw_imm_w(0));
1566 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1567 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1568 } else if (devinfo
->gen
== 7) {
1569 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1570 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1571 brw_set_src1(p
, insn
, brw_imm_w(0));
1573 brw_set_src0(p
, insn
, brw_imm_d(0));
1576 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1577 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1578 if (devinfo
->gen
< 6)
1579 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1581 /* Also pop item off the stack in the endif instruction: */
1582 if (devinfo
->gen
< 6) {
1583 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1584 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1585 } else if (devinfo
->gen
== 6) {
1586 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1588 brw_inst_set_jip(devinfo
, insn
, 2);
1590 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1594 brw_BREAK(struct brw_codegen
*p
)
1596 const struct gen_device_info
*devinfo
= p
->devinfo
;
1599 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1600 if (devinfo
->gen
>= 8) {
1601 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1602 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1603 } else if (devinfo
->gen
>= 6) {
1604 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1605 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1606 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1608 brw_set_dest(p
, insn
, brw_ip_reg());
1609 brw_set_src0(p
, insn
, brw_ip_reg());
1610 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1611 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1612 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1614 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1615 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1621 brw_CONT(struct brw_codegen
*p
)
1623 const struct gen_device_info
*devinfo
= p
->devinfo
;
1626 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1627 brw_set_dest(p
, insn
, brw_ip_reg());
1628 if (devinfo
->gen
>= 8) {
1629 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1631 brw_set_src0(p
, insn
, brw_ip_reg());
1632 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1635 if (devinfo
->gen
< 6) {
1636 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1637 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1639 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1640 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1645 gen6_HALT(struct brw_codegen
*p
)
1647 const struct gen_device_info
*devinfo
= p
->devinfo
;
1650 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1651 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1652 if (devinfo
->gen
>= 8) {
1653 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1655 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1656 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1659 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1660 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1666 * The DO/WHILE is just an unterminated loop -- break or continue are
1667 * used for control within the loop. We have a few ways they can be
1670 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1671 * jip and no DO instruction.
1673 * For non-uniform control flow pre-gen6, there's a DO instruction to
1674 * push the mask, and a WHILE to jump back, and BREAK to get out and
1677 * For gen6, there's no more mask stack, so no need for DO. WHILE
1678 * just points back to the first instruction of the loop.
1681 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1683 const struct gen_device_info
*devinfo
= p
->devinfo
;
1685 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1686 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1687 return &p
->store
[p
->nr_insn
];
1689 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1691 push_loop_stack(p
, insn
);
1693 /* Override the defaults for this instruction:
1695 brw_set_dest(p
, insn
, brw_null_reg());
1696 brw_set_src0(p
, insn
, brw_null_reg());
1697 brw_set_src1(p
, insn
, brw_null_reg());
1699 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1700 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1701 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1708 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1711 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1712 * nesting, since it can always just point to the end of the block/current loop.
1715 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1717 const struct gen_device_info
*devinfo
= p
->devinfo
;
1718 brw_inst
*do_inst
= get_inner_do_insn(p
);
1720 unsigned br
= brw_jump_scale(devinfo
);
1722 assert(devinfo
->gen
< 6);
1724 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1725 /* If the jump count is != 0, that means that this instruction has already
1726 * been patched because it's part of a loop inside of the one we're
1729 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1730 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1731 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1732 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1733 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1734 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1740 brw_WHILE(struct brw_codegen
*p
)
1742 const struct gen_device_info
*devinfo
= p
->devinfo
;
1743 brw_inst
*insn
, *do_insn
;
1744 unsigned br
= brw_jump_scale(devinfo
);
1746 if (devinfo
->gen
>= 6) {
1747 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1748 do_insn
= get_inner_do_insn(p
);
1750 if (devinfo
->gen
>= 8) {
1751 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1752 brw_set_src0(p
, insn
, brw_imm_d(0));
1753 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1754 } else if (devinfo
->gen
== 7) {
1755 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1756 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1757 brw_set_src1(p
, insn
, brw_imm_w(0));
1758 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1760 brw_set_dest(p
, insn
, brw_imm_w(0));
1761 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1762 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1763 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1766 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1769 if (p
->single_program_flow
) {
1770 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1771 do_insn
= get_inner_do_insn(p
);
1773 brw_set_dest(p
, insn
, brw_ip_reg());
1774 brw_set_src0(p
, insn
, brw_ip_reg());
1775 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1776 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1778 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1779 do_insn
= get_inner_do_insn(p
);
1781 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1783 brw_set_dest(p
, insn
, brw_ip_reg());
1784 brw_set_src0(p
, insn
, brw_ip_reg());
1785 brw_set_src1(p
, insn
, brw_imm_d(0));
1787 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1788 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1789 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1791 brw_patch_break_cont(p
, insn
);
1794 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1796 p
->loop_stack_depth
--;
1803 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1805 const struct gen_device_info
*devinfo
= p
->devinfo
;
1806 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1809 if (devinfo
->gen
>= 5)
1812 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1813 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1815 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1816 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1819 /* To integrate with the above, it makes sense that the comparison
1820 * instruction should populate the flag register. It might be simpler
1821 * just to use the flag reg for most WM tasks?
1823 void brw_CMP(struct brw_codegen
*p
,
1824 struct brw_reg dest
,
1825 unsigned conditional
,
1826 struct brw_reg src0
,
1827 struct brw_reg src1
)
1829 const struct gen_device_info
*devinfo
= p
->devinfo
;
1830 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1832 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1833 brw_set_dest(p
, insn
, dest
);
1834 brw_set_src0(p
, insn
, src0
);
1835 brw_set_src1(p
, insn
, src1
);
1837 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1839 * "Any CMP instruction with a null destination must use a {switch}."
1841 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1842 * mentioned on their work-arounds pages.
1844 if (devinfo
->gen
== 7) {
1845 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1846 dest
.nr
== BRW_ARF_NULL
) {
1847 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1852 /***********************************************************************
1853 * Helpers for the various SEND message types:
1856 /** Extended math function, float[8].
1858 void gen4_math(struct brw_codegen
*p
,
1859 struct brw_reg dest
,
1861 unsigned msg_reg_nr
,
1863 unsigned precision
)
1865 const struct gen_device_info
*devinfo
= p
->devinfo
;
1866 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1868 if (has_scalar_region(src
)) {
1869 data_type
= BRW_MATH_DATA_SCALAR
;
1871 data_type
= BRW_MATH_DATA_VECTOR
;
1874 assert(devinfo
->gen
< 6);
1876 /* Example code doesn't set predicate_control for send
1879 brw_inst_set_pred_control(devinfo
, insn
, 0);
1880 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1882 brw_set_dest(p
, insn
, dest
);
1883 brw_set_src0(p
, insn
, src
);
1884 brw_set_math_message(p
,
1887 src
.type
== BRW_REGISTER_TYPE_D
,
1892 void gen6_math(struct brw_codegen
*p
,
1893 struct brw_reg dest
,
1895 struct brw_reg src0
,
1896 struct brw_reg src1
)
1898 const struct gen_device_info
*devinfo
= p
->devinfo
;
1899 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1901 assert(devinfo
->gen
>= 6);
1903 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1904 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1906 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1907 if (devinfo
->gen
== 6) {
1908 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1909 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1912 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1913 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1914 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1915 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1916 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1917 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1918 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1920 assert(src0
.type
== BRW_REGISTER_TYPE_F
||
1921 (src0
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
1922 assert(src1
.type
== BRW_REGISTER_TYPE_F
||
1923 (src1
.type
== BRW_REGISTER_TYPE_HF
&& devinfo
->gen
>= 9));
1926 /* Source modifiers are ignored for extended math instructions on Gen6. */
1927 if (devinfo
->gen
== 6) {
1928 assert(!src0
.negate
);
1930 assert(!src1
.negate
);
1934 brw_inst_set_math_function(devinfo
, insn
, function
);
1936 brw_set_dest(p
, insn
, dest
);
1937 brw_set_src0(p
, insn
, src0
);
1938 brw_set_src1(p
, insn
, src1
);
1942 * Return the right surface index to access the thread scratch space using
1943 * stateless dataport messages.
1946 brw_scratch_surface_idx(const struct brw_codegen
*p
)
1948 /* The scratch space is thread-local so IA coherency is unnecessary. */
1949 if (p
->devinfo
->gen
>= 8)
1950 return GEN8_BTI_STATELESS_NON_COHERENT
;
1952 return BRW_BTI_STATELESS
;
1956 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1957 * using a constant offset per channel.
1959 * The offset must be aligned to oword size (16 bytes). Used for
1960 * register spilling.
1962 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1967 const struct gen_device_info
*devinfo
= p
->devinfo
;
1968 const unsigned target_cache
=
1969 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
1970 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
1971 BRW_SFID_DATAPORT_WRITE
);
1974 if (devinfo
->gen
>= 6)
1977 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1979 const unsigned mlen
= 1 + num_regs
;
1981 /* Set up the message header. This is g0, with g0.2 filled with
1982 * the offset. We don't want to leave our offset around in g0 or
1983 * it'll screw up texture samples, so set it up inside the message
1987 brw_push_insn_state(p
);
1988 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
1989 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
1990 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
1992 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
1994 /* set message header global offset field (reg 0, element 2) */
1995 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
1997 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
1999 2), BRW_REGISTER_TYPE_UD
),
2000 brw_imm_ud(offset
));
2002 brw_pop_insn_state(p
);
2006 struct brw_reg dest
;
2007 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2008 int send_commit_msg
;
2009 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2010 BRW_REGISTER_TYPE_UW
);
2012 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2013 brw_inst_set_compression(devinfo
, insn
, false);
2015 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2016 src_header
= vec16(src_header
);
2018 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2019 if (devinfo
->gen
< 6)
2020 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2022 /* Until gen6, writes followed by reads from the same location
2023 * are not guaranteed to be ordered unless write_commit is set.
2024 * If set, then a no-op write is issued to the destination
2025 * register to set a dependency, and a read from the destination
2026 * can be used to ensure the ordering.
2028 * For gen6, only writes between different threads need ordering
2029 * protection. Our use of DP writes is all about register
2030 * spilling within a thread.
2032 if (devinfo
->gen
>= 6) {
2033 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2034 send_commit_msg
= 0;
2037 send_commit_msg
= 1;
2040 brw_set_dest(p
, insn
, dest
);
2041 if (devinfo
->gen
>= 6) {
2042 brw_set_src0(p
, insn
, mrf
);
2044 brw_set_src0(p
, insn
, brw_null_reg());
2047 if (devinfo
->gen
>= 6)
2048 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2050 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2052 brw_set_desc(p
, insn
,
2053 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2054 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2055 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2056 msg_type
, 0, /* not a render target */
2063 * Read a block of owords (half a GRF each) from the scratch buffer
2064 * using a constant index per channel.
2066 * Offset must be aligned to oword size (16 bytes). Used for register
2070 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2071 struct brw_reg dest
,
2076 const struct gen_device_info
*devinfo
= p
->devinfo
;
2078 if (devinfo
->gen
>= 6)
2081 if (p
->devinfo
->gen
>= 7) {
2082 /* On gen 7 and above, we no longer have message registers and we can
2083 * send from any register we want. By using the destination register
2084 * for the message, we guarantee that the implied message write won't
2085 * accidentally overwrite anything. This has been a problem because
2086 * the MRF registers and source for the final FB write are both fixed
2089 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2091 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2093 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2095 const unsigned rlen
= num_regs
;
2096 const unsigned target_cache
=
2097 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2098 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2099 BRW_SFID_DATAPORT_READ
);
2102 brw_push_insn_state(p
);
2103 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2104 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2105 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2107 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2109 /* set message header global offset field (reg 0, element 2) */
2110 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2111 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2113 brw_pop_insn_state(p
);
2117 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2119 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2120 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2121 brw_inst_set_compression(devinfo
, insn
, false);
2123 brw_set_dest(p
, insn
, dest
); /* UW? */
2124 if (devinfo
->gen
>= 6) {
2125 brw_set_src0(p
, insn
, mrf
);
2127 brw_set_src0(p
, insn
, brw_null_reg());
2128 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2131 brw_set_desc(p
, insn
,
2132 brw_message_desc(devinfo
, 1, rlen
, true) |
2133 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2134 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2135 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2136 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2141 gen7_block_read_scratch(struct brw_codegen
*p
,
2142 struct brw_reg dest
,
2146 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2147 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2149 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2151 /* The HW requires that the header is present; this is to get the g0.5
2154 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2156 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2157 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2158 * is 32 bytes, which happens to be the size of a register.
2161 assert(offset
< (1 << 12));
2163 gen7_set_dp_scratch_message(p
, insn
,
2164 false, /* scratch read */
2166 false, /* invalidate after read */
2169 1, /* mlen: just g0 */
2170 num_regs
, /* rlen */
2171 true); /* header present */
2175 * Read float[4] vectors from the data port constant cache.
2176 * Location (in buffer) should be a multiple of 16.
2177 * Used for fetching shader constants.
2179 void brw_oword_block_read(struct brw_codegen
*p
,
2180 struct brw_reg dest
,
2183 uint32_t bind_table_index
)
2185 const struct gen_device_info
*devinfo
= p
->devinfo
;
2186 const unsigned target_cache
=
2187 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2188 BRW_SFID_DATAPORT_READ
);
2189 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2191 /* On newer hardware, offset is in units of owords. */
2192 if (devinfo
->gen
>= 6)
2195 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2197 brw_push_insn_state(p
);
2198 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2199 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2200 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2202 brw_push_insn_state(p
);
2203 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2204 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2206 /* set message header global offset field (reg 0, element 2) */
2207 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2209 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2211 2), BRW_REGISTER_TYPE_UD
),
2212 brw_imm_ud(offset
));
2213 brw_pop_insn_state(p
);
2215 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2217 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2219 /* cast dest to a uword[8] vector */
2220 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2222 brw_set_dest(p
, insn
, dest
);
2223 if (devinfo
->gen
>= 6) {
2224 brw_set_src0(p
, insn
, mrf
);
2226 brw_set_src0(p
, insn
, brw_null_reg());
2227 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2230 brw_set_desc(p
, insn
,
2231 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2232 brw_dp_read_desc(devinfo
, bind_table_index
,
2233 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2234 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2235 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2237 brw_pop_insn_state(p
);
2241 brw_fb_WRITE(struct brw_codegen
*p
,
2242 struct brw_reg payload
,
2243 struct brw_reg implied_header
,
2244 unsigned msg_control
,
2245 unsigned binding_table_index
,
2246 unsigned msg_length
,
2247 unsigned response_length
,
2249 bool last_render_target
,
2250 bool header_present
)
2252 const struct gen_device_info
*devinfo
= p
->devinfo
;
2253 const unsigned target_cache
=
2254 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2255 BRW_SFID_DATAPORT_WRITE
);
2258 struct brw_reg dest
, src0
;
2260 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2261 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2263 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2265 if (devinfo
->gen
>= 6) {
2266 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2268 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2270 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2271 brw_inst_set_compression(devinfo
, insn
, false);
2273 if (devinfo
->gen
>= 6) {
2274 /* headerless version, just submit color payload */
2277 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2279 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2280 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2281 src0
= implied_header
;
2283 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2286 brw_set_dest(p
, insn
, dest
);
2287 brw_set_src0(p
, insn
, src0
);
2288 brw_set_desc(p
, insn
,
2289 brw_message_desc(devinfo
, msg_length
, response_length
,
2291 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2292 msg_type
, last_render_target
,
2293 0 /* send_commit_msg */));
2294 brw_inst_set_eot(devinfo
, insn
, eot
);
2300 gen9_fb_READ(struct brw_codegen
*p
,
2302 struct brw_reg payload
,
2303 unsigned binding_table_index
,
2304 unsigned msg_length
,
2305 unsigned response_length
,
2308 const struct gen_device_info
*devinfo
= p
->devinfo
;
2309 assert(devinfo
->gen
>= 9);
2310 const unsigned msg_subtype
=
2311 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2312 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2314 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2315 brw_set_dest(p
, insn
, dst
);
2316 brw_set_src0(p
, insn
, payload
);
2319 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2320 brw_dp_read_desc(devinfo
, binding_table_index
,
2321 per_sample
<< 5 | msg_subtype
,
2322 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2323 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2324 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2330 * Texture sample instruction.
2331 * Note: the msg_type plus msg_length values determine exactly what kind
2332 * of sampling operation is performed. See volume 4, page 161 of docs.
2334 void brw_SAMPLE(struct brw_codegen
*p
,
2335 struct brw_reg dest
,
2336 unsigned msg_reg_nr
,
2337 struct brw_reg src0
,
2338 unsigned binding_table_index
,
2341 unsigned response_length
,
2342 unsigned msg_length
,
2343 unsigned header_present
,
2345 unsigned return_format
)
2347 const struct gen_device_info
*devinfo
= p
->devinfo
;
2350 if (msg_reg_nr
!= -1)
2351 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2353 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2354 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2355 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2357 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2359 * "Instruction compression is not allowed for this instruction (that
2360 * is, send). The hardware behavior is undefined if this instruction is
2361 * set as compressed. However, compress control can be set to "SecHalf"
2362 * to affect the EMask generation."
2364 * No similar wording is found in later PRMs, but there are examples
2365 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2366 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2367 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2369 brw_inst_set_compression(devinfo
, insn
, false);
2371 if (devinfo
->gen
< 6)
2372 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2374 brw_set_dest(p
, insn
, dest
);
2375 brw_set_src0(p
, insn
, src0
);
2376 brw_set_desc(p
, insn
,
2377 brw_message_desc(devinfo
, msg_length
, response_length
,
2379 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2380 msg_type
, simd_mode
, return_format
));
2383 /* Adjust the message header's sampler state pointer to
2384 * select the correct group of 16 samplers.
2386 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2387 struct brw_reg header
,
2388 struct brw_reg sampler_index
)
2390 /* The "Sampler Index" field can only store values between 0 and 15.
2391 * However, we can add an offset to the "Sampler State Pointer"
2392 * field, effectively selecting a different set of 16 samplers.
2394 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2395 * offset, and each sampler state is only 16-bytes, so we can't
2396 * exclusively use the offset - we have to use both.
2399 const struct gen_device_info
*devinfo
= p
->devinfo
;
2401 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2402 const int sampler_state_size
= 16; /* 16 bytes */
2403 uint32_t sampler
= sampler_index
.ud
;
2405 if (sampler
>= 16) {
2406 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2408 get_element_ud(header
, 3),
2409 get_element_ud(brw_vec8_grf(0, 0), 3),
2410 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2413 /* Non-const sampler array indexing case */
2414 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2418 struct brw_reg temp
= get_element_ud(header
, 3);
2420 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2421 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2423 get_element_ud(header
, 3),
2424 get_element_ud(brw_vec8_grf(0, 0), 3),
2429 /* All these variables are pretty confusing - we might be better off
2430 * using bitmasks and macros for this, in the old style. Or perhaps
2431 * just having the caller instantiate the fields in dword3 itself.
2433 void brw_urb_WRITE(struct brw_codegen
*p
,
2434 struct brw_reg dest
,
2435 unsigned msg_reg_nr
,
2436 struct brw_reg src0
,
2437 enum brw_urb_write_flags flags
,
2438 unsigned msg_length
,
2439 unsigned response_length
,
2443 const struct gen_device_info
*devinfo
= p
->devinfo
;
2446 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2448 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2449 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2450 brw_push_insn_state(p
);
2451 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2452 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2453 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2454 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2455 BRW_REGISTER_TYPE_UD
),
2456 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2457 brw_imm_ud(0xff00));
2458 brw_pop_insn_state(p
);
2461 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2463 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2465 brw_set_dest(p
, insn
, dest
);
2466 brw_set_src0(p
, insn
, src0
);
2467 brw_set_src1(p
, insn
, brw_imm_d(0));
2469 if (devinfo
->gen
< 6)
2470 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2472 brw_set_urb_message(p
,
2482 brw_send_indirect_message(struct brw_codegen
*p
,
2485 struct brw_reg payload
,
2486 struct brw_reg desc
,
2490 const struct gen_device_info
*devinfo
= p
->devinfo
;
2491 struct brw_inst
*send
;
2493 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2495 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2497 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2498 send
= next_insn(p
, BRW_OPCODE_SEND
);
2499 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2502 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2504 brw_push_insn_state(p
);
2505 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2506 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2507 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2508 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2510 /* Load the indirect descriptor to an address register using OR so the
2511 * caller can specify additional descriptor bits with the desc_imm
2514 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2516 brw_pop_insn_state(p
);
2518 send
= next_insn(p
, BRW_OPCODE_SEND
);
2519 brw_set_src1(p
, send
, addr
);
2522 brw_set_dest(p
, send
, dst
);
2523 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2524 brw_inst_set_sfid(devinfo
, send
, sfid
);
2525 brw_inst_set_eot(devinfo
, send
, eot
);
2529 brw_send_indirect_split_message(struct brw_codegen
*p
,
2532 struct brw_reg payload0
,
2533 struct brw_reg payload1
,
2534 struct brw_reg desc
,
2536 struct brw_reg ex_desc
,
2537 unsigned ex_desc_imm
,
2540 const struct gen_device_info
*devinfo
= p
->devinfo
;
2541 struct brw_inst
*send
;
2543 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2545 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2547 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2548 desc
.ud
|= desc_imm
;
2550 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2552 brw_push_insn_state(p
);
2553 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2554 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2555 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2556 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2558 /* Load the indirect descriptor to an address register using OR so the
2559 * caller can specify additional descriptor bits with the desc_imm
2562 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2564 brw_pop_insn_state(p
);
2568 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2569 ex_desc
.ud
|= ex_desc_imm
;
2571 struct brw_reg addr
= retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD
);
2573 brw_push_insn_state(p
);
2574 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2575 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2576 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2577 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2579 /* Load the indirect extended descriptor to an address register using OR
2580 * so the caller can specify additional descriptor bits with the
2581 * desc_imm immediate.
2583 * Even though the instruction dispatcher always pulls the SFID and EOT
2584 * fields from the instruction itself, actual external unit which
2585 * processes the message gets the SFID and EOT from the extended
2586 * descriptor which comes from the address register. If we don't OR
2587 * those two bits in, the external unit may get confused and hang.
2589 brw_OR(p
, addr
, ex_desc
, brw_imm_ud(ex_desc_imm
| sfid
| eot
<< 5));
2591 brw_pop_insn_state(p
);
2595 send
= next_insn(p
, BRW_OPCODE_SENDS
);
2596 brw_set_dest(p
, send
, dst
);
2597 brw_set_src0(p
, send
, retype(payload0
, BRW_REGISTER_TYPE_UD
));
2598 brw_set_src1(p
, send
, retype(payload1
, BRW_REGISTER_TYPE_UD
));
2600 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2601 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 0);
2602 brw_inst_set_send_desc(devinfo
, send
, desc
.ud
);
2604 assert(desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2605 assert(desc
.nr
== BRW_ARF_ADDRESS
);
2606 assert(desc
.subnr
== 0);
2607 brw_inst_set_send_sel_reg32_desc(devinfo
, send
, 1);
2610 if (ex_desc
.file
== BRW_IMMEDIATE_VALUE
) {
2611 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 0);
2612 brw_inst_set_send_ex_desc(devinfo
, send
, ex_desc
.ud
);
2614 assert(ex_desc
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
2615 assert(ex_desc
.nr
== BRW_ARF_ADDRESS
);
2616 assert((ex_desc
.subnr
& 0x3) == 0);
2617 brw_inst_set_send_sel_reg32_ex_desc(devinfo
, send
, 1);
2618 brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo
, send
, ex_desc
.subnr
>> 2);
2621 brw_inst_set_sfid(devinfo
, send
, sfid
);
2622 brw_inst_set_eot(devinfo
, send
, eot
);
2626 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2629 struct brw_reg payload
,
2630 struct brw_reg surface
,
2633 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2634 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2636 brw_push_insn_state(p
);
2637 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2638 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2639 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2640 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2642 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2643 * some surface array is accessed out of bounds.
2646 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2647 BRW_GET_SWZ(surface
.swizzle
, 0)),
2650 brw_pop_insn_state(p
);
2655 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
, false);
2659 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2660 brw_inst
*insn
, int while_offset
, int start_offset
)
2662 int scale
= 16 / brw_jump_scale(devinfo
);
2663 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2664 : brw_inst_jip(devinfo
, insn
);
2666 return while_offset
+ jip
* scale
<= start_offset
;
2671 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2674 void *store
= p
->store
;
2675 const struct gen_device_info
*devinfo
= p
->devinfo
;
2679 for (offset
= next_offset(devinfo
, store
, start_offset
);
2680 offset
< p
->next_insn_offset
;
2681 offset
= next_offset(devinfo
, store
, offset
)) {
2682 brw_inst
*insn
= store
+ offset
;
2684 switch (brw_inst_opcode(devinfo
, insn
)) {
2688 case BRW_OPCODE_ENDIF
:
2693 case BRW_OPCODE_WHILE
:
2694 /* If the while doesn't jump before our instruction, it's the end
2695 * of a sibling do...while loop. Ignore it.
2697 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2700 case BRW_OPCODE_ELSE
:
2701 case BRW_OPCODE_HALT
:
2710 /* There is no DO instruction on gen6, so to find the end of the loop
2711 * we have to see if the loop is jumping back before our start
2715 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2717 const struct gen_device_info
*devinfo
= p
->devinfo
;
2719 void *store
= p
->store
;
2721 assert(devinfo
->gen
>= 6);
2723 /* Always start after the instruction (such as a WHILE) we're trying to fix
2726 for (offset
= next_offset(devinfo
, store
, start_offset
);
2727 offset
< p
->next_insn_offset
;
2728 offset
= next_offset(devinfo
, store
, offset
)) {
2729 brw_inst
*insn
= store
+ offset
;
2731 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2732 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2736 assert(!"not reached");
2737 return start_offset
;
2740 /* After program generation, go back and update the UIP and JIP of
2741 * BREAK, CONT, and HALT instructions to their correct locations.
2744 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2746 const struct gen_device_info
*devinfo
= p
->devinfo
;
2748 int br
= brw_jump_scale(devinfo
);
2749 int scale
= 16 / br
;
2750 void *store
= p
->store
;
2752 if (devinfo
->gen
< 6)
2755 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2756 brw_inst
*insn
= store
+ offset
;
2757 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2759 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2760 switch (brw_inst_opcode(devinfo
, insn
)) {
2761 case BRW_OPCODE_BREAK
:
2762 assert(block_end_offset
!= 0);
2763 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2764 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2765 brw_inst_set_uip(devinfo
, insn
,
2766 (brw_find_loop_end(p
, offset
) - offset
+
2767 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2769 case BRW_OPCODE_CONTINUE
:
2770 assert(block_end_offset
!= 0);
2771 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2772 brw_inst_set_uip(devinfo
, insn
,
2773 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2775 assert(brw_inst_uip(devinfo
, insn
) != 0);
2776 assert(brw_inst_jip(devinfo
, insn
) != 0);
2779 case BRW_OPCODE_ENDIF
: {
2780 int32_t jump
= (block_end_offset
== 0) ?
2781 1 * br
: (block_end_offset
- offset
) / scale
;
2782 if (devinfo
->gen
>= 7)
2783 brw_inst_set_jip(devinfo
, insn
, jump
);
2785 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2789 case BRW_OPCODE_HALT
:
2790 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2792 * "In case of the halt instruction not inside any conditional
2793 * code block, the value of <JIP> and <UIP> should be the
2794 * same. In case of the halt instruction inside conditional code
2795 * block, the <UIP> should be the end of the program, and the
2796 * <JIP> should be end of the most inner conditional code block."
2798 * The uip will have already been set by whoever set up the
2801 if (block_end_offset
== 0) {
2802 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2804 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2806 assert(brw_inst_uip(devinfo
, insn
) != 0);
2807 assert(brw_inst_jip(devinfo
, insn
) != 0);
2813 void brw_ff_sync(struct brw_codegen
*p
,
2814 struct brw_reg dest
,
2815 unsigned msg_reg_nr
,
2816 struct brw_reg src0
,
2818 unsigned response_length
,
2821 const struct gen_device_info
*devinfo
= p
->devinfo
;
2824 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2826 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2827 brw_set_dest(p
, insn
, dest
);
2828 brw_set_src0(p
, insn
, src0
);
2829 brw_set_src1(p
, insn
, brw_imm_d(0));
2831 if (devinfo
->gen
< 6)
2832 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2834 brw_set_ff_sync_message(p
,
2842 * Emit the SEND instruction necessary to generate stream output data on Gen6
2843 * (for transform feedback).
2845 * If send_commit_msg is true, this is the last piece of stream output data
2846 * from this thread, so send the data as a committed write. According to the
2847 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2849 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2850 * writes are complete by sending the final write as a committed write."
2853 brw_svb_write(struct brw_codegen
*p
,
2854 struct brw_reg dest
,
2855 unsigned msg_reg_nr
,
2856 struct brw_reg src0
,
2857 unsigned binding_table_index
,
2858 bool send_commit_msg
)
2860 const struct gen_device_info
*devinfo
= p
->devinfo
;
2861 const unsigned target_cache
=
2862 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2863 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2864 BRW_SFID_DATAPORT_WRITE
);
2867 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2869 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2870 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2871 brw_set_dest(p
, insn
, dest
);
2872 brw_set_src0(p
, insn
, src0
);
2873 brw_set_desc(p
, insn
,
2874 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
2875 brw_dp_write_desc(devinfo
, binding_table_index
,
2876 0, /* msg_control: ignored */
2877 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2878 0, /* last_render_target: ignored */
2879 send_commit_msg
)); /* send_commit_msg */
2883 brw_surface_payload_size(struct brw_codegen
*p
,
2884 unsigned num_channels
,
2885 unsigned exec_size
/**< 0 for SIMD4x2 */)
2888 return 1; /* SIMD4x2 */
2889 else if (exec_size
<= 8)
2890 return num_channels
;
2892 return 2 * num_channels
;
2896 brw_untyped_atomic(struct brw_codegen
*p
,
2898 struct brw_reg payload
,
2899 struct brw_reg surface
,
2901 unsigned msg_length
,
2902 bool response_expected
,
2903 bool header_present
)
2905 const struct gen_device_info
*devinfo
= p
->devinfo
;
2906 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2907 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2908 GEN7_SFID_DATAPORT_DATA_CACHE
);
2909 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2910 /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
2911 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2912 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2913 has_simd4x2
? 0 : 8;
2914 const unsigned response_length
=
2915 brw_surface_payload_size(p
, response_expected
, exec_size
);
2916 const unsigned desc
=
2917 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
2918 brw_dp_untyped_atomic_desc(devinfo
, exec_size
, atomic_op
,
2920 /* Mask out unused components -- This is especially important in Align16
2921 * mode on generations that don't have native support for SIMD4x2 atomics,
2922 * because unused but enabled components will cause the dataport to perform
2923 * additional atomic operations on the addresses that happen to be in the
2924 * uninitialized Y, Z and W coordinates of the payload.
2926 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2928 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
2929 payload
, surface
, desc
);
2933 brw_untyped_surface_read(struct brw_codegen
*p
,
2935 struct brw_reg payload
,
2936 struct brw_reg surface
,
2937 unsigned msg_length
,
2938 unsigned num_channels
)
2940 const struct gen_device_info
*devinfo
= p
->devinfo
;
2941 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2942 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2943 GEN7_SFID_DATAPORT_DATA_CACHE
);
2944 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2945 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) : 0;
2946 const unsigned response_length
=
2947 brw_surface_payload_size(p
, num_channels
, exec_size
);
2948 const unsigned desc
=
2949 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
2950 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, false);
2952 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
2956 brw_untyped_surface_write(struct brw_codegen
*p
,
2957 struct brw_reg payload
,
2958 struct brw_reg surface
,
2959 unsigned msg_length
,
2960 unsigned num_channels
,
2961 bool header_present
)
2963 const struct gen_device_info
*devinfo
= p
->devinfo
;
2964 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2965 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2966 GEN7_SFID_DATAPORT_DATA_CACHE
);
2967 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2968 /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
2969 const bool has_simd4x2
= devinfo
->gen
>= 8 || devinfo
->is_haswell
;
2970 const unsigned exec_size
= align1
? 1 << brw_get_default_exec_size(p
) :
2971 has_simd4x2
? 0 : 8;
2972 const unsigned desc
=
2973 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
2974 brw_dp_untyped_surface_rw_desc(devinfo
, exec_size
, num_channels
, true);
2975 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2976 const unsigned mask
= !has_simd4x2
&& !align1
? WRITEMASK_X
: WRITEMASK_XYZW
;
2978 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2979 payload
, surface
, desc
);
2983 brw_set_memory_fence_message(struct brw_codegen
*p
,
2984 struct brw_inst
*insn
,
2985 enum brw_message_target sfid
,
2988 const struct gen_device_info
*devinfo
= p
->devinfo
;
2990 brw_set_desc(p
, insn
, brw_message_desc(
2991 devinfo
, 1, (commit_enable
? 1 : 0), true));
2993 brw_inst_set_sfid(devinfo
, insn
, sfid
);
2996 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
2997 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
2999 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3000 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3003 unreachable("Not reached");
3007 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3011 brw_memory_fence(struct brw_codegen
*p
,
3013 enum opcode send_op
)
3015 const struct gen_device_info
*devinfo
= p
->devinfo
;
3016 const bool commit_enable
=
3017 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3018 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3019 struct brw_inst
*insn
;
3021 brw_push_insn_state(p
);
3022 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3023 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3026 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3027 * message doesn't write anything back.
3029 insn
= next_insn(p
, send_op
);
3030 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3031 brw_set_dest(p
, insn
, dst
);
3032 brw_set_src0(p
, insn
, dst
);
3033 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3036 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3037 /* IVB does typed surface access through the render cache, so we need to
3038 * flush it too. Use a different register so both flushes can be
3039 * pipelined by the hardware.
3041 insn
= next_insn(p
, send_op
);
3042 brw_set_dest(p
, insn
, offset(dst
, 1));
3043 brw_set_src0(p
, insn
, offset(dst
, 1));
3044 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3047 /* Now write the response of the second message into the response of the
3048 * first to trigger a pipeline stall -- This way future render and data
3049 * cache messages will be properly ordered with respect to past data and
3050 * render cache messages.
3052 brw_MOV(p
, dst
, offset(dst
, 1));
3055 brw_pop_insn_state(p
);
3059 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3060 struct brw_reg dest
,
3064 struct brw_reg data
,
3065 unsigned msg_length
,
3066 unsigned response_length
)
3068 const struct gen_device_info
*devinfo
= p
->devinfo
;
3069 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3070 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3071 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3072 const unsigned desc
=
3073 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3074 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3077 /* brw_send_indirect_message will automatically use a direct send message
3078 * if data is actually immediate.
3080 brw_send_indirect_message(p
,
3081 GEN7_SFID_PIXEL_INTERPOLATOR
,
3090 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3091 struct brw_reg mask
)
3093 const struct gen_device_info
*devinfo
= p
->devinfo
;
3094 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3095 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3098 assert(devinfo
->gen
>= 7);
3099 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3101 brw_push_insn_state(p
);
3103 /* The flag register is only used on Gen7 in align1 mode, so avoid setting
3104 * unnecessary bits in the instruction words, get the information we need
3105 * and reset the default flag register. This allows more instructions to be
3108 const unsigned flag_subreg
= p
->current
->flag_subreg
;
3109 brw_set_default_flag_reg(p
, 0, 0);
3111 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3112 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3114 if (devinfo
->gen
>= 8) {
3115 /* Getting the first active channel index is easy on Gen8: Just find
3116 * the first bit set in the execution mask. The register exists on
3117 * HSW already but it reads back as all ones when the current
3118 * instruction has execution masking disabled, so it's kind of
3121 struct brw_reg exec_mask
=
3122 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3124 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3125 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3126 /* Unfortunately, ce0 does not take into account the thread
3127 * dispatch mask, which may be a problem in cases where it's not
3128 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3129 * some n). Combine ce0 with the given dispatch (or vector) mask
3130 * to mask off those channels which were never dispatched by the
3133 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3134 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3135 exec_mask
= vec1(dst
);
3138 /* Quarter control has the effect of magically shifting the value of
3139 * ce0 so you'll get the first active channel relative to the
3140 * specified quarter control as result.
3142 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3144 const struct brw_reg flag
= brw_flag_subreg(flag_subreg
);
3146 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3147 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3149 /* Run enough instructions returning zero with execution masking and
3150 * a conditional modifier enabled in order to get the full execution
3151 * mask in f1.0. We could use a single 32-wide move here if it
3152 * weren't because of the hardware bug that causes channel enables to
3153 * be applied incorrectly to the second half of 32-wide instructions
3156 const unsigned lower_size
= MIN2(16, exec_size
);
3157 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3158 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3160 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3161 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3162 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3163 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3164 brw_inst_set_flag_reg_nr(devinfo
, inst
, flag_subreg
/ 2);
3165 brw_inst_set_flag_subreg_nr(devinfo
, inst
, flag_subreg
% 2);
3168 /* Find the first bit set in the exec_size-wide portion of the flag
3169 * register that was updated by the last sequence of MOV
3172 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3173 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3174 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3177 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3179 if (devinfo
->gen
>= 8 &&
3180 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3181 /* In SIMD4x2 mode the first active channel index is just the
3182 * negation of the first bit of the mask register. Note that ce0
3183 * doesn't take into account the dispatch mask, so the Gen7 path
3184 * should be used instead unless you have the guarantee that the
3185 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3188 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3189 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3193 /* Overwrite the destination without and with execution masking to
3194 * find out which of the channels is active.
3196 brw_push_insn_state(p
);
3197 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3198 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3201 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3203 brw_pop_insn_state(p
);
3204 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3208 brw_pop_insn_state(p
);
3212 brw_broadcast(struct brw_codegen
*p
,
3217 const struct gen_device_info
*devinfo
= p
->devinfo
;
3218 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3221 brw_push_insn_state(p
);
3222 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3223 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3225 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3226 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3227 assert(!src
.abs
&& !src
.negate
);
3228 assert(src
.type
== dst
.type
);
3230 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3231 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3232 /* Trivial, the source is already uniform or the index is a constant.
3233 * We will typically not get here if the optimizer is doing its job, but
3234 * asserting would be mean.
3236 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3238 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3239 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3241 /* From the Haswell PRM section "Register Region Restrictions":
3243 * "The lower bits of the AddressImmediate must not overflow to
3244 * change the register address. The lower 5 bits of Address
3245 * Immediate when added to lower 5 bits of address register gives
3246 * the sub-register offset. The upper bits of Address Immediate
3247 * when added to upper bits of address register gives the register
3248 * address. Any overflow from sub-register offset is dropped."
3250 * Fortunately, for broadcast, we never have a sub-register offset so
3251 * this isn't an issue.
3253 assert(src
.subnr
== 0);
3256 const struct brw_reg addr
=
3257 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3258 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3259 /* Limit in bytes of the signed indirect addressing immediate. */
3260 const unsigned limit
= 512;
3262 brw_push_insn_state(p
);
3263 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3264 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3266 /* Take into account the component size and horizontal stride. */
3267 assert(src
.vstride
== src
.hstride
+ src
.width
);
3268 brw_SHL(p
, addr
, vec1(idx
),
3269 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3272 /* We can only address up to limit bytes using the indirect
3273 * addressing immediate, account for the difference if the source
3274 * register is above this limit.
3276 if (offset
>= limit
) {
3277 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3278 offset
= offset
% limit
;
3281 brw_pop_insn_state(p
);
3283 /* Use indirect addressing to fetch the specified component. */
3284 if (type_sz(src
.type
) > 4 &&
3285 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3286 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3288 * "When source or destination datatype is 64b or operation is
3289 * integer DWord multiply, indirect addressing must not be
3292 * To work around both of this issue, we do two integer MOVs
3293 * insead of one 64-bit MOV. Because no double value should ever
3294 * cross a register boundary, it's safe to use the immediate
3295 * offset in the indirect here to handle adding 4 bytes to the
3296 * offset and avoid the extra ADD to the register file.
3298 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3299 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3300 BRW_REGISTER_TYPE_D
));
3301 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3302 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3303 BRW_REGISTER_TYPE_D
));
3306 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3309 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3310 * to all bits of a flag register,
3314 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3315 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3316 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3317 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3319 /* and use predicated SEL to pick the right channel. */
3320 inst
= brw_SEL(p
, dst
,
3321 stride(suboffset(src
, 4), 4, 4, 1),
3322 stride(src
, 4, 4, 1));
3323 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3324 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3328 brw_pop_insn_state(p
);
3332 * This instruction is generated as a single-channel align1 instruction by
3333 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3335 * We can't use the typed atomic op in the FS because that has the execution
3336 * mask ANDed with the pixel mask, but we just want to write the one dword for
3339 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3340 * one u32. So we use the same untyped atomic write message as the pixel
3343 * The untyped atomic operation requires a BUFFER surface type with RAW
3344 * format, and is only accessible through the legacy DATA_CACHE dataport
3347 void brw_shader_time_add(struct brw_codegen
*p
,
3348 struct brw_reg payload
,
3349 uint32_t surf_index
)
3351 const struct gen_device_info
*devinfo
= p
->devinfo
;
3352 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3353 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3354 GEN7_SFID_DATAPORT_DATA_CACHE
);
3355 assert(devinfo
->gen
>= 7);
3357 brw_push_insn_state(p
);
3358 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3359 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3360 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3361 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3363 /* We use brw_vec1_reg and unmasked because we want to increment the given
3366 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3368 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3370 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3371 brw_dp_untyped_atomic_desc(devinfo
, 1, BRW_AOP_ADD
,
3374 brw_inst_set_sfid(devinfo
, send
, sfid
);
3375 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3377 brw_pop_insn_state(p
);
3382 * Emit the SEND message for a barrier
3385 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3387 const struct gen_device_info
*devinfo
= p
->devinfo
;
3388 struct brw_inst
*inst
;
3390 assert(devinfo
->gen
>= 7);
3392 brw_push_insn_state(p
);
3393 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3394 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3395 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3396 brw_set_src0(p
, inst
, src
);
3397 brw_set_src1(p
, inst
, brw_null_reg());
3398 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3400 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3401 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3402 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3403 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3405 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3406 brw_pop_insn_state(p
);
3411 * Emit the wait instruction for a barrier
3414 brw_WAIT(struct brw_codegen
*p
)
3416 const struct gen_device_info
*devinfo
= p
->devinfo
;
3417 struct brw_inst
*insn
;
3419 struct brw_reg src
= brw_notification_reg();
3421 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3422 brw_set_dest(p
, insn
, src
);
3423 brw_set_src0(p
, insn
, src
);
3424 brw_set_src1(p
, insn
, brw_null_reg());
3426 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3427 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3431 * Changes the floating point rounding mode updating the control register
3432 * field defined at cr0.0[5-6] bits. This function supports the changes to
3433 * RTNE (00), RU (01), RD (10) and RTZ (11) rounding using bitwise operations.
3434 * Only RTNE and RTZ rounding are enabled at nir.
3437 brw_rounding_mode(struct brw_codegen
*p
,
3438 enum brw_rnd_mode mode
)
3440 const unsigned bits
= mode
<< BRW_CR0_RND_MODE_SHIFT
;
3442 if (bits
!= BRW_CR0_RND_MODE_MASK
) {
3443 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3444 brw_imm_ud(~BRW_CR0_RND_MODE_MASK
));
3445 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3447 /* From the Skylake PRM, Volume 7, page 760:
3448 * "Implementation Restriction on Register Access: When the control
3449 * register is used as an explicit source and/or destination, hardware
3450 * does not ensure execution pipeline coherency. Software must set the
3451 * thread control field to ‘switch’ for an instruction that uses
3452 * control register as an explicit operand."
3454 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3458 brw_inst
*inst
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3460 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3461 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);