2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
90 const struct gen_device_info
*devinfo
= p
->devinfo
;
92 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
93 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
94 else if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
95 assert(dest
.nr
< 128);
97 gen7_convert_mrf_to_grf(p
, &dest
);
99 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
100 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
102 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
103 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
105 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
106 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
107 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
108 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
109 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
111 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
112 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
113 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
114 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
115 assert(dest
.writemask
!= 0);
117 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
118 * Although Dst.HorzStride is a don't care for Align16, HW needs
119 * this to be programmed as "01".
121 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
124 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
126 /* These are different sizes in align1 vs align16:
128 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
129 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
130 dest
.indirect_offset
);
131 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
132 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
133 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
135 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
136 dest
.indirect_offset
);
137 /* even ignored in da16, still need to set as '01' */
138 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
142 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
143 * or 16 (SIMD16), as that's normally correct. However, when dealing with
144 * small registers, it can be useful for us to automatically reduce it to
145 * match the register size.
147 if (p
->automatic_exec_sizes
) {
149 * In platforms that support fp64 we can emit instructions with a width
150 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
151 * these cases we need to make sure that these instructions have their
152 * exec sizes set properly when they are emitted and we can't rely on
153 * this code to fix it.
156 if (devinfo
->gen
>= 6)
157 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
159 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
162 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
167 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
169 const struct gen_device_info
*devinfo
= p
->devinfo
;
171 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
172 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
173 else if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
174 assert(reg
.nr
< 128);
176 gen7_convert_mrf_to_grf(p
, ®
);
178 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
179 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
180 /* Any source modifiers or regions will be ignored, since this just
181 * identifies the MRF/GRF to start reading the message contents from.
182 * Check for some likely failures.
186 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
189 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
190 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
191 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
192 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
194 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
195 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
196 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
197 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
198 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
199 reg
.type
== BRW_REGISTER_TYPE_Q
)
200 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
202 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
204 if (type_sz(reg
.type
) < 8) {
205 brw_inst_set_src1_reg_file(devinfo
, inst
,
206 BRW_ARCHITECTURE_REGISTER_FILE
);
207 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
208 brw_inst_src0_reg_hw_type(devinfo
, inst
));
211 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
212 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
213 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
214 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
216 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
219 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
221 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
222 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
224 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
228 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
229 if (reg
.width
== BRW_WIDTH_1
&&
230 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
231 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
232 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
233 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
235 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
236 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
237 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
240 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
241 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
242 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
243 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
244 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
245 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
246 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
247 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
249 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
250 /* This is an oddity of the fact we're using the same
251 * descriptions for registers in align_16 as align_1:
253 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
254 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
255 reg
.type
== BRW_REGISTER_TYPE_DF
&&
256 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
259 * "For Align16 access mode, only encodings of 0000 and 0011
260 * are allowed. Other codes are reserved."
262 * Presumably the DevSNB behavior applies to IVB as well.
264 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
266 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
274 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
276 const struct gen_device_info
*devinfo
= p
->devinfo
;
278 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
279 assert(reg
.nr
< 128);
281 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
283 * "Accumulator registers may be accessed explicitly as src0
286 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
287 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
289 gen7_convert_mrf_to_grf(p
, ®
);
290 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
292 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
293 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
294 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
296 /* Only src1 can be immediate in two-argument instructions.
298 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
300 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
301 /* two-argument instructions can only use 32-bit immediates */
302 assert(type_sz(reg
.type
) < 8);
303 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
305 /* This is a hardware restriction, which may or may not be lifted
308 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
309 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
311 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
312 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
313 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
315 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
318 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
319 if (reg
.width
== BRW_WIDTH_1
&&
320 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
321 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
322 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
323 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
325 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
326 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
327 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
330 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
331 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
332 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
333 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
334 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
335 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
336 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
337 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
339 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
340 /* This is an oddity of the fact we're using the same
341 * descriptions for registers in align_16 as align_1:
343 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
344 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
345 reg
.type
== BRW_REGISTER_TYPE_DF
&&
346 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
349 * "For Align16 access mode, only encodings of 0000 and 0011
350 * are allowed. Other codes are reserved."
352 * Presumably the DevSNB behavior applies to IVB as well.
354 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
356 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
363 * Specify the descriptor and extended descriptor immediate for a SEND(C)
364 * message instruction.
367 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
368 unsigned desc
, unsigned ex_desc
)
370 const struct gen_device_info
*devinfo
= p
->devinfo
;
371 brw_inst_set_src1_file_type(devinfo
, inst
,
372 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_D
);
373 brw_inst_set_send_desc(devinfo
, inst
, desc
);
374 if (devinfo
->gen
>= 9 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
375 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
))
376 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
379 static void brw_set_math_message( struct brw_codegen
*p
,
382 unsigned integer_type
,
386 const struct gen_device_info
*devinfo
= p
->devinfo
;
388 unsigned response_length
;
390 /* Infer message length from the function */
392 case BRW_MATH_FUNCTION_POW
:
393 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
394 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
395 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
403 /* Infer response length from the function */
405 case BRW_MATH_FUNCTION_SINCOS
:
406 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
414 brw_set_desc(p
, inst
, brw_message_desc(
415 devinfo
, msg_length
, response_length
, false));
417 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
418 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
419 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
420 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
421 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
422 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
423 brw_inst_set_saturate(devinfo
, inst
, 0);
427 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
430 unsigned response_length
,
433 const struct gen_device_info
*devinfo
= p
->devinfo
;
435 brw_set_desc(p
, insn
, brw_message_desc(
436 devinfo
, 1, response_length
, true));
438 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
439 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
440 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
441 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
442 /* The following fields are not used by FF_SYNC: */
443 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
444 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
445 brw_inst_set_urb_used(devinfo
, insn
, 0);
446 brw_inst_set_urb_complete(devinfo
, insn
, 0);
449 static void brw_set_urb_message( struct brw_codegen
*p
,
451 enum brw_urb_write_flags flags
,
453 unsigned response_length
,
455 unsigned swizzle_control
)
457 const struct gen_device_info
*devinfo
= p
->devinfo
;
459 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
460 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
461 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
463 brw_set_desc(p
, insn
, brw_message_desc(
464 devinfo
, msg_length
, response_length
, true));
466 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
467 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
469 if (flags
& BRW_URB_WRITE_OWORD
) {
470 assert(msg_length
== 2); /* header + one OWORD of data */
471 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
473 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
476 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
477 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
479 if (devinfo
->gen
< 8) {
480 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
483 if (devinfo
->gen
< 7) {
484 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
485 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
487 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
488 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
493 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
497 bool invalidate_after_read
,
499 unsigned addr_offset
,
504 const struct gen_device_info
*devinfo
= p
->devinfo
;
505 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
506 (devinfo
->gen
>= 8 && num_regs
== 8));
507 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
510 brw_set_desc(p
, inst
, brw_message_desc(
511 devinfo
, mlen
, rlen
, header_present
));
513 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
514 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
515 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
516 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
517 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
518 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
519 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
523 brw_inst_set_state(const struct gen_device_info
*devinfo
,
525 const struct brw_insn_state
*state
)
527 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
528 brw_inst_set_group(devinfo
, insn
, state
->group
);
529 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
530 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
531 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
532 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
533 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
534 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
536 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
537 state
->access_mode
== BRW_ALIGN_16
) {
538 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
539 if (devinfo
->gen
>= 7)
540 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
542 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
543 if (devinfo
->gen
>= 7)
544 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
547 if (devinfo
->gen
>= 6)
548 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
551 #define next_insn brw_next_insn
553 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
555 const struct gen_device_info
*devinfo
= p
->devinfo
;
558 if (p
->nr_insn
+ 1 > p
->store_size
) {
560 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
563 p
->next_insn_offset
+= 16;
564 insn
= &p
->store
[p
->nr_insn
++];
566 memset(insn
, 0, sizeof(*insn
));
567 brw_inst_set_opcode(devinfo
, insn
, opcode
);
569 /* Apply the default instruction state */
570 brw_inst_set_state(devinfo
, insn
, p
->current
);
576 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
577 struct brw_reg dest
, struct brw_reg src
)
579 brw_inst
*insn
= next_insn(p
, opcode
);
580 brw_set_dest(p
, insn
, dest
);
581 brw_set_src0(p
, insn
, src
);
586 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
587 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
589 /* 64-bit immediates are only supported on 1-src instructions */
590 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
591 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
593 brw_inst
*insn
= next_insn(p
, opcode
);
594 brw_set_dest(p
, insn
, dest
);
595 brw_set_src0(p
, insn
, src0
);
596 brw_set_src1(p
, insn
, src1
);
601 get_3src_subreg_nr(struct brw_reg reg
)
603 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
604 * use 32-bit units (components 0..7). Since they only support F/D/UD
605 * types, this doesn't lose any flexibility, but uses fewer bits.
607 return reg
.subnr
/ 4;
610 static enum gen10_align1_3src_vertical_stride
611 to_3src_align1_vstride(enum brw_vertical_stride vstride
)
614 case BRW_VERTICAL_STRIDE_0
:
615 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
616 case BRW_VERTICAL_STRIDE_2
:
617 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
618 case BRW_VERTICAL_STRIDE_4
:
619 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
620 case BRW_VERTICAL_STRIDE_8
:
621 case BRW_VERTICAL_STRIDE_16
:
622 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
624 unreachable("invalid vstride");
629 static enum gen10_align1_3src_src_horizontal_stride
630 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
633 case BRW_HORIZONTAL_STRIDE_0
:
634 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
635 case BRW_HORIZONTAL_STRIDE_1
:
636 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
637 case BRW_HORIZONTAL_STRIDE_2
:
638 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
639 case BRW_HORIZONTAL_STRIDE_4
:
640 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
642 unreachable("invalid hstride");
647 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
648 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
650 const struct gen_device_info
*devinfo
= p
->devinfo
;
651 brw_inst
*inst
= next_insn(p
, opcode
);
653 gen7_convert_mrf_to_grf(p
, &dest
);
655 assert(dest
.nr
< 128);
656 assert(src0
.nr
< 128);
657 assert(src1
.nr
< 128);
658 assert(src2
.nr
< 128);
659 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
660 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
661 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
662 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
664 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
665 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
666 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
668 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
669 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
670 BRW_ALIGN1_3SRC_ACCUMULATOR
);
671 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
673 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
674 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
675 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
677 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
679 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
681 if (brw_reg_type_is_floating_point(dest
.type
)) {
682 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
683 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
685 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
686 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
689 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
690 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
691 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
692 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
694 brw_inst_set_3src_a1_src0_vstride(devinfo
, inst
,
695 to_3src_align1_vstride(src0
.vstride
));
696 brw_inst_set_3src_a1_src1_vstride(devinfo
, inst
,
697 to_3src_align1_vstride(src1
.vstride
));
698 /* no vstride on src2 */
700 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
701 to_3src_align1_hstride(src0
.hstride
));
702 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
703 to_3src_align1_hstride(src1
.hstride
));
704 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
705 to_3src_align1_hstride(src2
.hstride
));
707 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
708 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
709 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
711 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
713 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
714 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
716 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
717 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
718 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
720 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
722 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
723 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
725 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
726 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
727 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
728 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
730 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
731 src0
.file
== BRW_IMMEDIATE_VALUE
||
732 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
733 src0
.type
== BRW_REGISTER_TYPE_NF
));
734 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
735 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
736 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
737 src2
.file
== BRW_IMMEDIATE_VALUE
);
739 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
740 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
741 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
742 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
743 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
744 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
745 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
746 BRW_ALIGN1_3SRC_ACCUMULATOR
);
747 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
748 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
749 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
750 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
752 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
753 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
754 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
755 dest
.type
== BRW_REGISTER_TYPE_DF
||
756 dest
.type
== BRW_REGISTER_TYPE_D
||
757 dest
.type
== BRW_REGISTER_TYPE_UD
);
758 if (devinfo
->gen
== 6) {
759 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
760 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
762 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
763 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
764 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
766 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
767 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
768 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
769 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
770 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
771 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
772 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
773 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
775 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
776 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
777 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
778 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
779 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
780 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
781 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
782 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
784 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
785 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
786 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
787 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
788 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
789 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
790 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
791 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
793 if (devinfo
->gen
>= 7) {
794 /* Set both the source and destination types based on dest.type,
795 * ignoring the source register types. The MAD and LRP emitters ensure
796 * that all four types are float. The BFE and BFI2 emitters, however,
797 * may send us mixed D and UD types and want us to ignore that and use
798 * the destination type.
800 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
801 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
809 /***********************************************************************
810 * Convenience routines.
813 brw_inst *brw_##OP(struct brw_codegen *p, \
814 struct brw_reg dest, \
815 struct brw_reg src0) \
817 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
821 brw_inst *brw_##OP(struct brw_codegen *p, \
822 struct brw_reg dest, \
823 struct brw_reg src0, \
824 struct brw_reg src1) \
826 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
830 brw_inst *brw_##OP(struct brw_codegen *p, \
831 struct brw_reg dest, \
832 struct brw_reg src0, \
833 struct brw_reg src1, \
834 struct brw_reg src2) \
836 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
840 brw_inst *brw_##OP(struct brw_codegen *p, \
841 struct brw_reg dest, \
842 struct brw_reg src0, \
843 struct brw_reg src1, \
844 struct brw_reg src2) \
846 assert(dest.type == BRW_REGISTER_TYPE_F || \
847 dest.type == BRW_REGISTER_TYPE_DF); \
848 if (dest.type == BRW_REGISTER_TYPE_F) { \
849 assert(src0.type == BRW_REGISTER_TYPE_F); \
850 assert(src1.type == BRW_REGISTER_TYPE_F); \
851 assert(src2.type == BRW_REGISTER_TYPE_F); \
852 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
853 assert(src0.type == BRW_REGISTER_TYPE_DF); \
854 assert(src1.type == BRW_REGISTER_TYPE_DF); \
855 assert(src2.type == BRW_REGISTER_TYPE_DF); \
857 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
860 /* Rounding operations (other than RNDD) require two instructions - the first
861 * stores a rounded value (possibly the wrong way) in the dest register, but
862 * also sets a per-channel "increment bit" in the flag register. A predicated
863 * add of 1.0 fixes dest to contain the desired result.
865 * Sandybridge and later appear to round correctly without an ADD.
868 void brw_##OP(struct brw_codegen *p, \
869 struct brw_reg dest, \
870 struct brw_reg src) \
872 const struct gen_device_info *devinfo = p->devinfo; \
873 brw_inst *rnd, *add; \
874 rnd = next_insn(p, BRW_OPCODE_##OP); \
875 brw_set_dest(p, rnd, dest); \
876 brw_set_src0(p, rnd, src); \
878 if (devinfo->gen < 6) { \
879 /* turn on round-increments */ \
880 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
881 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
882 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
922 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
924 const struct gen_device_info
*devinfo
= p
->devinfo
;
926 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
927 * To avoid the problems that causes, we use a <1,2,0> source region to read
928 * each element twice.
930 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
931 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
932 dest
.type
== BRW_REGISTER_TYPE_DF
&&
933 (src0
.type
== BRW_REGISTER_TYPE_F
||
934 src0
.type
== BRW_REGISTER_TYPE_D
||
935 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
936 !has_scalar_region(src0
)) {
937 assert(src0
.vstride
== BRW_VERTICAL_STRIDE_4
&&
938 src0
.width
== BRW_WIDTH_4
&&
939 src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
941 src0
.vstride
= BRW_VERTICAL_STRIDE_1
;
942 src0
.width
= BRW_WIDTH_2
;
943 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
946 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
950 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
951 struct brw_reg src0
, struct brw_reg src1
)
954 if (src0
.type
== BRW_REGISTER_TYPE_F
||
955 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
956 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
957 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
958 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
961 if (src1
.type
== BRW_REGISTER_TYPE_F
||
962 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
963 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
964 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
965 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
968 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
972 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
973 struct brw_reg src0
, struct brw_reg src1
)
975 assert(dest
.type
== src0
.type
);
976 assert(src0
.type
== src1
.type
);
978 case BRW_REGISTER_TYPE_B
:
979 case BRW_REGISTER_TYPE_UB
:
980 case BRW_REGISTER_TYPE_W
:
981 case BRW_REGISTER_TYPE_UW
:
982 case BRW_REGISTER_TYPE_D
:
983 case BRW_REGISTER_TYPE_UD
:
986 unreachable("Bad type for brw_AVG");
989 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
993 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
994 struct brw_reg src0
, struct brw_reg src1
)
997 if (src0
.type
== BRW_REGISTER_TYPE_D
||
998 src0
.type
== BRW_REGISTER_TYPE_UD
||
999 src1
.type
== BRW_REGISTER_TYPE_D
||
1000 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1001 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1004 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1005 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1006 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1007 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1008 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1011 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1012 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1013 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1014 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1015 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1018 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1019 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1020 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1021 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1023 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1027 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1028 struct brw_reg src0
, struct brw_reg src1
)
1030 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1031 src0
.width
= BRW_WIDTH_1
;
1032 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1033 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1037 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1038 struct brw_reg src0
, struct brw_reg src1
)
1040 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1041 src0
.width
= BRW_WIDTH_1
;
1042 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1043 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1044 src1
.width
= BRW_WIDTH_8
;
1045 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1046 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1050 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1052 const struct gen_device_info
*devinfo
= p
->devinfo
;
1053 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1054 /* The F32TO16 instruction doesn't support 32-bit destination types in
1055 * Align1 mode, and neither does the Gen8 implementation in terms of a
1056 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1057 * an undocumented feature.
1059 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1060 (!align16
|| devinfo
->gen
>= 8));
1064 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1066 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1067 dst
.type
== BRW_REGISTER_TYPE_W
||
1068 dst
.type
== BRW_REGISTER_TYPE_UW
||
1069 dst
.type
== BRW_REGISTER_TYPE_HF
);
1072 brw_push_insn_state(p
);
1074 if (needs_zero_fill
) {
1075 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1076 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1079 if (devinfo
->gen
>= 8) {
1080 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1082 assert(devinfo
->gen
== 7);
1083 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1086 if (needs_zero_fill
) {
1087 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1088 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1089 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1092 brw_pop_insn_state(p
);
1097 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1099 const struct gen_device_info
*devinfo
= p
->devinfo
;
1100 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1103 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1105 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1107 * Because this instruction does not have a 16-bit floating-point
1108 * type, the source data type must be Word (W). The destination type
1109 * must be F (Float).
1111 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1112 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1114 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1115 src
.type
== BRW_REGISTER_TYPE_UW
||
1116 src
.type
== BRW_REGISTER_TYPE_HF
);
1119 if (devinfo
->gen
>= 8) {
1120 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1122 assert(devinfo
->gen
== 7);
1123 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1128 void brw_NOP(struct brw_codegen
*p
)
1130 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1131 memset(insn
, 0, sizeof(*insn
));
1132 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1139 /***********************************************************************
1140 * Comparisons, if/else/endif
1144 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1145 unsigned predicate_control
)
1147 const struct gen_device_info
*devinfo
= p
->devinfo
;
1148 struct brw_reg ip
= brw_ip_reg();
1149 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1151 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1152 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1153 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1154 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1160 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1162 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1164 p
->if_stack_depth
++;
1165 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1166 p
->if_stack_array_size
*= 2;
1167 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1168 p
->if_stack_array_size
);
1173 pop_if_stack(struct brw_codegen
*p
)
1175 p
->if_stack_depth
--;
1176 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1180 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1182 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1183 p
->loop_stack_array_size
*= 2;
1184 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1185 p
->loop_stack_array_size
);
1186 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1187 p
->loop_stack_array_size
);
1190 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1191 p
->loop_stack_depth
++;
1192 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1196 get_inner_do_insn(struct brw_codegen
*p
)
1198 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1201 /* EU takes the value from the flag register and pushes it onto some
1202 * sort of a stack (presumably merging with any flag value already on
1203 * the stack). Within an if block, the flags at the top of the stack
1204 * control execution on each channel of the unit, eg. on each of the
1205 * 16 pixel values in our wm programs.
1207 * When the matching 'else' instruction is reached (presumably by
1208 * countdown of the instruction count patched in by our ELSE/ENDIF
1209 * functions), the relevant flags are inverted.
1211 * When the matching 'endif' instruction is reached, the flags are
1212 * popped off. If the stack is now empty, normal execution resumes.
1215 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1217 const struct gen_device_info
*devinfo
= p
->devinfo
;
1220 insn
= next_insn(p
, BRW_OPCODE_IF
);
1222 /* Override the defaults for this instruction:
1224 if (devinfo
->gen
< 6) {
1225 brw_set_dest(p
, insn
, brw_ip_reg());
1226 brw_set_src0(p
, insn
, brw_ip_reg());
1227 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1228 } else if (devinfo
->gen
== 6) {
1229 brw_set_dest(p
, insn
, brw_imm_w(0));
1230 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1231 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1232 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1233 } else if (devinfo
->gen
== 7) {
1234 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1235 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1236 brw_set_src1(p
, insn
, brw_imm_w(0));
1237 brw_inst_set_jip(devinfo
, insn
, 0);
1238 brw_inst_set_uip(devinfo
, insn
, 0);
1240 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1241 brw_set_src0(p
, insn
, brw_imm_d(0));
1242 brw_inst_set_jip(devinfo
, insn
, 0);
1243 brw_inst_set_uip(devinfo
, insn
, 0);
1246 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1247 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1248 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1249 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1250 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1251 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1253 push_if_stack(p
, insn
);
1254 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1258 /* This function is only used for gen6-style IF instructions with an
1259 * embedded comparison (conditional modifier). It is not used on gen7.
1262 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1263 struct brw_reg src0
, struct brw_reg src1
)
1265 const struct gen_device_info
*devinfo
= p
->devinfo
;
1268 insn
= next_insn(p
, BRW_OPCODE_IF
);
1270 brw_set_dest(p
, insn
, brw_imm_w(0));
1271 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1272 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1273 brw_set_src0(p
, insn
, src0
);
1274 brw_set_src1(p
, insn
, src1
);
1276 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1277 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1278 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1280 push_if_stack(p
, insn
);
1285 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1288 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1289 brw_inst
*if_inst
, brw_inst
*else_inst
)
1291 const struct gen_device_info
*devinfo
= p
->devinfo
;
1293 /* The next instruction (where the ENDIF would be, if it existed) */
1294 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1296 assert(p
->single_program_flow
);
1297 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1298 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1299 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1301 /* Convert IF to an ADD instruction that moves the instruction pointer
1302 * to the first instruction of the ELSE block. If there is no ELSE
1303 * block, point to where ENDIF would be. Reverse the predicate.
1305 * There's no need to execute an ENDIF since we don't need to do any
1306 * stack operations, and if we're currently executing, we just want to
1307 * continue normally.
1309 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1310 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1312 if (else_inst
!= NULL
) {
1313 /* Convert ELSE to an ADD instruction that points where the ENDIF
1316 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1318 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1319 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1321 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1326 * Patch IF and ELSE instructions with appropriate jump targets.
1329 patch_IF_ELSE(struct brw_codegen
*p
,
1330 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1332 const struct gen_device_info
*devinfo
= p
->devinfo
;
1334 /* We shouldn't be patching IF and ELSE instructions in single program flow
1335 * mode when gen < 6, because in single program flow mode on those
1336 * platforms, we convert flow control instructions to conditional ADDs that
1337 * operate on IP (see brw_ENDIF).
1339 * However, on Gen6, writing to IP doesn't work in single program flow mode
1340 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1341 * not be updated by non-flow control instructions."). And on later
1342 * platforms, there is no significant benefit to converting control flow
1343 * instructions to conditional ADDs. So we do patch IF and ELSE
1344 * instructions in single program flow mode on those platforms.
1346 if (devinfo
->gen
< 6)
1347 assert(!p
->single_program_flow
);
1349 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1350 assert(endif_inst
!= NULL
);
1351 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1353 unsigned br
= brw_jump_scale(devinfo
);
1355 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1356 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1358 if (else_inst
== NULL
) {
1359 /* Patch IF -> ENDIF */
1360 if (devinfo
->gen
< 6) {
1361 /* Turn it into an IFF, which means no mask stack operations for
1362 * all-false and jumping past the ENDIF.
1364 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1365 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1366 br
* (endif_inst
- if_inst
+ 1));
1367 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1368 } else if (devinfo
->gen
== 6) {
1369 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1370 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1372 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1373 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1376 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1378 /* Patch IF -> ELSE */
1379 if (devinfo
->gen
< 6) {
1380 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1381 br
* (else_inst
- if_inst
));
1382 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1383 } else if (devinfo
->gen
== 6) {
1384 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1385 br
* (else_inst
- if_inst
+ 1));
1388 /* Patch ELSE -> ENDIF */
1389 if (devinfo
->gen
< 6) {
1390 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1393 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1394 br
* (endif_inst
- else_inst
+ 1));
1395 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1396 } else if (devinfo
->gen
== 6) {
1397 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1398 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1399 br
* (endif_inst
- else_inst
));
1401 /* The IF instruction's JIP should point just past the ELSE */
1402 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1403 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1404 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1405 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1406 if (devinfo
->gen
>= 8) {
1407 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1408 * should point to ENDIF.
1410 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1417 brw_ELSE(struct brw_codegen
*p
)
1419 const struct gen_device_info
*devinfo
= p
->devinfo
;
1422 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1424 if (devinfo
->gen
< 6) {
1425 brw_set_dest(p
, insn
, brw_ip_reg());
1426 brw_set_src0(p
, insn
, brw_ip_reg());
1427 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1428 } else if (devinfo
->gen
== 6) {
1429 brw_set_dest(p
, insn
, brw_imm_w(0));
1430 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1431 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1432 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1433 } else if (devinfo
->gen
== 7) {
1434 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1435 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1436 brw_set_src1(p
, insn
, brw_imm_w(0));
1437 brw_inst_set_jip(devinfo
, insn
, 0);
1438 brw_inst_set_uip(devinfo
, insn
, 0);
1440 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1441 brw_set_src0(p
, insn
, brw_imm_d(0));
1442 brw_inst_set_jip(devinfo
, insn
, 0);
1443 brw_inst_set_uip(devinfo
, insn
, 0);
1446 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1447 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1448 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1449 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1451 push_if_stack(p
, insn
);
1455 brw_ENDIF(struct brw_codegen
*p
)
1457 const struct gen_device_info
*devinfo
= p
->devinfo
;
1458 brw_inst
*insn
= NULL
;
1459 brw_inst
*else_inst
= NULL
;
1460 brw_inst
*if_inst
= NULL
;
1462 bool emit_endif
= true;
1464 /* In single program flow mode, we can express IF and ELSE instructions
1465 * equivalently as ADD instructions that operate on IP. On platforms prior
1466 * to Gen6, flow control instructions cause an implied thread switch, so
1467 * this is a significant savings.
1469 * However, on Gen6, writing to IP doesn't work in single program flow mode
1470 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1471 * not be updated by non-flow control instructions."). And on later
1472 * platforms, there is no significant benefit to converting control flow
1473 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1476 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1480 * A single next_insn() may change the base address of instruction store
1481 * memory(p->store), so call it first before referencing the instruction
1482 * store pointer from an index
1485 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1487 /* Pop the IF and (optional) ELSE instructions from the stack */
1488 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1489 tmp
= pop_if_stack(p
);
1490 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1492 tmp
= pop_if_stack(p
);
1497 /* ENDIF is useless; don't bother emitting it. */
1498 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1502 if (devinfo
->gen
< 6) {
1503 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1504 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1505 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1506 } else if (devinfo
->gen
== 6) {
1507 brw_set_dest(p
, insn
, brw_imm_w(0));
1508 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1509 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1510 } else if (devinfo
->gen
== 7) {
1511 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1512 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1513 brw_set_src1(p
, insn
, brw_imm_w(0));
1515 brw_set_src0(p
, insn
, brw_imm_d(0));
1518 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1519 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1520 if (devinfo
->gen
< 6)
1521 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1523 /* Also pop item off the stack in the endif instruction: */
1524 if (devinfo
->gen
< 6) {
1525 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1526 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1527 } else if (devinfo
->gen
== 6) {
1528 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1530 brw_inst_set_jip(devinfo
, insn
, 2);
1532 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1536 brw_BREAK(struct brw_codegen
*p
)
1538 const struct gen_device_info
*devinfo
= p
->devinfo
;
1541 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1542 if (devinfo
->gen
>= 8) {
1543 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1544 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1545 } else if (devinfo
->gen
>= 6) {
1546 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1547 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1548 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1550 brw_set_dest(p
, insn
, brw_ip_reg());
1551 brw_set_src0(p
, insn
, brw_ip_reg());
1552 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1553 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1554 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1556 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1557 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1563 brw_CONT(struct brw_codegen
*p
)
1565 const struct gen_device_info
*devinfo
= p
->devinfo
;
1568 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1569 brw_set_dest(p
, insn
, brw_ip_reg());
1570 if (devinfo
->gen
>= 8) {
1571 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1573 brw_set_src0(p
, insn
, brw_ip_reg());
1574 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1577 if (devinfo
->gen
< 6) {
1578 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1579 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1581 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1582 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1587 gen6_HALT(struct brw_codegen
*p
)
1589 const struct gen_device_info
*devinfo
= p
->devinfo
;
1592 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1593 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1594 if (devinfo
->gen
>= 8) {
1595 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1597 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1598 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1601 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1602 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1608 * The DO/WHILE is just an unterminated loop -- break or continue are
1609 * used for control within the loop. We have a few ways they can be
1612 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1613 * jip and no DO instruction.
1615 * For non-uniform control flow pre-gen6, there's a DO instruction to
1616 * push the mask, and a WHILE to jump back, and BREAK to get out and
1619 * For gen6, there's no more mask stack, so no need for DO. WHILE
1620 * just points back to the first instruction of the loop.
1623 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1625 const struct gen_device_info
*devinfo
= p
->devinfo
;
1627 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1628 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1629 return &p
->store
[p
->nr_insn
];
1631 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1633 push_loop_stack(p
, insn
);
1635 /* Override the defaults for this instruction:
1637 brw_set_dest(p
, insn
, brw_null_reg());
1638 brw_set_src0(p
, insn
, brw_null_reg());
1639 brw_set_src1(p
, insn
, brw_null_reg());
1641 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1642 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1643 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1650 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1653 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1654 * nesting, since it can always just point to the end of the block/current loop.
1657 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1659 const struct gen_device_info
*devinfo
= p
->devinfo
;
1660 brw_inst
*do_inst
= get_inner_do_insn(p
);
1662 unsigned br
= brw_jump_scale(devinfo
);
1664 assert(devinfo
->gen
< 6);
1666 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1667 /* If the jump count is != 0, that means that this instruction has already
1668 * been patched because it's part of a loop inside of the one we're
1671 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1672 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1673 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1674 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1675 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1676 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1682 brw_WHILE(struct brw_codegen
*p
)
1684 const struct gen_device_info
*devinfo
= p
->devinfo
;
1685 brw_inst
*insn
, *do_insn
;
1686 unsigned br
= brw_jump_scale(devinfo
);
1688 if (devinfo
->gen
>= 6) {
1689 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1690 do_insn
= get_inner_do_insn(p
);
1692 if (devinfo
->gen
>= 8) {
1693 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1694 brw_set_src0(p
, insn
, brw_imm_d(0));
1695 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1696 } else if (devinfo
->gen
== 7) {
1697 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1698 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1699 brw_set_src1(p
, insn
, brw_imm_w(0));
1700 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1702 brw_set_dest(p
, insn
, brw_imm_w(0));
1703 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1704 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1705 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1708 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1711 if (p
->single_program_flow
) {
1712 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1713 do_insn
= get_inner_do_insn(p
);
1715 brw_set_dest(p
, insn
, brw_ip_reg());
1716 brw_set_src0(p
, insn
, brw_ip_reg());
1717 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1718 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1720 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1721 do_insn
= get_inner_do_insn(p
);
1723 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1725 brw_set_dest(p
, insn
, brw_ip_reg());
1726 brw_set_src0(p
, insn
, brw_ip_reg());
1727 brw_set_src1(p
, insn
, brw_imm_d(0));
1729 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1730 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1731 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1733 brw_patch_break_cont(p
, insn
);
1736 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1738 p
->loop_stack_depth
--;
1745 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1747 const struct gen_device_info
*devinfo
= p
->devinfo
;
1748 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1751 if (devinfo
->gen
>= 5)
1754 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1755 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1757 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1758 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1761 /* To integrate with the above, it makes sense that the comparison
1762 * instruction should populate the flag register. It might be simpler
1763 * just to use the flag reg for most WM tasks?
1765 void brw_CMP(struct brw_codegen
*p
,
1766 struct brw_reg dest
,
1767 unsigned conditional
,
1768 struct brw_reg src0
,
1769 struct brw_reg src1
)
1771 const struct gen_device_info
*devinfo
= p
->devinfo
;
1772 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1774 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1775 brw_set_dest(p
, insn
, dest
);
1776 brw_set_src0(p
, insn
, src0
);
1777 brw_set_src1(p
, insn
, src1
);
1779 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1781 * "Any CMP instruction with a null destination must use a {switch}."
1783 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1784 * mentioned on their work-arounds pages.
1786 if (devinfo
->gen
== 7) {
1787 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1788 dest
.nr
== BRW_ARF_NULL
) {
1789 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1794 /***********************************************************************
1795 * Helpers for the various SEND message types:
1798 /** Extended math function, float[8].
1800 void gen4_math(struct brw_codegen
*p
,
1801 struct brw_reg dest
,
1803 unsigned msg_reg_nr
,
1805 unsigned precision
)
1807 const struct gen_device_info
*devinfo
= p
->devinfo
;
1808 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1810 if (has_scalar_region(src
)) {
1811 data_type
= BRW_MATH_DATA_SCALAR
;
1813 data_type
= BRW_MATH_DATA_VECTOR
;
1816 assert(devinfo
->gen
< 6);
1818 /* Example code doesn't set predicate_control for send
1821 brw_inst_set_pred_control(devinfo
, insn
, 0);
1822 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1824 brw_set_dest(p
, insn
, dest
);
1825 brw_set_src0(p
, insn
, src
);
1826 brw_set_math_message(p
,
1829 src
.type
== BRW_REGISTER_TYPE_D
,
1834 void gen6_math(struct brw_codegen
*p
,
1835 struct brw_reg dest
,
1837 struct brw_reg src0
,
1838 struct brw_reg src1
)
1840 const struct gen_device_info
*devinfo
= p
->devinfo
;
1841 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1843 assert(devinfo
->gen
>= 6);
1845 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1846 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1848 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1849 if (devinfo
->gen
== 6) {
1850 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1851 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1854 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1855 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1856 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1857 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1858 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1859 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1860 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1862 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1863 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1866 /* Source modifiers are ignored for extended math instructions on Gen6. */
1867 if (devinfo
->gen
== 6) {
1868 assert(!src0
.negate
);
1870 assert(!src1
.negate
);
1874 brw_inst_set_math_function(devinfo
, insn
, function
);
1876 brw_set_dest(p
, insn
, dest
);
1877 brw_set_src0(p
, insn
, src0
);
1878 brw_set_src1(p
, insn
, src1
);
1882 * Return the right surface index to access the thread scratch space using
1883 * stateless dataport messages.
1886 brw_scratch_surface_idx(const struct brw_codegen
*p
)
1888 /* The scratch space is thread-local so IA coherency is unnecessary. */
1889 if (p
->devinfo
->gen
>= 8)
1890 return GEN8_BTI_STATELESS_NON_COHERENT
;
1892 return BRW_BTI_STATELESS
;
1896 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1897 * using a constant offset per channel.
1899 * The offset must be aligned to oword size (16 bytes). Used for
1900 * register spilling.
1902 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1907 const struct gen_device_info
*devinfo
= p
->devinfo
;
1908 const unsigned target_cache
=
1909 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
1910 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
1911 BRW_SFID_DATAPORT_WRITE
);
1914 if (devinfo
->gen
>= 6)
1917 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1919 const unsigned mlen
= 1 + num_regs
;
1921 /* Set up the message header. This is g0, with g0.2 filled with
1922 * the offset. We don't want to leave our offset around in g0 or
1923 * it'll screw up texture samples, so set it up inside the message
1927 brw_push_insn_state(p
);
1928 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
1929 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
1930 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
1932 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
1934 /* set message header global offset field (reg 0, element 2) */
1935 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
1937 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
1939 2), BRW_REGISTER_TYPE_UD
),
1940 brw_imm_ud(offset
));
1942 brw_pop_insn_state(p
);
1946 struct brw_reg dest
;
1947 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1948 int send_commit_msg
;
1949 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
1950 BRW_REGISTER_TYPE_UW
);
1952 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
1953 brw_inst_set_compression(devinfo
, insn
, false);
1955 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
1956 src_header
= vec16(src_header
);
1958 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1959 if (devinfo
->gen
< 6)
1960 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
1962 /* Until gen6, writes followed by reads from the same location
1963 * are not guaranteed to be ordered unless write_commit is set.
1964 * If set, then a no-op write is issued to the destination
1965 * register to set a dependency, and a read from the destination
1966 * can be used to ensure the ordering.
1968 * For gen6, only writes between different threads need ordering
1969 * protection. Our use of DP writes is all about register
1970 * spilling within a thread.
1972 if (devinfo
->gen
>= 6) {
1973 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
1974 send_commit_msg
= 0;
1977 send_commit_msg
= 1;
1980 brw_set_dest(p
, insn
, dest
);
1981 if (devinfo
->gen
>= 6) {
1982 brw_set_src0(p
, insn
, mrf
);
1984 brw_set_src0(p
, insn
, brw_null_reg());
1987 if (devinfo
->gen
>= 6)
1988 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
1990 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
1992 brw_set_desc(p
, insn
,
1993 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
1994 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
1995 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
1996 msg_type
, 0, /* not a render target */
2003 * Read a block of owords (half a GRF each) from the scratch buffer
2004 * using a constant index per channel.
2006 * Offset must be aligned to oword size (16 bytes). Used for register
2010 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2011 struct brw_reg dest
,
2016 const struct gen_device_info
*devinfo
= p
->devinfo
;
2018 if (devinfo
->gen
>= 6)
2021 if (p
->devinfo
->gen
>= 7) {
2022 /* On gen 7 and above, we no longer have message registers and we can
2023 * send from any register we want. By using the destination register
2024 * for the message, we guarantee that the implied message write won't
2025 * accidentally overwrite anything. This has been a problem because
2026 * the MRF registers and source for the final FB write are both fixed
2029 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2031 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2033 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2035 const unsigned rlen
= num_regs
;
2036 const unsigned target_cache
=
2037 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2038 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2039 BRW_SFID_DATAPORT_READ
);
2042 brw_push_insn_state(p
);
2043 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2044 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2045 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2047 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2049 /* set message header global offset field (reg 0, element 2) */
2050 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2051 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2053 brw_pop_insn_state(p
);
2057 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2059 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2060 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2061 brw_inst_set_compression(devinfo
, insn
, false);
2063 brw_set_dest(p
, insn
, dest
); /* UW? */
2064 if (devinfo
->gen
>= 6) {
2065 brw_set_src0(p
, insn
, mrf
);
2067 brw_set_src0(p
, insn
, brw_null_reg());
2068 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2071 brw_set_desc(p
, insn
,
2072 brw_message_desc(devinfo
, 1, rlen
, true) |
2073 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2074 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2075 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2076 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2081 gen7_block_read_scratch(struct brw_codegen
*p
,
2082 struct brw_reg dest
,
2086 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2087 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2089 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2091 /* The HW requires that the header is present; this is to get the g0.5
2094 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2096 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2097 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2098 * is 32 bytes, which happens to be the size of a register.
2101 assert(offset
< (1 << 12));
2103 gen7_set_dp_scratch_message(p
, insn
,
2104 false, /* scratch read */
2106 false, /* invalidate after read */
2109 1, /* mlen: just g0 */
2110 num_regs
, /* rlen */
2111 true); /* header present */
2115 * Read float[4] vectors from the data port constant cache.
2116 * Location (in buffer) should be a multiple of 16.
2117 * Used for fetching shader constants.
2119 void brw_oword_block_read(struct brw_codegen
*p
,
2120 struct brw_reg dest
,
2123 uint32_t bind_table_index
)
2125 const struct gen_device_info
*devinfo
= p
->devinfo
;
2126 const unsigned target_cache
=
2127 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2128 BRW_SFID_DATAPORT_READ
);
2129 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2131 /* On newer hardware, offset is in units of owords. */
2132 if (devinfo
->gen
>= 6)
2135 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2137 brw_push_insn_state(p
);
2138 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2139 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2140 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2142 brw_push_insn_state(p
);
2143 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2144 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2146 /* set message header global offset field (reg 0, element 2) */
2147 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2149 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2151 2), BRW_REGISTER_TYPE_UD
),
2152 brw_imm_ud(offset
));
2153 brw_pop_insn_state(p
);
2155 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2157 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2159 /* cast dest to a uword[8] vector */
2160 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2162 brw_set_dest(p
, insn
, dest
);
2163 if (devinfo
->gen
>= 6) {
2164 brw_set_src0(p
, insn
, mrf
);
2166 brw_set_src0(p
, insn
, brw_null_reg());
2167 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2170 brw_set_desc(p
, insn
,
2171 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2172 brw_dp_read_desc(devinfo
, bind_table_index
,
2173 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2174 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2175 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2177 brw_pop_insn_state(p
);
2181 brw_fb_WRITE(struct brw_codegen
*p
,
2182 struct brw_reg payload
,
2183 struct brw_reg implied_header
,
2184 unsigned msg_control
,
2185 unsigned binding_table_index
,
2186 unsigned msg_length
,
2187 unsigned response_length
,
2189 bool last_render_target
,
2190 bool header_present
)
2192 const struct gen_device_info
*devinfo
= p
->devinfo
;
2193 const unsigned target_cache
=
2194 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2195 BRW_SFID_DATAPORT_WRITE
);
2198 struct brw_reg dest
, src0
;
2200 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2201 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2203 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2205 if (devinfo
->gen
>= 6) {
2206 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2208 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2210 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2211 brw_inst_set_compression(devinfo
, insn
, false);
2213 if (devinfo
->gen
>= 6) {
2214 /* headerless version, just submit color payload */
2217 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2219 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2220 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2221 src0
= implied_header
;
2223 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2226 brw_set_dest(p
, insn
, dest
);
2227 brw_set_src0(p
, insn
, src0
);
2228 brw_set_desc(p
, insn
,
2229 brw_message_desc(devinfo
, msg_length
, response_length
,
2231 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2232 msg_type
, last_render_target
,
2233 0 /* send_commit_msg */));
2234 brw_inst_set_eot(devinfo
, insn
, eot
);
2240 gen9_fb_READ(struct brw_codegen
*p
,
2242 struct brw_reg payload
,
2243 unsigned binding_table_index
,
2244 unsigned msg_length
,
2245 unsigned response_length
,
2248 const struct gen_device_info
*devinfo
= p
->devinfo
;
2249 assert(devinfo
->gen
>= 9);
2250 const unsigned msg_subtype
=
2251 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2252 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2254 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2255 brw_set_dest(p
, insn
, dst
);
2256 brw_set_src0(p
, insn
, payload
);
2259 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2260 brw_dp_read_desc(devinfo
, binding_table_index
,
2261 per_sample
<< 5 | msg_subtype
,
2262 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2263 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2264 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2270 * Texture sample instruction.
2271 * Note: the msg_type plus msg_length values determine exactly what kind
2272 * of sampling operation is performed. See volume 4, page 161 of docs.
2274 void brw_SAMPLE(struct brw_codegen
*p
,
2275 struct brw_reg dest
,
2276 unsigned msg_reg_nr
,
2277 struct brw_reg src0
,
2278 unsigned binding_table_index
,
2281 unsigned response_length
,
2282 unsigned msg_length
,
2283 unsigned header_present
,
2285 unsigned return_format
)
2287 const struct gen_device_info
*devinfo
= p
->devinfo
;
2290 if (msg_reg_nr
!= -1)
2291 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2293 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2294 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2295 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2297 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2299 * "Instruction compression is not allowed for this instruction (that
2300 * is, send). The hardware behavior is undefined if this instruction is
2301 * set as compressed. However, compress control can be set to "SecHalf"
2302 * to affect the EMask generation."
2304 * No similar wording is found in later PRMs, but there are examples
2305 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2306 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2307 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2309 brw_inst_set_compression(devinfo
, insn
, false);
2311 if (devinfo
->gen
< 6)
2312 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2314 brw_set_dest(p
, insn
, dest
);
2315 brw_set_src0(p
, insn
, src0
);
2316 brw_set_desc(p
, insn
,
2317 brw_message_desc(devinfo
, msg_length
, response_length
,
2319 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2320 msg_type
, simd_mode
, return_format
));
2323 /* Adjust the message header's sampler state pointer to
2324 * select the correct group of 16 samplers.
2326 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2327 struct brw_reg header
,
2328 struct brw_reg sampler_index
)
2330 /* The "Sampler Index" field can only store values between 0 and 15.
2331 * However, we can add an offset to the "Sampler State Pointer"
2332 * field, effectively selecting a different set of 16 samplers.
2334 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2335 * offset, and each sampler state is only 16-bytes, so we can't
2336 * exclusively use the offset - we have to use both.
2339 const struct gen_device_info
*devinfo
= p
->devinfo
;
2341 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2342 const int sampler_state_size
= 16; /* 16 bytes */
2343 uint32_t sampler
= sampler_index
.ud
;
2345 if (sampler
>= 16) {
2346 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2348 get_element_ud(header
, 3),
2349 get_element_ud(brw_vec8_grf(0, 0), 3),
2350 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2353 /* Non-const sampler array indexing case */
2354 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2358 struct brw_reg temp
= get_element_ud(header
, 3);
2360 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2361 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2363 get_element_ud(header
, 3),
2364 get_element_ud(brw_vec8_grf(0, 0), 3),
2369 /* All these variables are pretty confusing - we might be better off
2370 * using bitmasks and macros for this, in the old style. Or perhaps
2371 * just having the caller instantiate the fields in dword3 itself.
2373 void brw_urb_WRITE(struct brw_codegen
*p
,
2374 struct brw_reg dest
,
2375 unsigned msg_reg_nr
,
2376 struct brw_reg src0
,
2377 enum brw_urb_write_flags flags
,
2378 unsigned msg_length
,
2379 unsigned response_length
,
2383 const struct gen_device_info
*devinfo
= p
->devinfo
;
2386 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2388 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2389 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2390 brw_push_insn_state(p
);
2391 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2392 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2393 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2394 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2395 BRW_REGISTER_TYPE_UD
),
2396 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2397 brw_imm_ud(0xff00));
2398 brw_pop_insn_state(p
);
2401 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2403 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2405 brw_set_dest(p
, insn
, dest
);
2406 brw_set_src0(p
, insn
, src0
);
2407 brw_set_src1(p
, insn
, brw_imm_d(0));
2409 if (devinfo
->gen
< 6)
2410 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2412 brw_set_urb_message(p
,
2422 brw_send_indirect_message(struct brw_codegen
*p
,
2425 struct brw_reg payload
,
2426 struct brw_reg desc
,
2429 const struct gen_device_info
*devinfo
= p
->devinfo
;
2430 struct brw_inst
*send
;
2433 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2435 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2437 /* We hold on to the setup instruction (the SEND in the direct case, the OR
2438 * in the indirect case) by its index in the instruction store. The
2439 * pointer returned by next_insn() may become invalid if emitting the SEND
2440 * in the indirect case reallocs the store.
2443 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2445 send
= next_insn(p
, BRW_OPCODE_SEND
);
2446 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2449 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2451 brw_push_insn_state(p
);
2452 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2453 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2454 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2455 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2457 /* Load the indirect descriptor to an address register using OR so the
2458 * caller can specify additional descriptor bits with the usual
2459 * brw_set_*_message() helper functions.
2462 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2464 brw_pop_insn_state(p
);
2466 send
= next_insn(p
, BRW_OPCODE_SEND
);
2467 brw_set_src1(p
, send
, addr
);
2470 if (dst
.width
< BRW_EXECUTE_8
)
2471 brw_inst_set_exec_size(devinfo
, send
, dst
.width
);
2473 brw_set_dest(p
, send
, dst
);
2474 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2475 brw_inst_set_sfid(devinfo
, send
, sfid
);
2477 return &p
->store
[setup
];
2480 static struct brw_inst
*
2481 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2484 struct brw_reg payload
,
2485 struct brw_reg surface
,
2486 unsigned message_len
,
2487 unsigned response_len
,
2488 bool header_present
)
2490 const struct gen_device_info
*devinfo
= p
->devinfo
;
2491 struct brw_inst
*insn
;
2493 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2494 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2496 brw_push_insn_state(p
);
2497 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2498 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2499 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2500 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2502 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2503 * some surface array is accessed out of bounds.
2505 insn
= brw_AND(p
, addr
,
2506 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2507 BRW_GET_SWZ(surface
.swizzle
, 0)),
2510 brw_pop_insn_state(p
);
2515 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, 0);
2516 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2517 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2518 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2524 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2525 brw_inst
*insn
, int while_offset
, int start_offset
)
2527 int scale
= 16 / brw_jump_scale(devinfo
);
2528 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2529 : brw_inst_jip(devinfo
, insn
);
2531 return while_offset
+ jip
* scale
<= start_offset
;
2536 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2539 void *store
= p
->store
;
2540 const struct gen_device_info
*devinfo
= p
->devinfo
;
2544 for (offset
= next_offset(devinfo
, store
, start_offset
);
2545 offset
< p
->next_insn_offset
;
2546 offset
= next_offset(devinfo
, store
, offset
)) {
2547 brw_inst
*insn
= store
+ offset
;
2549 switch (brw_inst_opcode(devinfo
, insn
)) {
2553 case BRW_OPCODE_ENDIF
:
2558 case BRW_OPCODE_WHILE
:
2559 /* If the while doesn't jump before our instruction, it's the end
2560 * of a sibling do...while loop. Ignore it.
2562 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2565 case BRW_OPCODE_ELSE
:
2566 case BRW_OPCODE_HALT
:
2575 /* There is no DO instruction on gen6, so to find the end of the loop
2576 * we have to see if the loop is jumping back before our start
2580 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2582 const struct gen_device_info
*devinfo
= p
->devinfo
;
2584 void *store
= p
->store
;
2586 assert(devinfo
->gen
>= 6);
2588 /* Always start after the instruction (such as a WHILE) we're trying to fix
2591 for (offset
= next_offset(devinfo
, store
, start_offset
);
2592 offset
< p
->next_insn_offset
;
2593 offset
= next_offset(devinfo
, store
, offset
)) {
2594 brw_inst
*insn
= store
+ offset
;
2596 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2597 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2601 assert(!"not reached");
2602 return start_offset
;
2605 /* After program generation, go back and update the UIP and JIP of
2606 * BREAK, CONT, and HALT instructions to their correct locations.
2609 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2611 const struct gen_device_info
*devinfo
= p
->devinfo
;
2613 int br
= brw_jump_scale(devinfo
);
2614 int scale
= 16 / br
;
2615 void *store
= p
->store
;
2617 if (devinfo
->gen
< 6)
2620 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2621 brw_inst
*insn
= store
+ offset
;
2622 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2624 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2625 switch (brw_inst_opcode(devinfo
, insn
)) {
2626 case BRW_OPCODE_BREAK
:
2627 assert(block_end_offset
!= 0);
2628 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2629 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2630 brw_inst_set_uip(devinfo
, insn
,
2631 (brw_find_loop_end(p
, offset
) - offset
+
2632 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2634 case BRW_OPCODE_CONTINUE
:
2635 assert(block_end_offset
!= 0);
2636 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2637 brw_inst_set_uip(devinfo
, insn
,
2638 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2640 assert(brw_inst_uip(devinfo
, insn
) != 0);
2641 assert(brw_inst_jip(devinfo
, insn
) != 0);
2644 case BRW_OPCODE_ENDIF
: {
2645 int32_t jump
= (block_end_offset
== 0) ?
2646 1 * br
: (block_end_offset
- offset
) / scale
;
2647 if (devinfo
->gen
>= 7)
2648 brw_inst_set_jip(devinfo
, insn
, jump
);
2650 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2654 case BRW_OPCODE_HALT
:
2655 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2657 * "In case of the halt instruction not inside any conditional
2658 * code block, the value of <JIP> and <UIP> should be the
2659 * same. In case of the halt instruction inside conditional code
2660 * block, the <UIP> should be the end of the program, and the
2661 * <JIP> should be end of the most inner conditional code block."
2663 * The uip will have already been set by whoever set up the
2666 if (block_end_offset
== 0) {
2667 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2669 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2671 assert(brw_inst_uip(devinfo
, insn
) != 0);
2672 assert(brw_inst_jip(devinfo
, insn
) != 0);
2678 void brw_ff_sync(struct brw_codegen
*p
,
2679 struct brw_reg dest
,
2680 unsigned msg_reg_nr
,
2681 struct brw_reg src0
,
2683 unsigned response_length
,
2686 const struct gen_device_info
*devinfo
= p
->devinfo
;
2689 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2691 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2692 brw_set_dest(p
, insn
, dest
);
2693 brw_set_src0(p
, insn
, src0
);
2694 brw_set_src1(p
, insn
, brw_imm_d(0));
2696 if (devinfo
->gen
< 6)
2697 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2699 brw_set_ff_sync_message(p
,
2707 * Emit the SEND instruction necessary to generate stream output data on Gen6
2708 * (for transform feedback).
2710 * If send_commit_msg is true, this is the last piece of stream output data
2711 * from this thread, so send the data as a committed write. According to the
2712 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2714 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2715 * writes are complete by sending the final write as a committed write."
2718 brw_svb_write(struct brw_codegen
*p
,
2719 struct brw_reg dest
,
2720 unsigned msg_reg_nr
,
2721 struct brw_reg src0
,
2722 unsigned binding_table_index
,
2723 bool send_commit_msg
)
2725 const struct gen_device_info
*devinfo
= p
->devinfo
;
2726 const unsigned target_cache
=
2727 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2728 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2729 BRW_SFID_DATAPORT_WRITE
);
2732 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2734 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2735 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2736 brw_set_dest(p
, insn
, dest
);
2737 brw_set_src0(p
, insn
, src0
);
2738 brw_set_desc(p
, insn
,
2739 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
2740 brw_dp_write_desc(devinfo
, binding_table_index
,
2741 0, /* msg_control: ignored */
2742 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2743 0, /* last_render_target: ignored */
2744 send_commit_msg
)); /* send_commit_msg */
2748 brw_surface_payload_size(struct brw_codegen
*p
,
2749 unsigned num_channels
,
2753 if (has_simd4x2
&& brw_get_default_access_mode(p
) == BRW_ALIGN_16
)
2755 else if (has_simd16
&& brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2756 return 2 * num_channels
;
2758 return num_channels
;
2762 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2765 bool response_expected
)
2767 const struct gen_device_info
*devinfo
= p
->devinfo
;
2768 unsigned msg_control
=
2769 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2770 (response_expected
? 1 << 5 : 0); /* Return data expected */
2772 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2773 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
2774 if (brw_get_default_exec_size(p
) != BRW_EXECUTE_16
)
2775 msg_control
|= 1 << 4; /* SIMD8 mode */
2777 brw_inst_set_dp_msg_type(devinfo
, insn
,
2778 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2780 brw_inst_set_dp_msg_type(devinfo
, insn
,
2781 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2784 brw_inst_set_dp_msg_type(devinfo
, insn
,
2785 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2787 if (brw_get_default_exec_size(p
) != BRW_EXECUTE_16
)
2788 msg_control
|= 1 << 4; /* SIMD8 mode */
2791 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2795 brw_untyped_atomic(struct brw_codegen
*p
,
2797 struct brw_reg payload
,
2798 struct brw_reg surface
,
2800 unsigned msg_length
,
2801 bool response_expected
,
2802 bool header_present
)
2804 const struct gen_device_info
*devinfo
= p
->devinfo
;
2805 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2806 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2807 GEN7_SFID_DATAPORT_DATA_CACHE
);
2808 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2809 /* Mask out unused components -- This is especially important in Align16
2810 * mode on generations that don't have native support for SIMD4x2 atomics,
2811 * because unused but enabled components will cause the dataport to perform
2812 * additional atomic operations on the addresses that happen to be in the
2813 * uninitialized Y, Z and W coordinates of the payload.
2815 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2816 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2817 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2818 brw_surface_payload_size(p
, response_expected
,
2819 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2822 brw_set_dp_untyped_atomic_message(
2823 p
, insn
, atomic_op
, response_expected
);
2827 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
2828 struct brw_inst
*insn
,
2829 unsigned num_channels
)
2831 const struct gen_device_info
*devinfo
= p
->devinfo
;
2832 /* Set mask of 32-bit channels to drop. */
2833 unsigned msg_control
= 0xf & (0xf << num_channels
);
2835 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
2836 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2837 msg_control
|= 1 << 4; /* SIMD16 mode */
2839 msg_control
|= 2 << 4; /* SIMD8 mode */
2842 brw_inst_set_dp_msg_type(devinfo
, insn
,
2843 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2844 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2845 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
2846 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2850 brw_untyped_surface_read(struct brw_codegen
*p
,
2852 struct brw_reg payload
,
2853 struct brw_reg surface
,
2854 unsigned msg_length
,
2855 unsigned num_channels
)
2857 const struct gen_device_info
*devinfo
= p
->devinfo
;
2858 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2859 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2860 GEN7_SFID_DATAPORT_DATA_CACHE
);
2861 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2862 p
, sfid
, dst
, payload
, surface
, msg_length
,
2863 brw_surface_payload_size(p
, num_channels
, true, true),
2866 brw_set_dp_untyped_surface_read_message(
2867 p
, insn
, num_channels
);
2871 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
2872 struct brw_inst
*insn
,
2873 unsigned num_channels
)
2875 const struct gen_device_info
*devinfo
= p
->devinfo
;
2876 /* Set mask of 32-bit channels to drop. */
2877 unsigned msg_control
= 0xf & (0xf << num_channels
);
2879 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
2880 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2881 msg_control
|= 1 << 4; /* SIMD16 mode */
2883 msg_control
|= 2 << 4; /* SIMD8 mode */
2885 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
2886 msg_control
|= 0 << 4; /* SIMD4x2 mode */
2888 msg_control
|= 2 << 4; /* SIMD8 mode */
2891 brw_inst_set_dp_msg_type(devinfo
, insn
,
2892 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2893 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
2894 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
2895 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2899 brw_untyped_surface_write(struct brw_codegen
*p
,
2900 struct brw_reg payload
,
2901 struct brw_reg surface
,
2902 unsigned msg_length
,
2903 unsigned num_channels
,
2904 bool header_present
)
2906 const struct gen_device_info
*devinfo
= p
->devinfo
;
2907 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2908 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2909 GEN7_SFID_DATAPORT_DATA_CACHE
);
2910 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2911 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2912 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
2913 WRITEMASK_X
: WRITEMASK_XYZW
;
2914 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2915 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2916 payload
, surface
, msg_length
, 0, header_present
);
2918 brw_set_dp_untyped_surface_write_message(
2919 p
, insn
, num_channels
);
2923 brw_byte_scattered_data_element_from_bit_size(unsigned bit_size
)
2927 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE
;
2929 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD
;
2931 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD
;
2933 unreachable("Unsupported bit_size for byte scattered messages");
2939 brw_byte_scattered_read(struct brw_codegen
*p
,
2941 struct brw_reg payload
,
2942 struct brw_reg surface
,
2943 unsigned msg_length
,
2946 const struct gen_device_info
*devinfo
= p
->devinfo
;
2947 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
2948 assert(brw_get_default_access_mode(p
) == BRW_ALIGN_1
);
2949 const unsigned sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
2951 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2952 p
, sfid
, dst
, payload
, surface
, msg_length
,
2953 brw_surface_payload_size(p
, 1, true, true),
2956 unsigned msg_control
=
2957 brw_byte_scattered_data_element_from_bit_size(bit_size
) << 2;
2959 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2960 msg_control
|= 1; /* SIMD16 mode */
2962 msg_control
|= 0; /* SIMD8 mode */
2964 brw_inst_set_dp_msg_type(devinfo
, insn
,
2965 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ
);
2966 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2970 brw_byte_scattered_write(struct brw_codegen
*p
,
2971 struct brw_reg payload
,
2972 struct brw_reg surface
,
2973 unsigned msg_length
,
2975 bool header_present
)
2977 const struct gen_device_info
*devinfo
= p
->devinfo
;
2978 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
2979 assert(brw_get_default_access_mode(p
) == BRW_ALIGN_1
);
2980 const unsigned sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
2982 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2983 p
, sfid
, brw_writemask(brw_null_reg(), WRITEMASK_XYZW
),
2984 payload
, surface
, msg_length
, 0, header_present
);
2986 unsigned msg_control
=
2987 brw_byte_scattered_data_element_from_bit_size(bit_size
) << 2;
2989 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2994 brw_inst_set_dp_msg_type(devinfo
, insn
,
2995 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE
);
2996 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3000 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
3001 struct brw_inst
*insn
,
3003 bool response_expected
)
3005 const struct gen_device_info
*devinfo
= p
->devinfo
;
3006 unsigned msg_control
=
3007 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3008 (response_expected
? 1 << 5 : 0); /* Return data expected */
3010 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3011 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3012 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3013 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3015 brw_inst_set_dp_msg_type(devinfo
, insn
,
3016 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
3018 brw_inst_set_dp_msg_type(devinfo
, insn
,
3019 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
3023 brw_inst_set_dp_msg_type(devinfo
, insn
,
3024 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
3026 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3027 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3030 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3034 brw_typed_atomic(struct brw_codegen
*p
,
3036 struct brw_reg payload
,
3037 struct brw_reg surface
,
3039 unsigned msg_length
,
3040 bool response_expected
,
3041 bool header_present
) {
3042 const struct gen_device_info
*devinfo
= p
->devinfo
;
3043 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3044 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3045 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3046 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3047 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3048 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3049 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3050 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
3051 brw_surface_payload_size(p
, response_expected
,
3052 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3055 brw_set_dp_typed_atomic_message(
3056 p
, insn
, atomic_op
, response_expected
);
3060 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3061 struct brw_inst
*insn
,
3062 unsigned num_channels
)
3064 const struct gen_device_info
*devinfo
= p
->devinfo
;
3065 /* Set mask of unused channels. */
3066 unsigned msg_control
= 0xf & (0xf << num_channels
);
3068 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3069 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3070 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3071 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3073 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3076 brw_inst_set_dp_msg_type(devinfo
, insn
,
3077 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3079 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3080 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3081 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3084 brw_inst_set_dp_msg_type(devinfo
, insn
,
3085 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3088 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3092 brw_typed_surface_read(struct brw_codegen
*p
,
3094 struct brw_reg payload
,
3095 struct brw_reg surface
,
3096 unsigned msg_length
,
3097 unsigned num_channels
,
3098 bool header_present
)
3100 const struct gen_device_info
*devinfo
= p
->devinfo
;
3101 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3102 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3103 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3104 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3105 p
, sfid
, dst
, payload
, surface
, msg_length
,
3106 brw_surface_payload_size(p
, num_channels
,
3107 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3110 brw_set_dp_typed_surface_read_message(
3111 p
, insn
, num_channels
);
3115 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3116 struct brw_inst
*insn
,
3117 unsigned num_channels
)
3119 const struct gen_device_info
*devinfo
= p
->devinfo
;
3120 /* Set mask of unused channels. */
3121 unsigned msg_control
= 0xf & (0xf << num_channels
);
3123 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3124 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3125 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3126 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3128 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3131 brw_inst_set_dp_msg_type(devinfo
, insn
,
3132 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3135 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3136 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3137 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3140 brw_inst_set_dp_msg_type(devinfo
, insn
,
3141 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3144 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3148 brw_typed_surface_write(struct brw_codegen
*p
,
3149 struct brw_reg payload
,
3150 struct brw_reg surface
,
3151 unsigned msg_length
,
3152 unsigned num_channels
,
3153 bool header_present
)
3155 const struct gen_device_info
*devinfo
= p
->devinfo
;
3156 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3157 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3158 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3159 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3160 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3161 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3162 WRITEMASK_X
: WRITEMASK_XYZW
);
3163 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3164 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3165 payload
, surface
, msg_length
, 0, header_present
);
3167 brw_set_dp_typed_surface_write_message(
3168 p
, insn
, num_channels
);
3172 brw_set_memory_fence_message(struct brw_codegen
*p
,
3173 struct brw_inst
*insn
,
3174 enum brw_message_target sfid
,
3177 const struct gen_device_info
*devinfo
= p
->devinfo
;
3179 brw_set_desc(p
, insn
, brw_message_desc(
3180 devinfo
, 1, (commit_enable
? 1 : 0), true));
3182 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3185 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3186 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3188 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3189 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3192 unreachable("Not reached");
3196 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3200 brw_memory_fence(struct brw_codegen
*p
,
3202 enum opcode send_op
)
3204 const struct gen_device_info
*devinfo
= p
->devinfo
;
3205 const bool commit_enable
=
3206 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3207 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3208 struct brw_inst
*insn
;
3210 brw_push_insn_state(p
);
3211 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3212 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3215 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3216 * message doesn't write anything back.
3218 insn
= next_insn(p
, send_op
);
3219 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3220 brw_set_dest(p
, insn
, dst
);
3221 brw_set_src0(p
, insn
, dst
);
3222 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3225 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3226 /* IVB does typed surface access through the render cache, so we need to
3227 * flush it too. Use a different register so both flushes can be
3228 * pipelined by the hardware.
3230 insn
= next_insn(p
, send_op
);
3231 brw_set_dest(p
, insn
, offset(dst
, 1));
3232 brw_set_src0(p
, insn
, offset(dst
, 1));
3233 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3236 /* Now write the response of the second message into the response of the
3237 * first to trigger a pipeline stall -- This way future render and data
3238 * cache messages will be properly ordered with respect to past data and
3239 * render cache messages.
3241 brw_MOV(p
, dst
, offset(dst
, 1));
3244 brw_pop_insn_state(p
);
3248 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3249 struct brw_reg dest
,
3253 struct brw_reg data
,
3254 unsigned msg_length
,
3255 unsigned response_length
)
3257 const struct gen_device_info
*devinfo
= p
->devinfo
;
3258 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3259 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3260 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3261 const unsigned desc
=
3262 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3263 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3266 /* brw_send_indirect_message will automatically use a direct send message
3267 * if data is actually immediate.
3269 brw_send_indirect_message(p
,
3270 GEN7_SFID_PIXEL_INTERPOLATOR
,
3278 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3279 struct brw_reg mask
)
3281 const struct gen_device_info
*devinfo
= p
->devinfo
;
3282 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3283 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3286 assert(devinfo
->gen
>= 7);
3287 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3289 brw_push_insn_state(p
);
3291 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3292 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3294 if (devinfo
->gen
>= 8) {
3295 /* Getting the first active channel index is easy on Gen8: Just find
3296 * the first bit set in the execution mask. The register exists on
3297 * HSW already but it reads back as all ones when the current
3298 * instruction has execution masking disabled, so it's kind of
3301 struct brw_reg exec_mask
=
3302 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3304 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3305 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3306 /* Unfortunately, ce0 does not take into account the thread
3307 * dispatch mask, which may be a problem in cases where it's not
3308 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3309 * some n). Combine ce0 with the given dispatch (or vector) mask
3310 * to mask off those channels which were never dispatched by the
3313 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3314 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3315 exec_mask
= vec1(dst
);
3318 /* Quarter control has the effect of magically shifting the value of
3319 * ce0 so you'll get the first active channel relative to the
3320 * specified quarter control as result.
3322 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3324 const struct brw_reg flag
= brw_flag_reg(p
->current
->flag_subreg
/ 2,
3325 p
->current
->flag_subreg
% 2);
3327 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3328 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3330 /* Run enough instructions returning zero with execution masking and
3331 * a conditional modifier enabled in order to get the full execution
3332 * mask in f1.0. We could use a single 32-wide move here if it
3333 * weren't because of the hardware bug that causes channel enables to
3334 * be applied incorrectly to the second half of 32-wide instructions
3337 const unsigned lower_size
= MIN2(16, exec_size
);
3338 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3339 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3341 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3342 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3343 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3344 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3347 /* Find the first bit set in the exec_size-wide portion of the flag
3348 * register that was updated by the last sequence of MOV
3351 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3352 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3353 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3356 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3358 if (devinfo
->gen
>= 8 &&
3359 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3360 /* In SIMD4x2 mode the first active channel index is just the
3361 * negation of the first bit of the mask register. Note that ce0
3362 * doesn't take into account the dispatch mask, so the Gen7 path
3363 * should be used instead unless you have the guarantee that the
3364 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3367 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3368 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3372 /* Overwrite the destination without and with execution masking to
3373 * find out which of the channels is active.
3375 brw_push_insn_state(p
);
3376 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3377 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3380 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3382 brw_pop_insn_state(p
);
3383 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3387 brw_pop_insn_state(p
);
3391 brw_broadcast(struct brw_codegen
*p
,
3396 const struct gen_device_info
*devinfo
= p
->devinfo
;
3397 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3400 brw_push_insn_state(p
);
3401 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3402 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3404 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3405 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3406 assert(!src
.abs
&& !src
.negate
);
3407 assert(src
.type
== dst
.type
);
3409 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3410 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3411 /* Trivial, the source is already uniform or the index is a constant.
3412 * We will typically not get here if the optimizer is doing its job, but
3413 * asserting would be mean.
3415 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3417 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3418 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3420 /* From the Haswell PRM section "Register Region Restrictions":
3422 * "The lower bits of the AddressImmediate must not overflow to
3423 * change the register address. The lower 5 bits of Address
3424 * Immediate when added to lower 5 bits of address register gives
3425 * the sub-register offset. The upper bits of Address Immediate
3426 * when added to upper bits of address register gives the register
3427 * address. Any overflow from sub-register offset is dropped."
3429 * Fortunately, for broadcast, we never have a sub-register offset so
3430 * this isn't an issue.
3432 assert(src
.subnr
== 0);
3435 const struct brw_reg addr
=
3436 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3437 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3438 /* Limit in bytes of the signed indirect addressing immediate. */
3439 const unsigned limit
= 512;
3441 brw_push_insn_state(p
);
3442 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3443 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3445 /* Take into account the component size and horizontal stride. */
3446 assert(src
.vstride
== src
.hstride
+ src
.width
);
3447 brw_SHL(p
, addr
, vec1(idx
),
3448 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3451 /* We can only address up to limit bytes using the indirect
3452 * addressing immediate, account for the difference if the source
3453 * register is above this limit.
3455 if (offset
>= limit
) {
3456 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3457 offset
= offset
% limit
;
3460 brw_pop_insn_state(p
);
3462 /* Use indirect addressing to fetch the specified component. */
3463 if (type_sz(src
.type
) > 4 &&
3464 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3465 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3467 * "When source or destination datatype is 64b or operation is
3468 * integer DWord multiply, indirect addressing must not be
3471 * To work around both of this issue, we do two integer MOVs
3472 * insead of one 64-bit MOV. Because no double value should ever
3473 * cross a register boundary, it's safe to use the immediate
3474 * offset in the indirect here to handle adding 4 bytes to the
3475 * offset and avoid the extra ADD to the register file.
3477 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3478 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3479 BRW_REGISTER_TYPE_D
));
3480 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3481 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3482 BRW_REGISTER_TYPE_D
));
3485 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3488 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3489 * to all bits of a flag register,
3493 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3494 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3495 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3496 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3498 /* and use predicated SEL to pick the right channel. */
3499 inst
= brw_SEL(p
, dst
,
3500 stride(suboffset(src
, 4), 4, 4, 1),
3501 stride(src
, 4, 4, 1));
3502 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3503 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3507 brw_pop_insn_state(p
);
3511 * This instruction is generated as a single-channel align1 instruction by
3512 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3514 * We can't use the typed atomic op in the FS because that has the execution
3515 * mask ANDed with the pixel mask, but we just want to write the one dword for
3518 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3519 * one u32. So we use the same untyped atomic write message as the pixel
3522 * The untyped atomic operation requires a BUFFER surface type with RAW
3523 * format, and is only accessible through the legacy DATA_CACHE dataport
3526 void brw_shader_time_add(struct brw_codegen
*p
,
3527 struct brw_reg payload
,
3528 uint32_t surf_index
)
3530 const struct gen_device_info
*devinfo
= p
->devinfo
;
3531 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3532 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3533 GEN7_SFID_DATAPORT_DATA_CACHE
);
3534 assert(devinfo
->gen
>= 7);
3536 brw_push_insn_state(p
);
3537 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3538 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3539 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3540 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3542 /* We use brw_vec1_reg and unmasked because we want to increment the given
3545 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3547 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3549 brw_set_src1(p
, send
, brw_imm_ud(0));
3550 brw_set_desc(p
, send
, brw_message_desc(devinfo
, 2, 0, false));
3551 brw_inst_set_sfid(devinfo
, send
, sfid
);
3552 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3553 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3555 brw_pop_insn_state(p
);
3560 * Emit the SEND message for a barrier
3563 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3565 const struct gen_device_info
*devinfo
= p
->devinfo
;
3566 struct brw_inst
*inst
;
3568 assert(devinfo
->gen
>= 7);
3570 brw_push_insn_state(p
);
3571 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3572 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3573 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3574 brw_set_src0(p
, inst
, src
);
3575 brw_set_src1(p
, inst
, brw_null_reg());
3576 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3578 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3579 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3580 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3581 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3583 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3584 brw_pop_insn_state(p
);
3589 * Emit the wait instruction for a barrier
3592 brw_WAIT(struct brw_codegen
*p
)
3594 const struct gen_device_info
*devinfo
= p
->devinfo
;
3595 struct brw_inst
*insn
;
3597 struct brw_reg src
= brw_notification_reg();
3599 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3600 brw_set_dest(p
, insn
, src
);
3601 brw_set_src0(p
, insn
, src
);
3602 brw_set_src1(p
, insn
, brw_null_reg());
3604 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3605 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3609 * Changes the floating point rounding mode updating the control register
3610 * field defined at cr0.0[5-6] bits. This function supports the changes to
3611 * RTNE (00), RU (01), RD (10) and RTZ (11) rounding using bitwise operations.
3612 * Only RTNE and RTZ rounding are enabled at nir.
3615 brw_rounding_mode(struct brw_codegen
*p
,
3616 enum brw_rnd_mode mode
)
3618 const unsigned bits
= mode
<< BRW_CR0_RND_MODE_SHIFT
;
3620 if (bits
!= BRW_CR0_RND_MODE_MASK
) {
3621 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3622 brw_imm_ud(~BRW_CR0_RND_MODE_MASK
));
3623 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3625 /* From the Skylake PRM, Volume 7, page 760:
3626 * "Implementation Restriction on Register Access: When the control
3627 * register is used as an explicit source and/or destination, hardware
3628 * does not ensure execution pipeline coherency. Software must set the
3629 * thread control field to ‘switch’ for an instruction that uses
3630 * control register as an explicit operand."
3632 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3636 brw_inst
*inst
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3638 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3639 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);