2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
90 const struct gen_device_info
*devinfo
= p
->devinfo
;
92 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
93 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
94 else if (dest
.file
== BRW_GENERAL_REGISTER_FILE
)
95 assert(dest
.nr
< 128);
97 gen7_convert_mrf_to_grf(p
, &dest
);
99 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
100 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
102 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
103 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
105 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
106 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
107 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
108 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
109 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
111 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
112 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
113 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
114 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
115 assert(dest
.writemask
!= 0);
117 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
118 * Although Dst.HorzStride is a don't care for Align16, HW needs
119 * this to be programmed as "01".
121 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
124 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
126 /* These are different sizes in align1 vs align16:
128 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
129 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
130 dest
.indirect_offset
);
131 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
132 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
133 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
135 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
136 dest
.indirect_offset
);
137 /* even ignored in da16, still need to set as '01' */
138 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
142 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
143 * or 16 (SIMD16), as that's normally correct. However, when dealing with
144 * small registers, it can be useful for us to automatically reduce it to
145 * match the register size.
147 if (p
->automatic_exec_sizes
) {
149 * In platforms that support fp64 we can emit instructions with a width
150 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
151 * these cases we need to make sure that these instructions have their
152 * exec sizes set properly when they are emitted and we can't rely on
153 * this code to fix it.
156 if (devinfo
->gen
>= 6)
157 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
159 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
162 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
167 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
169 const struct gen_device_info
*devinfo
= p
->devinfo
;
171 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
172 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
173 else if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
174 assert(reg
.nr
< 128);
176 gen7_convert_mrf_to_grf(p
, ®
);
178 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
179 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
180 /* Any source modifiers or regions will be ignored, since this just
181 * identifies the MRF/GRF to start reading the message contents from.
182 * Check for some likely failures.
186 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
189 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
190 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
191 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
192 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
194 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
195 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
196 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
197 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
198 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
199 reg
.type
== BRW_REGISTER_TYPE_Q
)
200 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
202 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
204 if (type_sz(reg
.type
) < 8) {
205 brw_inst_set_src1_reg_file(devinfo
, inst
,
206 BRW_ARCHITECTURE_REGISTER_FILE
);
207 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
208 brw_inst_src0_reg_hw_type(devinfo
, inst
));
211 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
212 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
213 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
214 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
216 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
219 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
221 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
222 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
224 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
228 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
229 if (reg
.width
== BRW_WIDTH_1
&&
230 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
231 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
232 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
233 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
235 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
236 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
237 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
240 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
241 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
242 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
243 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
244 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
245 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
246 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
247 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
249 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
250 /* This is an oddity of the fact we're using the same
251 * descriptions for registers in align_16 as align_1:
253 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
254 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
255 reg
.type
== BRW_REGISTER_TYPE_DF
&&
256 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
259 * "For Align16 access mode, only encodings of 0000 and 0011
260 * are allowed. Other codes are reserved."
262 * Presumably the DevSNB behavior applies to IVB as well.
264 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
266 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
274 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
276 const struct gen_device_info
*devinfo
= p
->devinfo
;
278 if (reg
.file
== BRW_GENERAL_REGISTER_FILE
)
279 assert(reg
.nr
< 128);
281 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
283 * "Accumulator registers may be accessed explicitly as src0
286 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
287 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
289 gen7_convert_mrf_to_grf(p
, ®
);
290 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
292 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
293 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
294 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
296 /* Only src1 can be immediate in two-argument instructions.
298 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
300 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
301 /* two-argument instructions can only use 32-bit immediates */
302 assert(type_sz(reg
.type
) < 8);
303 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
305 /* This is a hardware restriction, which may or may not be lifted
308 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
309 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
311 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
312 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
313 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
315 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
318 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
319 if (reg
.width
== BRW_WIDTH_1
&&
320 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
321 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
322 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
323 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
325 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
326 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
327 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
330 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
331 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
332 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
333 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
334 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
335 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
336 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
337 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
339 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
340 /* This is an oddity of the fact we're using the same
341 * descriptions for registers in align_16 as align_1:
343 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
344 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
345 reg
.type
== BRW_REGISTER_TYPE_DF
&&
346 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
349 * "For Align16 access mode, only encodings of 0000 and 0011
350 * are allowed. Other codes are reserved."
352 * Presumably the DevSNB behavior applies to IVB as well.
354 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
356 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
363 * Specify the descriptor and extended descriptor immediate for a SEND(C)
364 * message instruction.
367 brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*inst
,
368 unsigned desc
, unsigned ex_desc
)
370 const struct gen_device_info
*devinfo
= p
->devinfo
;
371 assert(brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
372 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
);
373 brw_inst_set_src1_file_type(devinfo
, inst
,
374 BRW_IMMEDIATE_VALUE
, BRW_REGISTER_TYPE_UD
);
375 brw_inst_set_send_desc(devinfo
, inst
, desc
);
376 if (devinfo
->gen
>= 9)
377 brw_inst_set_send_ex_desc(devinfo
, inst
, ex_desc
);
380 static void brw_set_math_message( struct brw_codegen
*p
,
383 unsigned integer_type
,
387 const struct gen_device_info
*devinfo
= p
->devinfo
;
389 unsigned response_length
;
391 /* Infer message length from the function */
393 case BRW_MATH_FUNCTION_POW
:
394 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
395 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
396 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
404 /* Infer response length from the function */
406 case BRW_MATH_FUNCTION_SINCOS
:
407 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
415 brw_set_desc(p
, inst
, brw_message_desc(
416 devinfo
, msg_length
, response_length
, false));
418 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MATH
);
419 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
420 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
421 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
422 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
423 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
424 brw_inst_set_saturate(devinfo
, inst
, 0);
428 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
431 unsigned response_length
,
434 const struct gen_device_info
*devinfo
= p
->devinfo
;
436 brw_set_desc(p
, insn
, brw_message_desc(
437 devinfo
, 1, response_length
, true));
439 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
440 brw_inst_set_eot(devinfo
, insn
, end_of_thread
);
441 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
442 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
443 /* The following fields are not used by FF_SYNC: */
444 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
445 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
446 brw_inst_set_urb_used(devinfo
, insn
, 0);
447 brw_inst_set_urb_complete(devinfo
, insn
, 0);
450 static void brw_set_urb_message( struct brw_codegen
*p
,
452 enum brw_urb_write_flags flags
,
454 unsigned response_length
,
456 unsigned swizzle_control
)
458 const struct gen_device_info
*devinfo
= p
->devinfo
;
460 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
461 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
462 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
464 brw_set_desc(p
, insn
, brw_message_desc(
465 devinfo
, msg_length
, response_length
, true));
467 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_URB
);
468 brw_inst_set_eot(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_EOT
));
470 if (flags
& BRW_URB_WRITE_OWORD
) {
471 assert(msg_length
== 2); /* header + one OWORD of data */
472 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
474 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
477 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
478 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
480 if (devinfo
->gen
< 8) {
481 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
484 if (devinfo
->gen
< 7) {
485 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
486 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
488 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
489 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
494 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
498 bool invalidate_after_read
,
500 unsigned addr_offset
,
505 const struct gen_device_info
*devinfo
= p
->devinfo
;
506 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
507 (devinfo
->gen
>= 8 && num_regs
== 8));
508 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
511 brw_set_desc(p
, inst
, brw_message_desc(
512 devinfo
, mlen
, rlen
, header_present
));
514 brw_inst_set_sfid(devinfo
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
);
515 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
516 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
517 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
518 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
519 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
520 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
524 brw_inst_set_state(const struct gen_device_info
*devinfo
,
526 const struct brw_insn_state
*state
)
528 brw_inst_set_exec_size(devinfo
, insn
, state
->exec_size
);
529 brw_inst_set_group(devinfo
, insn
, state
->group
);
530 brw_inst_set_compression(devinfo
, insn
, state
->compressed
);
531 brw_inst_set_access_mode(devinfo
, insn
, state
->access_mode
);
532 brw_inst_set_mask_control(devinfo
, insn
, state
->mask_control
);
533 brw_inst_set_saturate(devinfo
, insn
, state
->saturate
);
534 brw_inst_set_pred_control(devinfo
, insn
, state
->predicate
);
535 brw_inst_set_pred_inv(devinfo
, insn
, state
->pred_inv
);
537 if (is_3src(devinfo
, brw_inst_opcode(devinfo
, insn
)) &&
538 state
->access_mode
== BRW_ALIGN_16
) {
539 brw_inst_set_3src_a16_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
540 if (devinfo
->gen
>= 7)
541 brw_inst_set_3src_a16_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
543 brw_inst_set_flag_subreg_nr(devinfo
, insn
, state
->flag_subreg
% 2);
544 if (devinfo
->gen
>= 7)
545 brw_inst_set_flag_reg_nr(devinfo
, insn
, state
->flag_subreg
/ 2);
548 if (devinfo
->gen
>= 6)
549 brw_inst_set_acc_wr_control(devinfo
, insn
, state
->acc_wr_control
);
552 #define next_insn brw_next_insn
554 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
556 const struct gen_device_info
*devinfo
= p
->devinfo
;
559 if (p
->nr_insn
+ 1 > p
->store_size
) {
561 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
564 p
->next_insn_offset
+= 16;
565 insn
= &p
->store
[p
->nr_insn
++];
567 memset(insn
, 0, sizeof(*insn
));
568 brw_inst_set_opcode(devinfo
, insn
, opcode
);
570 /* Apply the default instruction state */
571 brw_inst_set_state(devinfo
, insn
, p
->current
);
577 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
578 struct brw_reg dest
, struct brw_reg src
)
580 brw_inst
*insn
= next_insn(p
, opcode
);
581 brw_set_dest(p
, insn
, dest
);
582 brw_set_src0(p
, insn
, src
);
587 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
588 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
590 /* 64-bit immediates are only supported on 1-src instructions */
591 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
592 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
594 brw_inst
*insn
= next_insn(p
, opcode
);
595 brw_set_dest(p
, insn
, dest
);
596 brw_set_src0(p
, insn
, src0
);
597 brw_set_src1(p
, insn
, src1
);
602 get_3src_subreg_nr(struct brw_reg reg
)
604 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
605 * use 32-bit units (components 0..7). Since they only support F/D/UD
606 * types, this doesn't lose any flexibility, but uses fewer bits.
608 return reg
.subnr
/ 4;
611 static enum gen10_align1_3src_vertical_stride
612 to_3src_align1_vstride(enum brw_vertical_stride vstride
)
615 case BRW_VERTICAL_STRIDE_0
:
616 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
;
617 case BRW_VERTICAL_STRIDE_2
:
618 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2
;
619 case BRW_VERTICAL_STRIDE_4
:
620 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4
;
621 case BRW_VERTICAL_STRIDE_8
:
622 case BRW_VERTICAL_STRIDE_16
:
623 return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
;
625 unreachable("invalid vstride");
630 static enum gen10_align1_3src_src_horizontal_stride
631 to_3src_align1_hstride(enum brw_horizontal_stride hstride
)
634 case BRW_HORIZONTAL_STRIDE_0
:
635 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
;
636 case BRW_HORIZONTAL_STRIDE_1
:
637 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
;
638 case BRW_HORIZONTAL_STRIDE_2
:
639 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2
;
640 case BRW_HORIZONTAL_STRIDE_4
:
641 return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4
;
643 unreachable("invalid hstride");
648 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
649 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
651 const struct gen_device_info
*devinfo
= p
->devinfo
;
652 brw_inst
*inst
= next_insn(p
, opcode
);
654 gen7_convert_mrf_to_grf(p
, &dest
);
656 assert(dest
.nr
< 128);
657 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| src0
.nr
< 128);
658 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| src1
.nr
< 128);
659 assert(src2
.file
!= BRW_IMMEDIATE_VALUE
|| src2
.nr
< 128);
660 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
661 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
662 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
663 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
665 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
666 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
667 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
669 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
670 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
671 BRW_ALIGN1_3SRC_ACCUMULATOR
);
672 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
674 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
675 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
676 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
678 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
680 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
682 if (brw_reg_type_is_floating_point(dest
.type
)) {
683 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
684 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
686 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
687 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
690 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
691 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
692 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
693 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
695 brw_inst_set_3src_a1_src0_vstride(devinfo
, inst
,
696 to_3src_align1_vstride(src0
.vstride
));
697 brw_inst_set_3src_a1_src1_vstride(devinfo
, inst
,
698 to_3src_align1_vstride(src1
.vstride
));
699 /* no vstride on src2 */
701 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
702 to_3src_align1_hstride(src0
.hstride
));
703 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
704 to_3src_align1_hstride(src1
.hstride
));
705 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
706 to_3src_align1_hstride(src2
.hstride
));
708 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
709 if (src0
.type
== BRW_REGISTER_TYPE_NF
) {
710 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
712 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
714 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
715 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
717 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
718 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
719 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
721 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
723 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
724 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
726 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
727 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
728 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
729 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
731 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
732 src0
.file
== BRW_IMMEDIATE_VALUE
||
733 (src0
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
734 src0
.type
== BRW_REGISTER_TYPE_NF
));
735 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
736 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
737 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
738 src2
.file
== BRW_IMMEDIATE_VALUE
);
740 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
741 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
742 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
743 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
744 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
745 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
746 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
747 BRW_ALIGN1_3SRC_ACCUMULATOR
);
748 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
749 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
750 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
751 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
753 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
754 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
755 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
756 dest
.type
== BRW_REGISTER_TYPE_DF
||
757 dest
.type
== BRW_REGISTER_TYPE_D
||
758 dest
.type
== BRW_REGISTER_TYPE_UD
);
759 if (devinfo
->gen
== 6) {
760 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
761 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
763 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
764 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
765 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
767 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
768 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
769 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
770 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
771 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
772 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
773 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
774 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
776 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
777 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
778 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
779 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
780 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
781 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
782 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
783 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
785 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
786 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
787 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
788 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
789 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
790 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
791 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
792 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
794 if (devinfo
->gen
>= 7) {
795 /* Set both the source and destination types based on dest.type,
796 * ignoring the source register types. The MAD and LRP emitters ensure
797 * that all four types are float. The BFE and BFI2 emitters, however,
798 * may send us mixed D and UD types and want us to ignore that and use
799 * the destination type.
801 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
802 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
810 /***********************************************************************
811 * Convenience routines.
814 brw_inst *brw_##OP(struct brw_codegen *p, \
815 struct brw_reg dest, \
816 struct brw_reg src0) \
818 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
822 brw_inst *brw_##OP(struct brw_codegen *p, \
823 struct brw_reg dest, \
824 struct brw_reg src0, \
825 struct brw_reg src1) \
827 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
831 brw_inst *brw_##OP(struct brw_codegen *p, \
832 struct brw_reg dest, \
833 struct brw_reg src0, \
834 struct brw_reg src1, \
835 struct brw_reg src2) \
837 if (p->current->access_mode == BRW_ALIGN_16) { \
838 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
839 src0.swizzle = BRW_SWIZZLE_XXXX; \
840 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
841 src1.swizzle = BRW_SWIZZLE_XXXX; \
842 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
843 src2.swizzle = BRW_SWIZZLE_XXXX; \
845 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
849 brw_inst *brw_##OP(struct brw_codegen *p, \
850 struct brw_reg dest, \
851 struct brw_reg src0, \
852 struct brw_reg src1, \
853 struct brw_reg src2) \
855 assert(dest.type == BRW_REGISTER_TYPE_F || \
856 dest.type == BRW_REGISTER_TYPE_DF); \
857 if (dest.type == BRW_REGISTER_TYPE_F) { \
858 assert(src0.type == BRW_REGISTER_TYPE_F); \
859 assert(src1.type == BRW_REGISTER_TYPE_F); \
860 assert(src2.type == BRW_REGISTER_TYPE_F); \
861 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
862 assert(src0.type == BRW_REGISTER_TYPE_DF); \
863 assert(src1.type == BRW_REGISTER_TYPE_DF); \
864 assert(src2.type == BRW_REGISTER_TYPE_DF); \
867 if (p->current->access_mode == BRW_ALIGN_16) { \
868 if (src0.vstride == BRW_VERTICAL_STRIDE_0) \
869 src0.swizzle = BRW_SWIZZLE_XXXX; \
870 if (src1.vstride == BRW_VERTICAL_STRIDE_0) \
871 src1.swizzle = BRW_SWIZZLE_XXXX; \
872 if (src2.vstride == BRW_VERTICAL_STRIDE_0) \
873 src2.swizzle = BRW_SWIZZLE_XXXX; \
875 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
878 /* Rounding operations (other than RNDD) require two instructions - the first
879 * stores a rounded value (possibly the wrong way) in the dest register, but
880 * also sets a per-channel "increment bit" in the flag register. A predicated
881 * add of 1.0 fixes dest to contain the desired result.
883 * Sandybridge and later appear to round correctly without an ADD.
886 void brw_##OP(struct brw_codegen *p, \
887 struct brw_reg dest, \
888 struct brw_reg src) \
890 const struct gen_device_info *devinfo = p->devinfo; \
891 brw_inst *rnd, *add; \
892 rnd = next_insn(p, BRW_OPCODE_##OP); \
893 brw_set_dest(p, rnd, dest); \
894 brw_set_src0(p, rnd, src); \
896 if (devinfo->gen < 6) { \
897 /* turn on round-increments */ \
898 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
899 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
900 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
940 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
942 const struct gen_device_info
*devinfo
= p
->devinfo
;
944 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
945 * To avoid the problems that causes, we use an <X,2,0> source region to
946 * read each element twice.
948 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
949 brw_get_default_access_mode(p
) == BRW_ALIGN_1
&&
950 dest
.type
== BRW_REGISTER_TYPE_DF
&&
951 (src0
.type
== BRW_REGISTER_TYPE_F
||
952 src0
.type
== BRW_REGISTER_TYPE_D
||
953 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
954 !has_scalar_region(src0
)) {
955 assert(src0
.vstride
== src0
.width
+ src0
.hstride
);
956 src0
.vstride
= src0
.hstride
;
957 src0
.width
= BRW_WIDTH_2
;
958 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
961 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
965 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
966 struct brw_reg src0
, struct brw_reg src1
)
969 if (src0
.type
== BRW_REGISTER_TYPE_F
||
970 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
971 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
972 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
973 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
976 if (src1
.type
== BRW_REGISTER_TYPE_F
||
977 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
978 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
979 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
980 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
983 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
987 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
988 struct brw_reg src0
, struct brw_reg src1
)
990 assert(dest
.type
== src0
.type
);
991 assert(src0
.type
== src1
.type
);
993 case BRW_REGISTER_TYPE_B
:
994 case BRW_REGISTER_TYPE_UB
:
995 case BRW_REGISTER_TYPE_W
:
996 case BRW_REGISTER_TYPE_UW
:
997 case BRW_REGISTER_TYPE_D
:
998 case BRW_REGISTER_TYPE_UD
:
1001 unreachable("Bad type for brw_AVG");
1004 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1008 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1009 struct brw_reg src0
, struct brw_reg src1
)
1012 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1013 src0
.type
== BRW_REGISTER_TYPE_UD
||
1014 src1
.type
== BRW_REGISTER_TYPE_D
||
1015 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1016 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1019 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1020 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1021 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1022 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1023 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1026 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1027 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1028 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1029 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1030 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1033 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1034 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1035 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1036 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1038 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1042 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1043 struct brw_reg src0
, struct brw_reg src1
)
1045 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1046 src0
.width
= BRW_WIDTH_1
;
1047 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1048 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1052 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1053 struct brw_reg src0
, struct brw_reg src1
)
1055 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1056 src0
.width
= BRW_WIDTH_1
;
1057 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1058 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1059 src1
.width
= BRW_WIDTH_8
;
1060 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1061 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1065 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1067 const struct gen_device_info
*devinfo
= p
->devinfo
;
1068 const bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1069 /* The F32TO16 instruction doesn't support 32-bit destination types in
1070 * Align1 mode, and neither does the Gen8 implementation in terms of a
1071 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1072 * an undocumented feature.
1074 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1075 (!align16
|| devinfo
->gen
>= 8));
1079 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1081 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1082 dst
.type
== BRW_REGISTER_TYPE_W
||
1083 dst
.type
== BRW_REGISTER_TYPE_UW
||
1084 dst
.type
== BRW_REGISTER_TYPE_HF
);
1087 brw_push_insn_state(p
);
1089 if (needs_zero_fill
) {
1090 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1091 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1094 if (devinfo
->gen
>= 8) {
1095 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1097 assert(devinfo
->gen
== 7);
1098 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1101 if (needs_zero_fill
) {
1102 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1103 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1104 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1107 brw_pop_insn_state(p
);
1112 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1114 const struct gen_device_info
*devinfo
= p
->devinfo
;
1115 bool align16
= brw_get_default_access_mode(p
) == BRW_ALIGN_16
;
1118 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1120 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1122 * Because this instruction does not have a 16-bit floating-point
1123 * type, the source data type must be Word (W). The destination type
1124 * must be F (Float).
1126 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1127 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1129 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1130 src
.type
== BRW_REGISTER_TYPE_UW
||
1131 src
.type
== BRW_REGISTER_TYPE_HF
);
1134 if (devinfo
->gen
>= 8) {
1135 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1137 assert(devinfo
->gen
== 7);
1138 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1143 void brw_NOP(struct brw_codegen
*p
)
1145 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1146 memset(insn
, 0, sizeof(*insn
));
1147 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1154 /***********************************************************************
1155 * Comparisons, if/else/endif
1159 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1160 unsigned predicate_control
)
1162 const struct gen_device_info
*devinfo
= p
->devinfo
;
1163 struct brw_reg ip
= brw_ip_reg();
1164 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1166 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1167 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1168 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1169 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1175 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1177 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1179 p
->if_stack_depth
++;
1180 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1181 p
->if_stack_array_size
*= 2;
1182 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1183 p
->if_stack_array_size
);
1188 pop_if_stack(struct brw_codegen
*p
)
1190 p
->if_stack_depth
--;
1191 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1195 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1197 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1198 p
->loop_stack_array_size
*= 2;
1199 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1200 p
->loop_stack_array_size
);
1201 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1202 p
->loop_stack_array_size
);
1205 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1206 p
->loop_stack_depth
++;
1207 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1211 get_inner_do_insn(struct brw_codegen
*p
)
1213 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1216 /* EU takes the value from the flag register and pushes it onto some
1217 * sort of a stack (presumably merging with any flag value already on
1218 * the stack). Within an if block, the flags at the top of the stack
1219 * control execution on each channel of the unit, eg. on each of the
1220 * 16 pixel values in our wm programs.
1222 * When the matching 'else' instruction is reached (presumably by
1223 * countdown of the instruction count patched in by our ELSE/ENDIF
1224 * functions), the relevant flags are inverted.
1226 * When the matching 'endif' instruction is reached, the flags are
1227 * popped off. If the stack is now empty, normal execution resumes.
1230 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1232 const struct gen_device_info
*devinfo
= p
->devinfo
;
1235 insn
= next_insn(p
, BRW_OPCODE_IF
);
1237 /* Override the defaults for this instruction:
1239 if (devinfo
->gen
< 6) {
1240 brw_set_dest(p
, insn
, brw_ip_reg());
1241 brw_set_src0(p
, insn
, brw_ip_reg());
1242 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1243 } else if (devinfo
->gen
== 6) {
1244 brw_set_dest(p
, insn
, brw_imm_w(0));
1245 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1246 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1247 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1248 } else if (devinfo
->gen
== 7) {
1249 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1250 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1251 brw_set_src1(p
, insn
, brw_imm_w(0));
1252 brw_inst_set_jip(devinfo
, insn
, 0);
1253 brw_inst_set_uip(devinfo
, insn
, 0);
1255 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1256 brw_set_src0(p
, insn
, brw_imm_d(0));
1257 brw_inst_set_jip(devinfo
, insn
, 0);
1258 brw_inst_set_uip(devinfo
, insn
, 0);
1261 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1262 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1263 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1264 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1265 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1266 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1268 push_if_stack(p
, insn
);
1269 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1273 /* This function is only used for gen6-style IF instructions with an
1274 * embedded comparison (conditional modifier). It is not used on gen7.
1277 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1278 struct brw_reg src0
, struct brw_reg src1
)
1280 const struct gen_device_info
*devinfo
= p
->devinfo
;
1283 insn
= next_insn(p
, BRW_OPCODE_IF
);
1285 brw_set_dest(p
, insn
, brw_imm_w(0));
1286 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1287 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1288 brw_set_src0(p
, insn
, src0
);
1289 brw_set_src1(p
, insn
, src1
);
1291 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1292 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1293 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1295 push_if_stack(p
, insn
);
1300 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1303 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1304 brw_inst
*if_inst
, brw_inst
*else_inst
)
1306 const struct gen_device_info
*devinfo
= p
->devinfo
;
1308 /* The next instruction (where the ENDIF would be, if it existed) */
1309 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1311 assert(p
->single_program_flow
);
1312 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1313 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1314 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1316 /* Convert IF to an ADD instruction that moves the instruction pointer
1317 * to the first instruction of the ELSE block. If there is no ELSE
1318 * block, point to where ENDIF would be. Reverse the predicate.
1320 * There's no need to execute an ENDIF since we don't need to do any
1321 * stack operations, and if we're currently executing, we just want to
1322 * continue normally.
1324 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1325 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1327 if (else_inst
!= NULL
) {
1328 /* Convert ELSE to an ADD instruction that points where the ENDIF
1331 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1333 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1334 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1336 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1341 * Patch IF and ELSE instructions with appropriate jump targets.
1344 patch_IF_ELSE(struct brw_codegen
*p
,
1345 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1347 const struct gen_device_info
*devinfo
= p
->devinfo
;
1349 /* We shouldn't be patching IF and ELSE instructions in single program flow
1350 * mode when gen < 6, because in single program flow mode on those
1351 * platforms, we convert flow control instructions to conditional ADDs that
1352 * operate on IP (see brw_ENDIF).
1354 * However, on Gen6, writing to IP doesn't work in single program flow mode
1355 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1356 * not be updated by non-flow control instructions."). And on later
1357 * platforms, there is no significant benefit to converting control flow
1358 * instructions to conditional ADDs. So we do patch IF and ELSE
1359 * instructions in single program flow mode on those platforms.
1361 if (devinfo
->gen
< 6)
1362 assert(!p
->single_program_flow
);
1364 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1365 assert(endif_inst
!= NULL
);
1366 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1368 unsigned br
= brw_jump_scale(devinfo
);
1370 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1371 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1373 if (else_inst
== NULL
) {
1374 /* Patch IF -> ENDIF */
1375 if (devinfo
->gen
< 6) {
1376 /* Turn it into an IFF, which means no mask stack operations for
1377 * all-false and jumping past the ENDIF.
1379 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1380 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1381 br
* (endif_inst
- if_inst
+ 1));
1382 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1383 } else if (devinfo
->gen
== 6) {
1384 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1385 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1387 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1388 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1391 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1393 /* Patch IF -> ELSE */
1394 if (devinfo
->gen
< 6) {
1395 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1396 br
* (else_inst
- if_inst
));
1397 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1398 } else if (devinfo
->gen
== 6) {
1399 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1400 br
* (else_inst
- if_inst
+ 1));
1403 /* Patch ELSE -> ENDIF */
1404 if (devinfo
->gen
< 6) {
1405 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1408 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1409 br
* (endif_inst
- else_inst
+ 1));
1410 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1411 } else if (devinfo
->gen
== 6) {
1412 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1413 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1414 br
* (endif_inst
- else_inst
));
1416 /* The IF instruction's JIP should point just past the ELSE */
1417 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1418 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1419 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1420 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1421 if (devinfo
->gen
>= 8) {
1422 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1423 * should point to ENDIF.
1425 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1432 brw_ELSE(struct brw_codegen
*p
)
1434 const struct gen_device_info
*devinfo
= p
->devinfo
;
1437 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1439 if (devinfo
->gen
< 6) {
1440 brw_set_dest(p
, insn
, brw_ip_reg());
1441 brw_set_src0(p
, insn
, brw_ip_reg());
1442 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1443 } else if (devinfo
->gen
== 6) {
1444 brw_set_dest(p
, insn
, brw_imm_w(0));
1445 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1446 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1447 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1448 } else if (devinfo
->gen
== 7) {
1449 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1450 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1451 brw_set_src1(p
, insn
, brw_imm_w(0));
1452 brw_inst_set_jip(devinfo
, insn
, 0);
1453 brw_inst_set_uip(devinfo
, insn
, 0);
1455 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1456 brw_set_src0(p
, insn
, brw_imm_d(0));
1457 brw_inst_set_jip(devinfo
, insn
, 0);
1458 brw_inst_set_uip(devinfo
, insn
, 0);
1461 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1462 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1463 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1464 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1466 push_if_stack(p
, insn
);
1470 brw_ENDIF(struct brw_codegen
*p
)
1472 const struct gen_device_info
*devinfo
= p
->devinfo
;
1473 brw_inst
*insn
= NULL
;
1474 brw_inst
*else_inst
= NULL
;
1475 brw_inst
*if_inst
= NULL
;
1477 bool emit_endif
= true;
1479 /* In single program flow mode, we can express IF and ELSE instructions
1480 * equivalently as ADD instructions that operate on IP. On platforms prior
1481 * to Gen6, flow control instructions cause an implied thread switch, so
1482 * this is a significant savings.
1484 * However, on Gen6, writing to IP doesn't work in single program flow mode
1485 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1486 * not be updated by non-flow control instructions."). And on later
1487 * platforms, there is no significant benefit to converting control flow
1488 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1491 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1495 * A single next_insn() may change the base address of instruction store
1496 * memory(p->store), so call it first before referencing the instruction
1497 * store pointer from an index
1500 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1502 /* Pop the IF and (optional) ELSE instructions from the stack */
1503 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1504 tmp
= pop_if_stack(p
);
1505 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1507 tmp
= pop_if_stack(p
);
1512 /* ENDIF is useless; don't bother emitting it. */
1513 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1517 if (devinfo
->gen
< 6) {
1518 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1519 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1520 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1521 } else if (devinfo
->gen
== 6) {
1522 brw_set_dest(p
, insn
, brw_imm_w(0));
1523 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1524 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1525 } else if (devinfo
->gen
== 7) {
1526 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1527 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1528 brw_set_src1(p
, insn
, brw_imm_w(0));
1530 brw_set_src0(p
, insn
, brw_imm_d(0));
1533 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1534 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1535 if (devinfo
->gen
< 6)
1536 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1538 /* Also pop item off the stack in the endif instruction: */
1539 if (devinfo
->gen
< 6) {
1540 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1541 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1542 } else if (devinfo
->gen
== 6) {
1543 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1545 brw_inst_set_jip(devinfo
, insn
, 2);
1547 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1551 brw_BREAK(struct brw_codegen
*p
)
1553 const struct gen_device_info
*devinfo
= p
->devinfo
;
1556 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1557 if (devinfo
->gen
>= 8) {
1558 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1559 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1560 } else if (devinfo
->gen
>= 6) {
1561 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1562 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1563 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1565 brw_set_dest(p
, insn
, brw_ip_reg());
1566 brw_set_src0(p
, insn
, brw_ip_reg());
1567 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1568 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1569 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1571 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1572 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1578 brw_CONT(struct brw_codegen
*p
)
1580 const struct gen_device_info
*devinfo
= p
->devinfo
;
1583 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1584 brw_set_dest(p
, insn
, brw_ip_reg());
1585 if (devinfo
->gen
>= 8) {
1586 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1588 brw_set_src0(p
, insn
, brw_ip_reg());
1589 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1592 if (devinfo
->gen
< 6) {
1593 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1594 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1596 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1597 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1602 gen6_HALT(struct brw_codegen
*p
)
1604 const struct gen_device_info
*devinfo
= p
->devinfo
;
1607 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1608 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1609 if (devinfo
->gen
>= 8) {
1610 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1612 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1613 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1616 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1617 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1623 * The DO/WHILE is just an unterminated loop -- break or continue are
1624 * used for control within the loop. We have a few ways they can be
1627 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1628 * jip and no DO instruction.
1630 * For non-uniform control flow pre-gen6, there's a DO instruction to
1631 * push the mask, and a WHILE to jump back, and BREAK to get out and
1634 * For gen6, there's no more mask stack, so no need for DO. WHILE
1635 * just points back to the first instruction of the loop.
1638 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1640 const struct gen_device_info
*devinfo
= p
->devinfo
;
1642 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1643 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1644 return &p
->store
[p
->nr_insn
];
1646 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1648 push_loop_stack(p
, insn
);
1650 /* Override the defaults for this instruction:
1652 brw_set_dest(p
, insn
, brw_null_reg());
1653 brw_set_src0(p
, insn
, brw_null_reg());
1654 brw_set_src1(p
, insn
, brw_null_reg());
1656 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1657 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1658 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1665 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1668 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1669 * nesting, since it can always just point to the end of the block/current loop.
1672 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1674 const struct gen_device_info
*devinfo
= p
->devinfo
;
1675 brw_inst
*do_inst
= get_inner_do_insn(p
);
1677 unsigned br
= brw_jump_scale(devinfo
);
1679 assert(devinfo
->gen
< 6);
1681 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1682 /* If the jump count is != 0, that means that this instruction has already
1683 * been patched because it's part of a loop inside of the one we're
1686 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1687 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1688 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1689 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1690 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1691 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1697 brw_WHILE(struct brw_codegen
*p
)
1699 const struct gen_device_info
*devinfo
= p
->devinfo
;
1700 brw_inst
*insn
, *do_insn
;
1701 unsigned br
= brw_jump_scale(devinfo
);
1703 if (devinfo
->gen
>= 6) {
1704 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1705 do_insn
= get_inner_do_insn(p
);
1707 if (devinfo
->gen
>= 8) {
1708 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1709 brw_set_src0(p
, insn
, brw_imm_d(0));
1710 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1711 } else if (devinfo
->gen
== 7) {
1712 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1713 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1714 brw_set_src1(p
, insn
, brw_imm_w(0));
1715 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1717 brw_set_dest(p
, insn
, brw_imm_w(0));
1718 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1719 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1720 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1723 brw_inst_set_exec_size(devinfo
, insn
, brw_get_default_exec_size(p
));
1726 if (p
->single_program_flow
) {
1727 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1728 do_insn
= get_inner_do_insn(p
);
1730 brw_set_dest(p
, insn
, brw_ip_reg());
1731 brw_set_src0(p
, insn
, brw_ip_reg());
1732 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1733 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1735 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1736 do_insn
= get_inner_do_insn(p
);
1738 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1740 brw_set_dest(p
, insn
, brw_ip_reg());
1741 brw_set_src0(p
, insn
, brw_ip_reg());
1742 brw_set_src1(p
, insn
, brw_imm_d(0));
1744 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1745 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1746 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1748 brw_patch_break_cont(p
, insn
);
1751 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1753 p
->loop_stack_depth
--;
1760 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1762 const struct gen_device_info
*devinfo
= p
->devinfo
;
1763 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1766 if (devinfo
->gen
>= 5)
1769 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1770 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1772 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1773 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1776 /* To integrate with the above, it makes sense that the comparison
1777 * instruction should populate the flag register. It might be simpler
1778 * just to use the flag reg for most WM tasks?
1780 void brw_CMP(struct brw_codegen
*p
,
1781 struct brw_reg dest
,
1782 unsigned conditional
,
1783 struct brw_reg src0
,
1784 struct brw_reg src1
)
1786 const struct gen_device_info
*devinfo
= p
->devinfo
;
1787 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1789 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1790 brw_set_dest(p
, insn
, dest
);
1791 brw_set_src0(p
, insn
, src0
);
1792 brw_set_src1(p
, insn
, src1
);
1794 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1796 * "Any CMP instruction with a null destination must use a {switch}."
1798 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1799 * mentioned on their work-arounds pages.
1801 if (devinfo
->gen
== 7) {
1802 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1803 dest
.nr
== BRW_ARF_NULL
) {
1804 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1809 /***********************************************************************
1810 * Helpers for the various SEND message types:
1813 /** Extended math function, float[8].
1815 void gen4_math(struct brw_codegen
*p
,
1816 struct brw_reg dest
,
1818 unsigned msg_reg_nr
,
1820 unsigned precision
)
1822 const struct gen_device_info
*devinfo
= p
->devinfo
;
1823 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1825 if (has_scalar_region(src
)) {
1826 data_type
= BRW_MATH_DATA_SCALAR
;
1828 data_type
= BRW_MATH_DATA_VECTOR
;
1831 assert(devinfo
->gen
< 6);
1833 /* Example code doesn't set predicate_control for send
1836 brw_inst_set_pred_control(devinfo
, insn
, 0);
1837 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1839 brw_set_dest(p
, insn
, dest
);
1840 brw_set_src0(p
, insn
, src
);
1841 brw_set_math_message(p
,
1844 src
.type
== BRW_REGISTER_TYPE_D
,
1849 void gen6_math(struct brw_codegen
*p
,
1850 struct brw_reg dest
,
1852 struct brw_reg src0
,
1853 struct brw_reg src1
)
1855 const struct gen_device_info
*devinfo
= p
->devinfo
;
1856 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1858 assert(devinfo
->gen
>= 6);
1860 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1861 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1863 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1864 if (devinfo
->gen
== 6) {
1865 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1866 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1869 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1870 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1871 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1872 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1873 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1874 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1875 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1877 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1878 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1881 /* Source modifiers are ignored for extended math instructions on Gen6. */
1882 if (devinfo
->gen
== 6) {
1883 assert(!src0
.negate
);
1885 assert(!src1
.negate
);
1889 brw_inst_set_math_function(devinfo
, insn
, function
);
1891 brw_set_dest(p
, insn
, dest
);
1892 brw_set_src0(p
, insn
, src0
);
1893 brw_set_src1(p
, insn
, src1
);
1897 * Return the right surface index to access the thread scratch space using
1898 * stateless dataport messages.
1901 brw_scratch_surface_idx(const struct brw_codegen
*p
)
1903 /* The scratch space is thread-local so IA coherency is unnecessary. */
1904 if (p
->devinfo
->gen
>= 8)
1905 return GEN8_BTI_STATELESS_NON_COHERENT
;
1907 return BRW_BTI_STATELESS
;
1911 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1912 * using a constant offset per channel.
1914 * The offset must be aligned to oword size (16 bytes). Used for
1915 * register spilling.
1917 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1922 const struct gen_device_info
*devinfo
= p
->devinfo
;
1923 const unsigned target_cache
=
1924 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
1925 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
1926 BRW_SFID_DATAPORT_WRITE
);
1929 if (devinfo
->gen
>= 6)
1932 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1934 const unsigned mlen
= 1 + num_regs
;
1936 /* Set up the message header. This is g0, with g0.2 filled with
1937 * the offset. We don't want to leave our offset around in g0 or
1938 * it'll screw up texture samples, so set it up inside the message
1942 brw_push_insn_state(p
);
1943 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
1944 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
1945 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
1947 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
1949 /* set message header global offset field (reg 0, element 2) */
1950 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
1952 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
1954 2), BRW_REGISTER_TYPE_UD
),
1955 brw_imm_ud(offset
));
1957 brw_pop_insn_state(p
);
1961 struct brw_reg dest
;
1962 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1963 int send_commit_msg
;
1964 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
1965 BRW_REGISTER_TYPE_UW
);
1967 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
1968 brw_inst_set_compression(devinfo
, insn
, false);
1970 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
1971 src_header
= vec16(src_header
);
1973 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1974 if (devinfo
->gen
< 6)
1975 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
1977 /* Until gen6, writes followed by reads from the same location
1978 * are not guaranteed to be ordered unless write_commit is set.
1979 * If set, then a no-op write is issued to the destination
1980 * register to set a dependency, and a read from the destination
1981 * can be used to ensure the ordering.
1983 * For gen6, only writes between different threads need ordering
1984 * protection. Our use of DP writes is all about register
1985 * spilling within a thread.
1987 if (devinfo
->gen
>= 6) {
1988 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
1989 send_commit_msg
= 0;
1992 send_commit_msg
= 1;
1995 brw_set_dest(p
, insn
, dest
);
1996 if (devinfo
->gen
>= 6) {
1997 brw_set_src0(p
, insn
, mrf
);
1999 brw_set_src0(p
, insn
, brw_null_reg());
2002 if (devinfo
->gen
>= 6)
2003 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2005 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2007 brw_set_desc(p
, insn
,
2008 brw_message_desc(devinfo
, mlen
, send_commit_msg
, true) |
2009 brw_dp_write_desc(devinfo
, brw_scratch_surface_idx(p
),
2010 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2011 msg_type
, 0, /* not a render target */
2018 * Read a block of owords (half a GRF each) from the scratch buffer
2019 * using a constant index per channel.
2021 * Offset must be aligned to oword size (16 bytes). Used for register
2025 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2026 struct brw_reg dest
,
2031 const struct gen_device_info
*devinfo
= p
->devinfo
;
2033 if (devinfo
->gen
>= 6)
2036 if (p
->devinfo
->gen
>= 7) {
2037 /* On gen 7 and above, we no longer have message registers and we can
2038 * send from any register we want. By using the destination register
2039 * for the message, we guarantee that the implied message write won't
2040 * accidentally overwrite anything. This has been a problem because
2041 * the MRF registers and source for the final FB write are both fixed
2044 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2046 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2048 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2050 const unsigned rlen
= num_regs
;
2051 const unsigned target_cache
=
2052 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2053 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2054 BRW_SFID_DATAPORT_READ
);
2057 brw_push_insn_state(p
);
2058 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2059 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2060 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2062 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2064 /* set message header global offset field (reg 0, element 2) */
2065 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2066 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2068 brw_pop_insn_state(p
);
2072 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2074 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2075 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2076 brw_inst_set_compression(devinfo
, insn
, false);
2078 brw_set_dest(p
, insn
, dest
); /* UW? */
2079 if (devinfo
->gen
>= 6) {
2080 brw_set_src0(p
, insn
, mrf
);
2082 brw_set_src0(p
, insn
, brw_null_reg());
2083 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2086 brw_set_desc(p
, insn
,
2087 brw_message_desc(devinfo
, 1, rlen
, true) |
2088 brw_dp_read_desc(devinfo
, brw_scratch_surface_idx(p
),
2089 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2090 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2091 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2096 gen7_block_read_scratch(struct brw_codegen
*p
,
2097 struct brw_reg dest
,
2101 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2102 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2104 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2106 /* The HW requires that the header is present; this is to get the g0.5
2109 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2111 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2112 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2113 * is 32 bytes, which happens to be the size of a register.
2116 assert(offset
< (1 << 12));
2118 gen7_set_dp_scratch_message(p
, insn
,
2119 false, /* scratch read */
2121 false, /* invalidate after read */
2124 1, /* mlen: just g0 */
2125 num_regs
, /* rlen */
2126 true); /* header present */
2130 * Read float[4] vectors from the data port constant cache.
2131 * Location (in buffer) should be a multiple of 16.
2132 * Used for fetching shader constants.
2134 void brw_oword_block_read(struct brw_codegen
*p
,
2135 struct brw_reg dest
,
2138 uint32_t bind_table_index
)
2140 const struct gen_device_info
*devinfo
= p
->devinfo
;
2141 const unsigned target_cache
=
2142 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2143 BRW_SFID_DATAPORT_READ
);
2144 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
2146 /* On newer hardware, offset is in units of owords. */
2147 if (devinfo
->gen
>= 6)
2150 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2152 brw_push_insn_state(p
);
2153 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2154 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2155 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2157 brw_push_insn_state(p
);
2158 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2159 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2161 /* set message header global offset field (reg 0, element 2) */
2162 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2164 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2166 2), BRW_REGISTER_TYPE_UD
),
2167 brw_imm_ud(offset
));
2168 brw_pop_insn_state(p
);
2170 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2172 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2174 /* cast dest to a uword[8] vector */
2175 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2177 brw_set_dest(p
, insn
, dest
);
2178 if (devinfo
->gen
>= 6) {
2179 brw_set_src0(p
, insn
, mrf
);
2181 brw_set_src0(p
, insn
, brw_null_reg());
2182 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2185 brw_set_desc(p
, insn
,
2186 brw_message_desc(devinfo
, 1, DIV_ROUND_UP(exec_size
, 8), true) |
2187 brw_dp_read_desc(devinfo
, bind_table_index
,
2188 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2189 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2190 BRW_DATAPORT_READ_TARGET_DATA_CACHE
));
2192 brw_pop_insn_state(p
);
2196 brw_fb_WRITE(struct brw_codegen
*p
,
2197 struct brw_reg payload
,
2198 struct brw_reg implied_header
,
2199 unsigned msg_control
,
2200 unsigned binding_table_index
,
2201 unsigned msg_length
,
2202 unsigned response_length
,
2204 bool last_render_target
,
2205 bool header_present
)
2207 const struct gen_device_info
*devinfo
= p
->devinfo
;
2208 const unsigned target_cache
=
2209 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2210 BRW_SFID_DATAPORT_WRITE
);
2213 struct brw_reg dest
, src0
;
2215 if (brw_get_default_exec_size(p
) >= BRW_EXECUTE_16
)
2216 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2218 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2220 if (devinfo
->gen
>= 6) {
2221 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2223 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2225 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2226 brw_inst_set_compression(devinfo
, insn
, false);
2228 if (devinfo
->gen
>= 6) {
2229 /* headerless version, just submit color payload */
2232 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2234 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2235 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2236 src0
= implied_header
;
2238 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2241 brw_set_dest(p
, insn
, dest
);
2242 brw_set_src0(p
, insn
, src0
);
2243 brw_set_desc(p
, insn
,
2244 brw_message_desc(devinfo
, msg_length
, response_length
,
2246 brw_dp_write_desc(devinfo
, binding_table_index
, msg_control
,
2247 msg_type
, last_render_target
,
2248 0 /* send_commit_msg */));
2249 brw_inst_set_eot(devinfo
, insn
, eot
);
2255 gen9_fb_READ(struct brw_codegen
*p
,
2257 struct brw_reg payload
,
2258 unsigned binding_table_index
,
2259 unsigned msg_length
,
2260 unsigned response_length
,
2263 const struct gen_device_info
*devinfo
= p
->devinfo
;
2264 assert(devinfo
->gen
>= 9);
2265 const unsigned msg_subtype
=
2266 brw_get_default_exec_size(p
) == BRW_EXECUTE_16
? 0 : 1;
2267 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2269 brw_inst_set_sfid(devinfo
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
);
2270 brw_set_dest(p
, insn
, dst
);
2271 brw_set_src0(p
, insn
, payload
);
2274 brw_message_desc(devinfo
, msg_length
, response_length
, true) |
2275 brw_dp_read_desc(devinfo
, binding_table_index
,
2276 per_sample
<< 5 | msg_subtype
,
2277 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2278 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
));
2279 brw_inst_set_rt_slot_group(devinfo
, insn
, brw_get_default_group(p
) / 16);
2285 * Texture sample instruction.
2286 * Note: the msg_type plus msg_length values determine exactly what kind
2287 * of sampling operation is performed. See volume 4, page 161 of docs.
2289 void brw_SAMPLE(struct brw_codegen
*p
,
2290 struct brw_reg dest
,
2291 unsigned msg_reg_nr
,
2292 struct brw_reg src0
,
2293 unsigned binding_table_index
,
2296 unsigned response_length
,
2297 unsigned msg_length
,
2298 unsigned header_present
,
2300 unsigned return_format
)
2302 const struct gen_device_info
*devinfo
= p
->devinfo
;
2305 if (msg_reg_nr
!= -1)
2306 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2308 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2309 brw_inst_set_sfid(devinfo
, insn
, BRW_SFID_SAMPLER
);
2310 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2312 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2314 * "Instruction compression is not allowed for this instruction (that
2315 * is, send). The hardware behavior is undefined if this instruction is
2316 * set as compressed. However, compress control can be set to "SecHalf"
2317 * to affect the EMask generation."
2319 * No similar wording is found in later PRMs, but there are examples
2320 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2321 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2322 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2324 brw_inst_set_compression(devinfo
, insn
, false);
2326 if (devinfo
->gen
< 6)
2327 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2329 brw_set_dest(p
, insn
, dest
);
2330 brw_set_src0(p
, insn
, src0
);
2331 brw_set_desc(p
, insn
,
2332 brw_message_desc(devinfo
, msg_length
, response_length
,
2334 brw_sampler_desc(devinfo
, binding_table_index
, sampler
,
2335 msg_type
, simd_mode
, return_format
));
2338 /* Adjust the message header's sampler state pointer to
2339 * select the correct group of 16 samplers.
2341 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2342 struct brw_reg header
,
2343 struct brw_reg sampler_index
)
2345 /* The "Sampler Index" field can only store values between 0 and 15.
2346 * However, we can add an offset to the "Sampler State Pointer"
2347 * field, effectively selecting a different set of 16 samplers.
2349 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2350 * offset, and each sampler state is only 16-bytes, so we can't
2351 * exclusively use the offset - we have to use both.
2354 const struct gen_device_info
*devinfo
= p
->devinfo
;
2356 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2357 const int sampler_state_size
= 16; /* 16 bytes */
2358 uint32_t sampler
= sampler_index
.ud
;
2360 if (sampler
>= 16) {
2361 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2363 get_element_ud(header
, 3),
2364 get_element_ud(brw_vec8_grf(0, 0), 3),
2365 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2368 /* Non-const sampler array indexing case */
2369 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2373 struct brw_reg temp
= get_element_ud(header
, 3);
2375 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2376 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2378 get_element_ud(header
, 3),
2379 get_element_ud(brw_vec8_grf(0, 0), 3),
2384 /* All these variables are pretty confusing - we might be better off
2385 * using bitmasks and macros for this, in the old style. Or perhaps
2386 * just having the caller instantiate the fields in dword3 itself.
2388 void brw_urb_WRITE(struct brw_codegen
*p
,
2389 struct brw_reg dest
,
2390 unsigned msg_reg_nr
,
2391 struct brw_reg src0
,
2392 enum brw_urb_write_flags flags
,
2393 unsigned msg_length
,
2394 unsigned response_length
,
2398 const struct gen_device_info
*devinfo
= p
->devinfo
;
2401 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2403 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2404 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2405 brw_push_insn_state(p
);
2406 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2407 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2408 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2409 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2410 BRW_REGISTER_TYPE_UD
),
2411 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2412 brw_imm_ud(0xff00));
2413 brw_pop_insn_state(p
);
2416 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2418 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2420 brw_set_dest(p
, insn
, dest
);
2421 brw_set_src0(p
, insn
, src0
);
2422 brw_set_src1(p
, insn
, brw_imm_d(0));
2424 if (devinfo
->gen
< 6)
2425 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2427 brw_set_urb_message(p
,
2437 brw_send_indirect_message(struct brw_codegen
*p
,
2440 struct brw_reg payload
,
2441 struct brw_reg desc
,
2444 const struct gen_device_info
*devinfo
= p
->devinfo
;
2445 struct brw_inst
*send
;
2447 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2449 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2451 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2452 send
= next_insn(p
, BRW_OPCODE_SEND
);
2453 brw_set_desc(p
, send
, desc
.ud
| desc_imm
);
2456 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2458 brw_push_insn_state(p
);
2459 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2460 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2461 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2462 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2464 /* Load the indirect descriptor to an address register using OR so the
2465 * caller can specify additional descriptor bits with the desc_imm
2468 brw_OR(p
, addr
, desc
, brw_imm_ud(desc_imm
));
2470 brw_pop_insn_state(p
);
2472 send
= next_insn(p
, BRW_OPCODE_SEND
);
2473 brw_set_src1(p
, send
, addr
);
2476 brw_set_dest(p
, send
, dst
);
2477 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2478 brw_inst_set_sfid(devinfo
, send
, sfid
);
2482 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2485 struct brw_reg payload
,
2486 struct brw_reg surface
,
2489 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2490 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2492 brw_push_insn_state(p
);
2493 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2494 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2495 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2496 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2498 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2499 * some surface array is accessed out of bounds.
2502 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2503 BRW_GET_SWZ(surface
.swizzle
, 0)),
2506 brw_pop_insn_state(p
);
2511 brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
, desc_imm
);
2515 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2516 brw_inst
*insn
, int while_offset
, int start_offset
)
2518 int scale
= 16 / brw_jump_scale(devinfo
);
2519 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2520 : brw_inst_jip(devinfo
, insn
);
2522 return while_offset
+ jip
* scale
<= start_offset
;
2527 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2530 void *store
= p
->store
;
2531 const struct gen_device_info
*devinfo
= p
->devinfo
;
2535 for (offset
= next_offset(devinfo
, store
, start_offset
);
2536 offset
< p
->next_insn_offset
;
2537 offset
= next_offset(devinfo
, store
, offset
)) {
2538 brw_inst
*insn
= store
+ offset
;
2540 switch (brw_inst_opcode(devinfo
, insn
)) {
2544 case BRW_OPCODE_ENDIF
:
2549 case BRW_OPCODE_WHILE
:
2550 /* If the while doesn't jump before our instruction, it's the end
2551 * of a sibling do...while loop. Ignore it.
2553 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2556 case BRW_OPCODE_ELSE
:
2557 case BRW_OPCODE_HALT
:
2566 /* There is no DO instruction on gen6, so to find the end of the loop
2567 * we have to see if the loop is jumping back before our start
2571 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2573 const struct gen_device_info
*devinfo
= p
->devinfo
;
2575 void *store
= p
->store
;
2577 assert(devinfo
->gen
>= 6);
2579 /* Always start after the instruction (such as a WHILE) we're trying to fix
2582 for (offset
= next_offset(devinfo
, store
, start_offset
);
2583 offset
< p
->next_insn_offset
;
2584 offset
= next_offset(devinfo
, store
, offset
)) {
2585 brw_inst
*insn
= store
+ offset
;
2587 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2588 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2592 assert(!"not reached");
2593 return start_offset
;
2596 /* After program generation, go back and update the UIP and JIP of
2597 * BREAK, CONT, and HALT instructions to their correct locations.
2600 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2602 const struct gen_device_info
*devinfo
= p
->devinfo
;
2604 int br
= brw_jump_scale(devinfo
);
2605 int scale
= 16 / br
;
2606 void *store
= p
->store
;
2608 if (devinfo
->gen
< 6)
2611 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2612 brw_inst
*insn
= store
+ offset
;
2613 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2615 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2616 switch (brw_inst_opcode(devinfo
, insn
)) {
2617 case BRW_OPCODE_BREAK
:
2618 assert(block_end_offset
!= 0);
2619 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2620 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2621 brw_inst_set_uip(devinfo
, insn
,
2622 (brw_find_loop_end(p
, offset
) - offset
+
2623 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2625 case BRW_OPCODE_CONTINUE
:
2626 assert(block_end_offset
!= 0);
2627 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2628 brw_inst_set_uip(devinfo
, insn
,
2629 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2631 assert(brw_inst_uip(devinfo
, insn
) != 0);
2632 assert(brw_inst_jip(devinfo
, insn
) != 0);
2635 case BRW_OPCODE_ENDIF
: {
2636 int32_t jump
= (block_end_offset
== 0) ?
2637 1 * br
: (block_end_offset
- offset
) / scale
;
2638 if (devinfo
->gen
>= 7)
2639 brw_inst_set_jip(devinfo
, insn
, jump
);
2641 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2645 case BRW_OPCODE_HALT
:
2646 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2648 * "In case of the halt instruction not inside any conditional
2649 * code block, the value of <JIP> and <UIP> should be the
2650 * same. In case of the halt instruction inside conditional code
2651 * block, the <UIP> should be the end of the program, and the
2652 * <JIP> should be end of the most inner conditional code block."
2654 * The uip will have already been set by whoever set up the
2657 if (block_end_offset
== 0) {
2658 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2660 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2662 assert(brw_inst_uip(devinfo
, insn
) != 0);
2663 assert(brw_inst_jip(devinfo
, insn
) != 0);
2669 void brw_ff_sync(struct brw_codegen
*p
,
2670 struct brw_reg dest
,
2671 unsigned msg_reg_nr
,
2672 struct brw_reg src0
,
2674 unsigned response_length
,
2677 const struct gen_device_info
*devinfo
= p
->devinfo
;
2680 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2682 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2683 brw_set_dest(p
, insn
, dest
);
2684 brw_set_src0(p
, insn
, src0
);
2685 brw_set_src1(p
, insn
, brw_imm_d(0));
2687 if (devinfo
->gen
< 6)
2688 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2690 brw_set_ff_sync_message(p
,
2698 * Emit the SEND instruction necessary to generate stream output data on Gen6
2699 * (for transform feedback).
2701 * If send_commit_msg is true, this is the last piece of stream output data
2702 * from this thread, so send the data as a committed write. According to the
2703 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2705 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2706 * writes are complete by sending the final write as a committed write."
2709 brw_svb_write(struct brw_codegen
*p
,
2710 struct brw_reg dest
,
2711 unsigned msg_reg_nr
,
2712 struct brw_reg src0
,
2713 unsigned binding_table_index
,
2714 bool send_commit_msg
)
2716 const struct gen_device_info
*devinfo
= p
->devinfo
;
2717 const unsigned target_cache
=
2718 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2719 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2720 BRW_SFID_DATAPORT_WRITE
);
2723 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2725 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2726 brw_inst_set_sfid(devinfo
, insn
, target_cache
);
2727 brw_set_dest(p
, insn
, dest
);
2728 brw_set_src0(p
, insn
, src0
);
2729 brw_set_desc(p
, insn
,
2730 brw_message_desc(devinfo
, 1, send_commit_msg
, true) |
2731 brw_dp_write_desc(devinfo
, binding_table_index
,
2732 0, /* msg_control: ignored */
2733 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2734 0, /* last_render_target: ignored */
2735 send_commit_msg
)); /* send_commit_msg */
2739 brw_surface_payload_size(struct brw_codegen
*p
,
2740 unsigned num_channels
,
2744 if (has_simd4x2
&& brw_get_default_access_mode(p
) == BRW_ALIGN_16
)
2746 else if (has_simd16
&& brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2747 return 2 * num_channels
;
2749 return num_channels
;
2753 brw_dp_untyped_atomic_desc(struct brw_codegen
*p
,
2755 bool response_expected
)
2757 const struct gen_device_info
*devinfo
= p
->devinfo
;
2758 unsigned msg_control
=
2759 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2760 (response_expected
? 1 << 5 : 0); /* Return data expected */
2763 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2764 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
2765 if (brw_get_default_exec_size(p
) != BRW_EXECUTE_16
)
2766 msg_control
|= 1 << 4; /* SIMD8 mode */
2768 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
;
2770 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
;
2773 if (brw_get_default_exec_size(p
) != BRW_EXECUTE_16
)
2774 msg_control
|= 1 << 4; /* SIMD8 mode */
2776 msg_type
= GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
;
2779 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
2783 brw_untyped_atomic(struct brw_codegen
*p
,
2785 struct brw_reg payload
,
2786 struct brw_reg surface
,
2788 unsigned msg_length
,
2789 bool response_expected
,
2790 bool header_present
)
2792 const struct gen_device_info
*devinfo
= p
->devinfo
;
2793 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2794 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2795 GEN7_SFID_DATAPORT_DATA_CACHE
);
2796 const unsigned response_length
= brw_surface_payload_size(
2797 p
, response_expected
, devinfo
->gen
>= 8 || devinfo
->is_haswell
, true);
2798 const unsigned desc
=
2799 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
2800 brw_dp_untyped_atomic_desc(p
, atomic_op
, response_expected
);
2801 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2802 /* Mask out unused components -- This is especially important in Align16
2803 * mode on generations that don't have native support for SIMD4x2 atomics,
2804 * because unused but enabled components will cause the dataport to perform
2805 * additional atomic operations on the addresses that happen to be in the
2806 * uninitialized Y, Z and W coordinates of the payload.
2808 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2810 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
2811 payload
, surface
, desc
);
2815 brw_dp_untyped_atomic_float_desc(struct brw_codegen
*p
,
2817 bool response_expected
)
2819 const struct gen_device_info
*devinfo
= p
->devinfo
;
2820 const unsigned msg_type
= GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP
;
2821 unsigned msg_control
=
2822 atomic_op
| /* Atomic Operation Type: BRW_AOP_F* */
2823 (response_expected
? 1 << 5 : 0); /* Return data expected */
2825 assert(devinfo
->gen
>= 9);
2826 assert(brw_get_default_access_mode(p
) == BRW_ALIGN_1
);
2828 if (brw_get_default_exec_size(p
) != BRW_EXECUTE_16
)
2829 msg_control
|= 1 << 4; /* SIMD8 mode */
2831 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
2835 brw_untyped_atomic_float(struct brw_codegen
*p
,
2837 struct brw_reg payload
,
2838 struct brw_reg surface
,
2840 unsigned msg_length
,
2841 bool response_expected
,
2842 bool header_present
)
2844 const struct gen_device_info
*devinfo
= p
->devinfo
;
2846 assert(devinfo
->gen
>= 9);
2847 assert(brw_get_default_access_mode(p
) == BRW_ALIGN_1
);
2849 const unsigned sfid
= HSW_SFID_DATAPORT_DATA_CACHE_1
;
2850 const unsigned response_length
= brw_surface_payload_size(
2851 p
, response_expected
, true, true);
2852 const unsigned desc
=
2853 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
2854 brw_dp_untyped_atomic_float_desc(p
, atomic_op
, response_expected
);
2856 brw_send_indirect_surface_message(p
, sfid
,
2857 brw_writemask(dst
, WRITEMASK_XYZW
),
2858 payload
, surface
, desc
);
2862 brw_dp_untyped_surface_read_desc(struct brw_codegen
*p
,
2863 unsigned num_channels
)
2865 const struct gen_device_info
*devinfo
= p
->devinfo
;
2866 const unsigned msg_type
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2867 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2868 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
);
2869 /* Set mask of 32-bit channels to drop. */
2870 unsigned msg_control
= 0xf & (0xf << num_channels
);
2872 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
2873 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2874 msg_control
|= 1 << 4; /* SIMD16 mode */
2876 msg_control
|= 2 << 4; /* SIMD8 mode */
2879 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
2883 brw_untyped_surface_read(struct brw_codegen
*p
,
2885 struct brw_reg payload
,
2886 struct brw_reg surface
,
2887 unsigned msg_length
,
2888 unsigned num_channels
)
2890 const struct gen_device_info
*devinfo
= p
->devinfo
;
2891 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2892 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2893 GEN7_SFID_DATAPORT_DATA_CACHE
);
2894 const unsigned response_length
=
2895 brw_surface_payload_size(p
, num_channels
, true, true);
2896 const unsigned desc
=
2897 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
2898 brw_dp_untyped_surface_read_desc(p
, num_channels
);
2900 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
2904 brw_dp_untyped_surface_write_desc(struct brw_codegen
*p
,
2905 unsigned num_channels
)
2907 const struct gen_device_info
*devinfo
= p
->devinfo
;
2908 const unsigned msg_type
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2909 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
2910 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
2911 /* Set mask of 32-bit channels to drop. */
2912 unsigned msg_control
= 0xf & (0xf << num_channels
);
2914 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
2915 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2916 msg_control
|= 1 << 4; /* SIMD16 mode */
2918 msg_control
|= 2 << 4; /* SIMD8 mode */
2920 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
2921 msg_control
|= 0 << 4; /* SIMD4x2 mode */
2923 msg_control
|= 2 << 4; /* SIMD8 mode */
2926 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
2930 brw_untyped_surface_write(struct brw_codegen
*p
,
2931 struct brw_reg payload
,
2932 struct brw_reg surface
,
2933 unsigned msg_length
,
2934 unsigned num_channels
,
2935 bool header_present
)
2937 const struct gen_device_info
*devinfo
= p
->devinfo
;
2938 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2939 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2940 GEN7_SFID_DATAPORT_DATA_CACHE
);
2941 const unsigned desc
=
2942 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
2943 brw_dp_untyped_surface_write_desc(p
, num_channels
);
2944 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
2945 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2946 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
2947 WRITEMASK_X
: WRITEMASK_XYZW
;
2949 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2950 payload
, surface
, desc
);
2954 brw_byte_scattered_data_element_from_bit_size(unsigned bit_size
)
2958 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE
;
2960 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD
;
2962 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD
;
2964 unreachable("Unsupported bit_size for byte scattered messages");
2969 brw_dp_byte_scattered_desc(struct brw_codegen
*p
, unsigned bit_size
,
2972 const struct gen_device_info
*devinfo
= p
->devinfo
;
2973 unsigned msg_control
=
2974 brw_byte_scattered_data_element_from_bit_size(bit_size
) << 2;
2976 if (brw_get_default_exec_size(p
) == BRW_EXECUTE_16
)
2977 msg_control
|= 1; /* SIMD16 mode */
2979 msg_control
|= 0; /* SIMD8 mode */
2981 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
2985 brw_byte_scattered_read(struct brw_codegen
*p
,
2987 struct brw_reg payload
,
2988 struct brw_reg surface
,
2989 unsigned msg_length
,
2992 const struct gen_device_info
*devinfo
= p
->devinfo
;
2993 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
2994 assert(brw_get_default_access_mode(p
) == BRW_ALIGN_1
);
2995 const unsigned response_length
=
2996 brw_surface_payload_size(p
, 1, true, true);
2997 const unsigned desc
=
2998 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
2999 brw_dp_byte_scattered_desc(p
, bit_size
,
3000 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ
);
3002 brw_send_indirect_surface_message(p
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3003 dst
, payload
, surface
, desc
);
3007 brw_byte_scattered_write(struct brw_codegen
*p
,
3008 struct brw_reg payload
,
3009 struct brw_reg surface
,
3010 unsigned msg_length
,
3012 bool header_present
)
3014 const struct gen_device_info
*devinfo
= p
->devinfo
;
3015 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
3016 assert(brw_get_default_access_mode(p
) == BRW_ALIGN_1
);
3017 const unsigned desc
=
3018 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3019 brw_dp_byte_scattered_desc(p
, bit_size
,
3020 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE
);
3022 brw_send_indirect_surface_message(p
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3023 brw_writemask(brw_null_reg(),
3025 payload
, surface
, desc
);
3029 brw_dp_typed_atomic_desc(struct brw_codegen
*p
,
3031 bool response_expected
)
3033 const struct gen_device_info
*devinfo
= p
->devinfo
;
3034 unsigned msg_control
=
3035 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3036 (response_expected
? 1 << 5 : 0); /* Return data expected */
3039 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3040 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3041 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3042 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3044 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
;
3046 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
;
3050 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3051 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3053 msg_type
= GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
;
3056 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
3060 brw_typed_atomic(struct brw_codegen
*p
,
3062 struct brw_reg payload
,
3063 struct brw_reg surface
,
3065 unsigned msg_length
,
3066 bool response_expected
,
3067 bool header_present
) {
3068 const struct gen_device_info
*devinfo
= p
->devinfo
;
3069 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3070 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3071 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3072 const unsigned response_length
= brw_surface_payload_size(
3073 p
, response_expected
, devinfo
->gen
>= 8 || devinfo
->is_haswell
, false);
3074 const unsigned desc
=
3075 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3076 brw_dp_typed_atomic_desc(p
, atomic_op
, response_expected
);
3077 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3078 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3079 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3081 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(dst
, mask
),
3082 payload
, surface
, desc
);
3086 brw_dp_typed_surface_read_desc(struct brw_codegen
*p
,
3087 unsigned num_channels
)
3089 const struct gen_device_info
*devinfo
= p
->devinfo
;
3090 /* Set mask of unused channels. */
3091 unsigned msg_control
= 0xf & (0xf << num_channels
);
3094 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3095 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3096 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3097 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3099 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3102 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
;
3104 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3105 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3106 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3109 msg_type
= GEN7_DATAPORT_RC_TYPED_SURFACE_READ
;
3112 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
3116 brw_typed_surface_read(struct brw_codegen
*p
,
3118 struct brw_reg payload
,
3119 struct brw_reg surface
,
3120 unsigned msg_length
,
3121 unsigned num_channels
,
3122 bool header_present
)
3124 const struct gen_device_info
*devinfo
= p
->devinfo
;
3125 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3126 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3127 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3128 const unsigned response_length
= brw_surface_payload_size(
3129 p
, num_channels
, devinfo
->gen
>= 8 || devinfo
->is_haswell
, false);
3130 const unsigned desc
=
3131 brw_message_desc(devinfo
, msg_length
, response_length
, header_present
) |
3132 brw_dp_typed_surface_read_desc(p
, num_channels
);
3134 brw_send_indirect_surface_message(p
, sfid
, dst
, payload
, surface
, desc
);
3138 brw_dp_typed_surface_write_desc(struct brw_codegen
*p
,
3139 unsigned num_channels
)
3141 const struct gen_device_info
*devinfo
= p
->devinfo
;
3142 /* Set mask of unused channels. */
3143 unsigned msg_control
= 0xf & (0xf << num_channels
);
3146 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3147 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3148 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3149 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3151 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3154 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
;
3157 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3158 if ((brw_get_default_group(p
) / 8) % 2 == 1)
3159 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3162 msg_type
= GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
;
3165 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
3169 brw_typed_surface_write(struct brw_codegen
*p
,
3170 struct brw_reg payload
,
3171 struct brw_reg surface
,
3172 unsigned msg_length
,
3173 unsigned num_channels
,
3174 bool header_present
)
3176 const struct gen_device_info
*devinfo
= p
->devinfo
;
3177 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3178 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3179 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3180 const unsigned desc
=
3181 brw_message_desc(devinfo
, msg_length
, 0, header_present
) |
3182 brw_dp_typed_surface_write_desc(p
, num_channels
);
3183 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3184 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3185 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3186 WRITEMASK_X
: WRITEMASK_XYZW
);
3188 brw_send_indirect_surface_message(p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3189 payload
, surface
, desc
);
3193 brw_set_memory_fence_message(struct brw_codegen
*p
,
3194 struct brw_inst
*insn
,
3195 enum brw_message_target sfid
,
3198 const struct gen_device_info
*devinfo
= p
->devinfo
;
3200 brw_set_desc(p
, insn
, brw_message_desc(
3201 devinfo
, 1, (commit_enable
? 1 : 0), true));
3203 brw_inst_set_sfid(devinfo
, insn
, sfid
);
3206 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3207 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3209 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3210 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3213 unreachable("Not reached");
3217 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3221 brw_memory_fence(struct brw_codegen
*p
,
3223 enum opcode send_op
)
3225 const struct gen_device_info
*devinfo
= p
->devinfo
;
3226 const bool commit_enable
=
3227 devinfo
->gen
>= 10 || /* HSD ES # 1404612949 */
3228 (devinfo
->gen
== 7 && !devinfo
->is_haswell
);
3229 struct brw_inst
*insn
;
3231 brw_push_insn_state(p
);
3232 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3233 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3236 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3237 * message doesn't write anything back.
3239 insn
= next_insn(p
, send_op
);
3240 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3241 brw_set_dest(p
, insn
, dst
);
3242 brw_set_src0(p
, insn
, dst
);
3243 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3246 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3247 /* IVB does typed surface access through the render cache, so we need to
3248 * flush it too. Use a different register so both flushes can be
3249 * pipelined by the hardware.
3251 insn
= next_insn(p
, send_op
);
3252 brw_set_dest(p
, insn
, offset(dst
, 1));
3253 brw_set_src0(p
, insn
, offset(dst
, 1));
3254 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3257 /* Now write the response of the second message into the response of the
3258 * first to trigger a pipeline stall -- This way future render and data
3259 * cache messages will be properly ordered with respect to past data and
3260 * render cache messages.
3262 brw_MOV(p
, dst
, offset(dst
, 1));
3265 brw_pop_insn_state(p
);
3269 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3270 struct brw_reg dest
,
3274 struct brw_reg data
,
3275 unsigned msg_length
,
3276 unsigned response_length
)
3278 const struct gen_device_info
*devinfo
= p
->devinfo
;
3279 const uint16_t exec_size
= brw_get_default_exec_size(p
);
3280 const unsigned slot_group
= brw_get_default_group(p
) / 16;
3281 const unsigned simd_mode
= (exec_size
== BRW_EXECUTE_16
);
3282 const unsigned desc
=
3283 brw_message_desc(devinfo
, msg_length
, response_length
, false) |
3284 brw_pixel_interp_desc(devinfo
, mode
, noperspective
, simd_mode
,
3287 /* brw_send_indirect_message will automatically use a direct send message
3288 * if data is actually immediate.
3290 brw_send_indirect_message(p
,
3291 GEN7_SFID_PIXEL_INTERPOLATOR
,
3299 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3300 struct brw_reg mask
)
3302 const struct gen_device_info
*devinfo
= p
->devinfo
;
3303 const unsigned exec_size
= 1 << brw_get_default_exec_size(p
);
3304 const unsigned qtr_control
= brw_get_default_group(p
) / 8;
3307 assert(devinfo
->gen
>= 7);
3308 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3310 brw_push_insn_state(p
);
3312 if (brw_get_default_access_mode(p
) == BRW_ALIGN_1
) {
3313 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3315 if (devinfo
->gen
>= 8) {
3316 /* Getting the first active channel index is easy on Gen8: Just find
3317 * the first bit set in the execution mask. The register exists on
3318 * HSW already but it reads back as all ones when the current
3319 * instruction has execution masking disabled, so it's kind of
3322 struct brw_reg exec_mask
=
3323 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3325 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3326 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3327 /* Unfortunately, ce0 does not take into account the thread
3328 * dispatch mask, which may be a problem in cases where it's not
3329 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3330 * some n). Combine ce0 with the given dispatch (or vector) mask
3331 * to mask off those channels which were never dispatched by the
3334 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3335 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3336 exec_mask
= vec1(dst
);
3339 /* Quarter control has the effect of magically shifting the value of
3340 * ce0 so you'll get the first active channel relative to the
3341 * specified quarter control as result.
3343 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3345 const struct brw_reg flag
= brw_flag_reg(p
->current
->flag_subreg
/ 2,
3346 p
->current
->flag_subreg
% 2);
3348 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3349 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3351 /* Run enough instructions returning zero with execution masking and
3352 * a conditional modifier enabled in order to get the full execution
3353 * mask in f1.0. We could use a single 32-wide move here if it
3354 * weren't because of the hardware bug that causes channel enables to
3355 * be applied incorrectly to the second half of 32-wide instructions
3358 const unsigned lower_size
= MIN2(16, exec_size
);
3359 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3360 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3362 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3363 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3364 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3365 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3368 /* Find the first bit set in the exec_size-wide portion of the flag
3369 * register that was updated by the last sequence of MOV
3372 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3373 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3374 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3377 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3379 if (devinfo
->gen
>= 8 &&
3380 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3381 /* In SIMD4x2 mode the first active channel index is just the
3382 * negation of the first bit of the mask register. Note that ce0
3383 * doesn't take into account the dispatch mask, so the Gen7 path
3384 * should be used instead unless you have the guarantee that the
3385 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3388 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3389 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3393 /* Overwrite the destination without and with execution masking to
3394 * find out which of the channels is active.
3396 brw_push_insn_state(p
);
3397 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3398 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3401 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3403 brw_pop_insn_state(p
);
3404 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3408 brw_pop_insn_state(p
);
3412 brw_broadcast(struct brw_codegen
*p
,
3417 const struct gen_device_info
*devinfo
= p
->devinfo
;
3418 const bool align1
= brw_get_default_access_mode(p
) == BRW_ALIGN_1
;
3421 brw_push_insn_state(p
);
3422 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3423 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3425 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3426 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3427 assert(!src
.abs
&& !src
.negate
);
3428 assert(src
.type
== dst
.type
);
3430 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3431 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3432 /* Trivial, the source is already uniform or the index is a constant.
3433 * We will typically not get here if the optimizer is doing its job, but
3434 * asserting would be mean.
3436 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3438 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3439 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3441 /* From the Haswell PRM section "Register Region Restrictions":
3443 * "The lower bits of the AddressImmediate must not overflow to
3444 * change the register address. The lower 5 bits of Address
3445 * Immediate when added to lower 5 bits of address register gives
3446 * the sub-register offset. The upper bits of Address Immediate
3447 * when added to upper bits of address register gives the register
3448 * address. Any overflow from sub-register offset is dropped."
3450 * Fortunately, for broadcast, we never have a sub-register offset so
3451 * this isn't an issue.
3453 assert(src
.subnr
== 0);
3456 const struct brw_reg addr
=
3457 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3458 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3459 /* Limit in bytes of the signed indirect addressing immediate. */
3460 const unsigned limit
= 512;
3462 brw_push_insn_state(p
);
3463 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3464 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3466 /* Take into account the component size and horizontal stride. */
3467 assert(src
.vstride
== src
.hstride
+ src
.width
);
3468 brw_SHL(p
, addr
, vec1(idx
),
3469 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3472 /* We can only address up to limit bytes using the indirect
3473 * addressing immediate, account for the difference if the source
3474 * register is above this limit.
3476 if (offset
>= limit
) {
3477 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3478 offset
= offset
% limit
;
3481 brw_pop_insn_state(p
);
3483 /* Use indirect addressing to fetch the specified component. */
3484 if (type_sz(src
.type
) > 4 &&
3485 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3486 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3488 * "When source or destination datatype is 64b or operation is
3489 * integer DWord multiply, indirect addressing must not be
3492 * To work around both of this issue, we do two integer MOVs
3493 * insead of one 64-bit MOV. Because no double value should ever
3494 * cross a register boundary, it's safe to use the immediate
3495 * offset in the indirect here to handle adding 4 bytes to the
3496 * offset and avoid the extra ADD to the register file.
3498 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3499 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3500 BRW_REGISTER_TYPE_D
));
3501 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3502 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3503 BRW_REGISTER_TYPE_D
));
3506 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3509 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3510 * to all bits of a flag register,
3514 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3515 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3516 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3517 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3519 /* and use predicated SEL to pick the right channel. */
3520 inst
= brw_SEL(p
, dst
,
3521 stride(suboffset(src
, 4), 4, 4, 1),
3522 stride(src
, 4, 4, 1));
3523 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3524 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3528 brw_pop_insn_state(p
);
3532 * This instruction is generated as a single-channel align1 instruction by
3533 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3535 * We can't use the typed atomic op in the FS because that has the execution
3536 * mask ANDed with the pixel mask, but we just want to write the one dword for
3539 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3540 * one u32. So we use the same untyped atomic write message as the pixel
3543 * The untyped atomic operation requires a BUFFER surface type with RAW
3544 * format, and is only accessible through the legacy DATA_CACHE dataport
3547 void brw_shader_time_add(struct brw_codegen
*p
,
3548 struct brw_reg payload
,
3549 uint32_t surf_index
)
3551 const struct gen_device_info
*devinfo
= p
->devinfo
;
3552 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3553 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3554 GEN7_SFID_DATAPORT_DATA_CACHE
);
3555 assert(devinfo
->gen
>= 7);
3557 brw_push_insn_state(p
);
3558 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3559 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3560 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3561 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3563 /* We use brw_vec1_reg and unmasked because we want to increment the given
3566 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3568 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3570 brw_set_desc(p
, send
, (brw_message_desc(devinfo
, 2, 0, false) |
3571 brw_dp_untyped_atomic_desc(p
, BRW_AOP_ADD
, false)));
3573 brw_inst_set_sfid(devinfo
, send
, sfid
);
3574 brw_inst_set_binding_table_index(devinfo
, send
, surf_index
);
3576 brw_pop_insn_state(p
);
3581 * Emit the SEND message for a barrier
3584 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3586 const struct gen_device_info
*devinfo
= p
->devinfo
;
3587 struct brw_inst
*inst
;
3589 assert(devinfo
->gen
>= 7);
3591 brw_push_insn_state(p
);
3592 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3593 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3594 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3595 brw_set_src0(p
, inst
, src
);
3596 brw_set_src1(p
, inst
, brw_null_reg());
3597 brw_set_desc(p
, inst
, brw_message_desc(devinfo
, 1, 0, false));
3599 brw_inst_set_sfid(devinfo
, inst
, BRW_SFID_MESSAGE_GATEWAY
);
3600 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3601 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3602 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3604 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3605 brw_pop_insn_state(p
);
3610 * Emit the wait instruction for a barrier
3613 brw_WAIT(struct brw_codegen
*p
)
3615 const struct gen_device_info
*devinfo
= p
->devinfo
;
3616 struct brw_inst
*insn
;
3618 struct brw_reg src
= brw_notification_reg();
3620 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3621 brw_set_dest(p
, insn
, src
);
3622 brw_set_src0(p
, insn
, src
);
3623 brw_set_src1(p
, insn
, brw_null_reg());
3625 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3626 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3630 * Changes the floating point rounding mode updating the control register
3631 * field defined at cr0.0[5-6] bits. This function supports the changes to
3632 * RTNE (00), RU (01), RD (10) and RTZ (11) rounding using bitwise operations.
3633 * Only RTNE and RTZ rounding are enabled at nir.
3636 brw_rounding_mode(struct brw_codegen
*p
,
3637 enum brw_rnd_mode mode
)
3639 const unsigned bits
= mode
<< BRW_CR0_RND_MODE_SHIFT
;
3641 if (bits
!= BRW_CR0_RND_MODE_MASK
) {
3642 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3643 brw_imm_ud(~BRW_CR0_RND_MODE_MASK
));
3644 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3646 /* From the Skylake PRM, Volume 7, page 760:
3647 * "Implementation Restriction on Register Access: When the control
3648 * register is used as an explicit source and/or destination, hardware
3649 * does not ensure execution pipeline coherency. Software must set the
3650 * thread control field to ‘switch’ for an instruction that uses
3651 * control register as an explicit operand."
3653 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3657 brw_inst
*inst
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3659 brw_inst_set_exec_size(p
->devinfo
, inst
, BRW_EXECUTE_1
);
3660 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);