2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_eu_defines.h"
36 #include "util/ralloc.h"
39 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
40 * registers, implicitly moving the operand to a message register.
42 * On Sandybridge, this is no longer the case. This function performs the
43 * explicit move; it should be called before emitting a SEND instruction.
46 gen6_resolve_implied_move(struct brw_codegen
*p
,
50 const struct gen_device_info
*devinfo
= p
->devinfo
;
54 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
57 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
58 brw_push_insn_state(p
);
59 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
60 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
61 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
62 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
63 retype(*src
, BRW_REGISTER_TYPE_UD
));
64 brw_pop_insn_state(p
);
66 *src
= brw_message_reg(msg_reg_nr
);
70 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
72 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
73 * "The send with EOT should use register space R112-R127 for <src>. This is
74 * to enable loading of a new thread into the same slot while the message
75 * with EOT for current thread is pending dispatch."
77 * Since we're pretending to have 16 MRFs anyway, we may as well use the
78 * registers required for messages with EOT.
80 const struct gen_device_info
*devinfo
= p
->devinfo
;
81 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
82 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
83 reg
->nr
+= GEN7_MRF_HACK_START
;
88 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
90 const struct gen_device_info
*devinfo
= p
->devinfo
;
92 if (dest
.file
== BRW_MESSAGE_REGISTER_FILE
)
93 assert((dest
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
94 else if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
95 assert(dest
.nr
< 128);
97 gen7_convert_mrf_to_grf(p
, &dest
);
99 brw_inst_set_dst_file_type(devinfo
, inst
, dest
.file
, dest
.type
);
100 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
102 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
103 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
105 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
106 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
107 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
108 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
109 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
111 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
112 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.writemask
);
113 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
114 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
115 assert(dest
.writemask
!= 0);
117 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
118 * Although Dst.HorzStride is a don't care for Align16, HW needs
119 * this to be programmed as "01".
121 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
124 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
126 /* These are different sizes in align1 vs align16:
128 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
129 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
130 dest
.indirect_offset
);
131 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
132 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
133 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
135 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
136 dest
.indirect_offset
);
137 /* even ignored in da16, still need to set as '01' */
138 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
142 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
143 * or 16 (SIMD16), as that's normally correct. However, when dealing with
144 * small registers, it can be useful for us to automatically reduce it to
145 * match the register size.
147 if (p
->automatic_exec_sizes
) {
149 * In platforms that support fp64 we can emit instructions with a width
150 * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
151 * these cases we need to make sure that these instructions have their
152 * exec sizes set properly when they are emitted and we can't rely on
153 * this code to fix it.
156 if (devinfo
->gen
>= 6)
157 fix_exec_size
= dest
.width
< BRW_EXECUTE_4
;
159 fix_exec_size
= dest
.width
< BRW_EXECUTE_8
;
162 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
167 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
169 const struct gen_device_info
*devinfo
= p
->devinfo
;
171 if (reg
.file
== BRW_MESSAGE_REGISTER_FILE
)
172 assert((reg
.nr
& ~BRW_MRF_COMPR4
) < BRW_MAX_MRF(devinfo
->gen
));
173 else if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
174 assert(reg
.nr
< 128);
176 gen7_convert_mrf_to_grf(p
, ®
);
178 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
179 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
180 /* Any source modifiers or regions will be ignored, since this just
181 * identifies the MRF/GRF to start reading the message contents from.
182 * Check for some likely failures.
186 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
189 brw_inst_set_src0_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
190 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
191 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
192 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
194 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
195 if (reg
.type
== BRW_REGISTER_TYPE_DF
||
196 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_DIM
)
197 brw_inst_set_imm_df(devinfo
, inst
, reg
.df
);
198 else if (reg
.type
== BRW_REGISTER_TYPE_UQ
||
199 reg
.type
== BRW_REGISTER_TYPE_Q
)
200 brw_inst_set_imm_uq(devinfo
, inst
, reg
.u64
);
202 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
204 if (type_sz(reg
.type
) < 8) {
205 brw_inst_set_src1_reg_file(devinfo
, inst
,
206 BRW_ARCHITECTURE_REGISTER_FILE
);
207 brw_inst_set_src1_reg_hw_type(devinfo
, inst
,
208 brw_inst_src0_reg_hw_type(devinfo
, inst
));
211 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
212 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
213 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
214 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
216 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
219 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
221 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
222 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
224 brw_inst_set_src0_ia16_addr_imm(devinfo
, inst
, reg
.indirect_offset
);
228 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
229 if (reg
.width
== BRW_WIDTH_1
&&
230 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
231 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
232 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
233 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
235 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
236 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
237 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
240 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
241 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
242 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
243 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
244 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
245 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
246 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
247 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
249 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
250 /* This is an oddity of the fact we're using the same
251 * descriptions for registers in align_16 as align_1:
253 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
254 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
255 reg
.type
== BRW_REGISTER_TYPE_DF
&&
256 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
259 * "For Align16 access mode, only encodings of 0000 and 0011
260 * are allowed. Other codes are reserved."
262 * Presumably the DevSNB behavior applies to IVB as well.
264 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
266 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
274 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
276 const struct gen_device_info
*devinfo
= p
->devinfo
;
278 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
279 assert(reg
.nr
< 128);
281 /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
283 * "Accumulator registers may be accessed explicitly as src0
286 assert(reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
287 reg
.nr
!= BRW_ARF_ACCUMULATOR
);
289 gen7_convert_mrf_to_grf(p
, ®
);
290 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
292 brw_inst_set_src1_file_type(devinfo
, inst
, reg
.file
, reg
.type
);
293 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
294 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
296 /* Only src1 can be immediate in two-argument instructions.
298 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
300 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
301 /* two-argument instructions can only use 32-bit immediates */
302 assert(type_sz(reg
.type
) < 8);
303 brw_inst_set_imm_ud(devinfo
, inst
, reg
.ud
);
305 /* This is a hardware restriction, which may or may not be lifted
308 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
309 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
311 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
312 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
313 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
315 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
318 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
319 if (reg
.width
== BRW_WIDTH_1
&&
320 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
321 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
322 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
323 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
325 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
326 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
327 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
330 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
331 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_X
));
332 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
333 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Y
));
334 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
335 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_Z
));
336 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
337 BRW_GET_SWZ(reg
.swizzle
, BRW_CHANNEL_W
));
339 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
) {
340 /* This is an oddity of the fact we're using the same
341 * descriptions for registers in align_16 as align_1:
343 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
344 } else if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
345 reg
.type
== BRW_REGISTER_TYPE_DF
&&
346 reg
.vstride
== BRW_VERTICAL_STRIDE_2
) {
349 * "For Align16 access mode, only encodings of 0000 and 0011
350 * are allowed. Other codes are reserved."
352 * Presumably the DevSNB behavior applies to IVB as well.
354 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
356 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
363 * Set the Message Descriptor and Extended Message Descriptor fields
366 * \note This zeroes out the Function Control bits, so it must be called
367 * \b before filling out any message-specific data. Callers can
368 * choose not to fill in irrelevant bits; they will be zero.
371 brw_set_message_descriptor(struct brw_codegen
*p
,
373 enum brw_message_target sfid
,
375 unsigned response_length
,
379 const struct gen_device_info
*devinfo
= p
->devinfo
;
381 brw_set_src1(p
, inst
, brw_imm_d(0));
383 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
384 * itself; instead, it will be a MOV/OR into the address register.
386 * In this case, we avoid setting the extended message descriptor bits,
387 * since they go on the later SEND/SENDC instead and if set here would
388 * instead clobber the conditionalmod bits.
390 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
391 if (opcode
== BRW_OPCODE_SEND
|| opcode
== BRW_OPCODE_SENDC
) {
392 brw_inst_set_sfid(devinfo
, inst
, sfid
);
395 brw_inst_set_mlen(devinfo
, inst
, msg_length
);
396 brw_inst_set_rlen(devinfo
, inst
, response_length
);
397 brw_inst_set_eot(devinfo
, inst
, end_of_thread
);
399 if (devinfo
->gen
>= 5) {
400 brw_inst_set_header_present(devinfo
, inst
, header_present
);
404 static void brw_set_math_message( struct brw_codegen
*p
,
407 unsigned integer_type
,
411 const struct gen_device_info
*devinfo
= p
->devinfo
;
413 unsigned response_length
;
415 /* Infer message length from the function */
417 case BRW_MATH_FUNCTION_POW
:
418 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
419 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
420 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
428 /* Infer response length from the function */
430 case BRW_MATH_FUNCTION_SINCOS
:
431 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
440 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
441 msg_length
, response_length
, false, false);
442 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
443 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
444 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
445 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
446 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
447 brw_inst_set_saturate(devinfo
, inst
, 0);
451 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
454 unsigned response_length
,
457 const struct gen_device_info
*devinfo
= p
->devinfo
;
459 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
460 1, response_length
, true, end_of_thread
);
461 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
462 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
463 /* The following fields are not used by FF_SYNC: */
464 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
465 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
466 brw_inst_set_urb_used(devinfo
, insn
, 0);
467 brw_inst_set_urb_complete(devinfo
, insn
, 0);
470 static void brw_set_urb_message( struct brw_codegen
*p
,
472 enum brw_urb_write_flags flags
,
474 unsigned response_length
,
476 unsigned swizzle_control
)
478 const struct gen_device_info
*devinfo
= p
->devinfo
;
480 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
481 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
482 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
484 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
485 msg_length
, response_length
, true,
486 flags
& BRW_URB_WRITE_EOT
);
488 if (flags
& BRW_URB_WRITE_OWORD
) {
489 assert(msg_length
== 2); /* header + one OWORD of data */
490 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
492 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
495 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
496 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
498 if (devinfo
->gen
< 8) {
499 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
502 if (devinfo
->gen
< 7) {
503 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
504 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
506 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
507 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
512 brw_set_dp_write_message(struct brw_codegen
*p
,
514 unsigned binding_table_index
,
515 unsigned msg_control
,
517 unsigned target_cache
,
520 unsigned last_render_target
,
521 unsigned response_length
,
522 unsigned end_of_thread
,
523 unsigned send_commit_msg
)
525 const struct gen_device_info
*devinfo
= p
->devinfo
;
526 const unsigned sfid
= (devinfo
->gen
>= 6 ? target_cache
:
527 BRW_SFID_DATAPORT_WRITE
);
529 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
530 header_present
, end_of_thread
);
532 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
533 brw_inst_set_dp_write_msg_type(devinfo
, insn
, msg_type
);
534 brw_inst_set_dp_write_msg_control(devinfo
, insn
, msg_control
);
535 brw_inst_set_rt_last(devinfo
, insn
, last_render_target
);
536 if (devinfo
->gen
< 7) {
537 brw_inst_set_dp_write_commit(devinfo
, insn
, send_commit_msg
);
542 brw_set_dp_read_message(struct brw_codegen
*p
,
544 unsigned binding_table_index
,
545 unsigned msg_control
,
547 unsigned target_cache
,
550 unsigned response_length
)
552 const struct gen_device_info
*devinfo
= p
->devinfo
;
553 const unsigned sfid
= (devinfo
->gen
>= 6 ? target_cache
:
554 BRW_SFID_DATAPORT_READ
);
556 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
557 header_present
, false);
559 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
560 brw_inst_set_dp_read_msg_type(devinfo
, insn
, msg_type
);
561 brw_inst_set_dp_read_msg_control(devinfo
, insn
, msg_control
);
562 if (devinfo
->gen
< 6)
563 brw_inst_set_dp_read_target_cache(devinfo
, insn
, target_cache
);
567 brw_set_sampler_message(struct brw_codegen
*p
,
569 unsigned binding_table_index
,
572 unsigned response_length
,
574 unsigned header_present
,
576 unsigned return_format
)
578 const struct gen_device_info
*devinfo
= p
->devinfo
;
580 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
581 response_length
, header_present
, false);
583 brw_inst_set_binding_table_index(devinfo
, inst
, binding_table_index
);
584 brw_inst_set_sampler(devinfo
, inst
, sampler
);
585 brw_inst_set_sampler_msg_type(devinfo
, inst
, msg_type
);
586 if (devinfo
->gen
>= 5) {
587 brw_inst_set_sampler_simd_mode(devinfo
, inst
, simd_mode
);
588 } else if (devinfo
->gen
== 4 && !devinfo
->is_g4x
) {
589 brw_inst_set_sampler_return_format(devinfo
, inst
, return_format
);
594 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
598 bool invalidate_after_read
,
600 unsigned addr_offset
,
605 const struct gen_device_info
*devinfo
= p
->devinfo
;
606 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
607 (devinfo
->gen
>= 8 && num_regs
== 8));
608 const unsigned block_size
= (devinfo
->gen
>= 8 ? _mesa_logbase2(num_regs
) :
611 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
612 mlen
, rlen
, header_present
, false);
613 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
614 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
615 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
616 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
617 brw_inst_set_scratch_block_size(devinfo
, inst
, block_size
);
618 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
621 #define next_insn brw_next_insn
623 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
625 const struct gen_device_info
*devinfo
= p
->devinfo
;
628 if (p
->nr_insn
+ 1 > p
->store_size
) {
630 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
633 p
->next_insn_offset
+= 16;
634 insn
= &p
->store
[p
->nr_insn
++];
635 memcpy(insn
, p
->current
, sizeof(*insn
));
637 brw_inst_set_opcode(devinfo
, insn
, opcode
);
642 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
643 struct brw_reg dest
, struct brw_reg src
)
645 brw_inst
*insn
= next_insn(p
, opcode
);
646 brw_set_dest(p
, insn
, dest
);
647 brw_set_src0(p
, insn
, src
);
652 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
653 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
655 /* 64-bit immediates are only supported on 1-src instructions */
656 assert(src0
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src0
.type
) <= 4);
657 assert(src1
.file
!= BRW_IMMEDIATE_VALUE
|| type_sz(src1
.type
) <= 4);
659 brw_inst
*insn
= next_insn(p
, opcode
);
660 brw_set_dest(p
, insn
, dest
);
661 brw_set_src0(p
, insn
, src0
);
662 brw_set_src1(p
, insn
, src1
);
667 get_3src_subreg_nr(struct brw_reg reg
)
669 /* Normally, SubRegNum is in bytes (0..31). However, 3-src instructions
670 * use 32-bit units (components 0..7). Since they only support F/D/UD
671 * types, this doesn't lose any flexibility, but uses fewer bits.
673 return reg
.subnr
/ 4;
677 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
678 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
680 const struct gen_device_info
*devinfo
= p
->devinfo
;
681 brw_inst
*inst
= next_insn(p
, opcode
);
683 gen7_convert_mrf_to_grf(p
, &dest
);
685 assert(dest
.nr
< 128);
686 assert(src0
.nr
< 128);
687 assert(src1
.nr
< 128);
688 assert(src2
.nr
< 128);
689 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
690 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
691 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
692 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
694 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
695 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
696 dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
698 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
699 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
700 BRW_ALIGN1_3SRC_ACCUMULATOR
);
701 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
703 brw_inst_set_3src_a1_dst_reg_file(devinfo
, inst
,
704 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
);
705 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
707 brw_inst_set_3src_a1_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 8);
709 brw_inst_set_3src_a1_dst_hstride(devinfo
, inst
, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1
);
711 if (brw_reg_type_is_floating_point(dest
.type
)) {
712 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
713 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT
);
715 brw_inst_set_3src_a1_exec_type(devinfo
, inst
,
716 BRW_ALIGN1_3SRC_EXEC_TYPE_INT
);
719 brw_inst_set_3src_a1_dst_type(devinfo
, inst
, dest
.type
);
720 brw_inst_set_3src_a1_src0_type(devinfo
, inst
, src0
.type
);
721 brw_inst_set_3src_a1_src1_type(devinfo
, inst
, src1
.type
);
722 brw_inst_set_3src_a1_src2_type(devinfo
, inst
, src2
.type
);
724 assert((src0
.vstride
== BRW_VERTICAL_STRIDE_0
&&
725 src0
.hstride
== BRW_HORIZONTAL_STRIDE_0
) ||
726 (src0
.vstride
== BRW_VERTICAL_STRIDE_8
&&
727 src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
));
728 assert((src1
.vstride
== BRW_VERTICAL_STRIDE_0
&&
729 src1
.hstride
== BRW_HORIZONTAL_STRIDE_0
) ||
730 (src1
.vstride
== BRW_VERTICAL_STRIDE_8
&&
731 src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
));
732 assert((src2
.vstride
== BRW_VERTICAL_STRIDE_0
&&
733 src2
.hstride
== BRW_HORIZONTAL_STRIDE_0
) ||
734 (src2
.vstride
== BRW_VERTICAL_STRIDE_8
&&
735 src2
.hstride
== BRW_HORIZONTAL_STRIDE_1
));
737 brw_inst_set_3src_a1_src0_vstride(devinfo
, inst
,
738 src0
.vstride
== BRW_VERTICAL_STRIDE_0
?
739 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
:
740 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
);
741 brw_inst_set_3src_a1_src1_vstride(devinfo
, inst
,
742 src1
.vstride
== BRW_VERTICAL_STRIDE_0
?
743 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0
:
744 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8
);
745 /* no vstride on src2 */
747 brw_inst_set_3src_a1_src0_hstride(devinfo
, inst
,
748 src0
.hstride
== BRW_HORIZONTAL_STRIDE_0
?
749 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
:
750 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
);
751 brw_inst_set_3src_a1_src1_hstride(devinfo
, inst
,
752 src1
.hstride
== BRW_HORIZONTAL_STRIDE_0
?
753 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
:
754 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
);
755 brw_inst_set_3src_a1_src2_hstride(devinfo
, inst
,
756 src2
.hstride
== BRW_HORIZONTAL_STRIDE_0
?
757 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0
:
758 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1
);
760 brw_inst_set_3src_a1_src0_subreg_nr(devinfo
, inst
, src0
.subnr
);
761 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
762 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
763 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
765 brw_inst_set_3src_a1_src1_subreg_nr(devinfo
, inst
, src1
.subnr
);
766 if (src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
) {
767 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, BRW_ARF_ACCUMULATOR
);
769 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
771 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
772 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
774 brw_inst_set_3src_a1_src2_subreg_nr(devinfo
, inst
, src2
.subnr
);
775 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
776 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
777 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
779 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
780 src0
.file
== BRW_IMMEDIATE_VALUE
);
781 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
782 src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
);
783 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
||
784 src2
.file
== BRW_IMMEDIATE_VALUE
);
786 brw_inst_set_3src_a1_src0_reg_file(devinfo
, inst
,
787 src0
.file
== BRW_GENERAL_REGISTER_FILE
?
788 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
789 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
790 brw_inst_set_3src_a1_src1_reg_file(devinfo
, inst
,
791 src1
.file
== BRW_GENERAL_REGISTER_FILE
?
792 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
793 BRW_ALIGN1_3SRC_ACCUMULATOR
);
794 brw_inst_set_3src_a1_src2_reg_file(devinfo
, inst
,
795 src2
.file
== BRW_GENERAL_REGISTER_FILE
?
796 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE
:
797 BRW_ALIGN1_3SRC_IMMEDIATE_VALUE
);
799 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
800 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
801 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
802 dest
.type
== BRW_REGISTER_TYPE_DF
||
803 dest
.type
== BRW_REGISTER_TYPE_D
||
804 dest
.type
== BRW_REGISTER_TYPE_UD
);
805 if (devinfo
->gen
== 6) {
806 brw_inst_set_3src_a16_dst_reg_file(devinfo
, inst
,
807 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
809 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
810 brw_inst_set_3src_a16_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
811 brw_inst_set_3src_a16_dst_writemask(devinfo
, inst
, dest
.writemask
);
813 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
814 brw_inst_set_3src_a16_src0_swizzle(devinfo
, inst
, src0
.swizzle
);
815 brw_inst_set_3src_a16_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
816 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
817 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
818 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
819 brw_inst_set_3src_a16_src0_rep_ctrl(devinfo
, inst
,
820 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
822 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
823 brw_inst_set_3src_a16_src1_swizzle(devinfo
, inst
, src1
.swizzle
);
824 brw_inst_set_3src_a16_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
825 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
826 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
827 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
828 brw_inst_set_3src_a16_src1_rep_ctrl(devinfo
, inst
,
829 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
831 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
832 brw_inst_set_3src_a16_src2_swizzle(devinfo
, inst
, src2
.swizzle
);
833 brw_inst_set_3src_a16_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
834 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
835 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
836 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
837 brw_inst_set_3src_a16_src2_rep_ctrl(devinfo
, inst
,
838 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
840 if (devinfo
->gen
>= 7) {
841 /* Set both the source and destination types based on dest.type,
842 * ignoring the source register types. The MAD and LRP emitters ensure
843 * that all four types are float. The BFE and BFI2 emitters, however,
844 * may send us mixed D and UD types and want us to ignore that and use
845 * the destination type.
847 brw_inst_set_3src_a16_src_type(devinfo
, inst
, dest
.type
);
848 brw_inst_set_3src_a16_dst_type(devinfo
, inst
, dest
.type
);
856 /***********************************************************************
857 * Convenience routines.
860 brw_inst *brw_##OP(struct brw_codegen *p, \
861 struct brw_reg dest, \
862 struct brw_reg src0) \
864 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
868 brw_inst *brw_##OP(struct brw_codegen *p, \
869 struct brw_reg dest, \
870 struct brw_reg src0, \
871 struct brw_reg src1) \
873 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
877 brw_inst *brw_##OP(struct brw_codegen *p, \
878 struct brw_reg dest, \
879 struct brw_reg src0, \
880 struct brw_reg src1, \
881 struct brw_reg src2) \
883 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
887 brw_inst *brw_##OP(struct brw_codegen *p, \
888 struct brw_reg dest, \
889 struct brw_reg src0, \
890 struct brw_reg src1, \
891 struct brw_reg src2) \
893 assert(dest.type == BRW_REGISTER_TYPE_F || \
894 dest.type == BRW_REGISTER_TYPE_DF); \
895 if (dest.type == BRW_REGISTER_TYPE_F) { \
896 assert(src0.type == BRW_REGISTER_TYPE_F); \
897 assert(src1.type == BRW_REGISTER_TYPE_F); \
898 assert(src2.type == BRW_REGISTER_TYPE_F); \
899 } else if (dest.type == BRW_REGISTER_TYPE_DF) { \
900 assert(src0.type == BRW_REGISTER_TYPE_DF); \
901 assert(src1.type == BRW_REGISTER_TYPE_DF); \
902 assert(src2.type == BRW_REGISTER_TYPE_DF); \
904 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
907 /* Rounding operations (other than RNDD) require two instructions - the first
908 * stores a rounded value (possibly the wrong way) in the dest register, but
909 * also sets a per-channel "increment bit" in the flag register. A predicated
910 * add of 1.0 fixes dest to contain the desired result.
912 * Sandybridge and later appear to round correctly without an ADD.
915 void brw_##OP(struct brw_codegen *p, \
916 struct brw_reg dest, \
917 struct brw_reg src) \
919 const struct gen_device_info *devinfo = p->devinfo; \
920 brw_inst *rnd, *add; \
921 rnd = next_insn(p, BRW_OPCODE_##OP); \
922 brw_set_dest(p, rnd, dest); \
923 brw_set_src0(p, rnd, src); \
925 if (devinfo->gen < 6) { \
926 /* turn on round-increments */ \
927 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
928 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
929 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
968 brw_MOV(struct brw_codegen
*p
, struct brw_reg dest
, struct brw_reg src0
)
970 const struct gen_device_info
*devinfo
= p
->devinfo
;
972 /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
973 * To avoid the problems that causes, we use a <1,2,0> source region to read
974 * each element twice.
976 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
977 brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
&&
978 dest
.type
== BRW_REGISTER_TYPE_DF
&&
979 (src0
.type
== BRW_REGISTER_TYPE_F
||
980 src0
.type
== BRW_REGISTER_TYPE_D
||
981 src0
.type
== BRW_REGISTER_TYPE_UD
) &&
982 !has_scalar_region(src0
)) {
983 assert(src0
.vstride
== BRW_VERTICAL_STRIDE_4
&&
984 src0
.width
== BRW_WIDTH_4
&&
985 src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
987 src0
.vstride
= BRW_VERTICAL_STRIDE_1
;
988 src0
.width
= BRW_WIDTH_2
;
989 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
992 return brw_alu1(p
, BRW_OPCODE_MOV
, dest
, src0
);
996 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
997 struct brw_reg src0
, struct brw_reg src1
)
1000 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1001 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1002 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1003 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1004 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1007 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1008 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1009 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1010 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1011 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1014 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1018 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1019 struct brw_reg src0
, struct brw_reg src1
)
1021 assert(dest
.type
== src0
.type
);
1022 assert(src0
.type
== src1
.type
);
1023 switch (src0
.type
) {
1024 case BRW_REGISTER_TYPE_B
:
1025 case BRW_REGISTER_TYPE_UB
:
1026 case BRW_REGISTER_TYPE_W
:
1027 case BRW_REGISTER_TYPE_UW
:
1028 case BRW_REGISTER_TYPE_D
:
1029 case BRW_REGISTER_TYPE_UD
:
1032 unreachable("Bad type for brw_AVG");
1035 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1039 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1040 struct brw_reg src0
, struct brw_reg src1
)
1043 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1044 src0
.type
== BRW_REGISTER_TYPE_UD
||
1045 src1
.type
== BRW_REGISTER_TYPE_D
||
1046 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1047 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1050 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1051 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1052 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1053 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1054 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1057 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1058 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1059 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1060 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1061 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1064 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1065 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1066 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1067 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1069 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1073 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1074 struct brw_reg src0
, struct brw_reg src1
)
1076 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1077 src0
.width
= BRW_WIDTH_1
;
1078 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1079 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1083 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1084 struct brw_reg src0
, struct brw_reg src1
)
1086 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1087 src0
.width
= BRW_WIDTH_1
;
1088 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1089 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1090 src1
.width
= BRW_WIDTH_8
;
1091 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1092 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1096 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1098 const struct gen_device_info
*devinfo
= p
->devinfo
;
1099 const bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1100 /* The F32TO16 instruction doesn't support 32-bit destination types in
1101 * Align1 mode, and neither does the Gen8 implementation in terms of a
1102 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1103 * an undocumented feature.
1105 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1106 (!align16
|| devinfo
->gen
>= 8));
1110 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1112 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1113 dst
.type
== BRW_REGISTER_TYPE_W
||
1114 dst
.type
== BRW_REGISTER_TYPE_UW
||
1115 dst
.type
== BRW_REGISTER_TYPE_HF
);
1118 brw_push_insn_state(p
);
1120 if (needs_zero_fill
) {
1121 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1122 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1125 if (devinfo
->gen
>= 8) {
1126 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1128 assert(devinfo
->gen
== 7);
1129 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1132 if (needs_zero_fill
) {
1133 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1134 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_w(0));
1135 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1138 brw_pop_insn_state(p
);
1143 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1145 const struct gen_device_info
*devinfo
= p
->devinfo
;
1146 bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1149 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1151 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1153 * Because this instruction does not have a 16-bit floating-point
1154 * type, the source data type must be Word (W). The destination type
1155 * must be F (Float).
1157 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1158 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1160 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1161 src
.type
== BRW_REGISTER_TYPE_UW
||
1162 src
.type
== BRW_REGISTER_TYPE_HF
);
1165 if (devinfo
->gen
>= 8) {
1166 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1168 assert(devinfo
->gen
== 7);
1169 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1174 void brw_NOP(struct brw_codegen
*p
)
1176 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1177 memset(insn
, 0, sizeof(*insn
));
1178 brw_inst_set_opcode(p
->devinfo
, insn
, BRW_OPCODE_NOP
);
1185 /***********************************************************************
1186 * Comparisons, if/else/endif
1190 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1191 unsigned predicate_control
)
1193 const struct gen_device_info
*devinfo
= p
->devinfo
;
1194 struct brw_reg ip
= brw_ip_reg();
1195 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1197 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_1
);
1198 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1199 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1200 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1206 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1208 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1210 p
->if_stack_depth
++;
1211 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1212 p
->if_stack_array_size
*= 2;
1213 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1214 p
->if_stack_array_size
);
1219 pop_if_stack(struct brw_codegen
*p
)
1221 p
->if_stack_depth
--;
1222 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1226 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1228 if (p
->loop_stack_array_size
<= (p
->loop_stack_depth
+ 1)) {
1229 p
->loop_stack_array_size
*= 2;
1230 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1231 p
->loop_stack_array_size
);
1232 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1233 p
->loop_stack_array_size
);
1236 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1237 p
->loop_stack_depth
++;
1238 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1242 get_inner_do_insn(struct brw_codegen
*p
)
1244 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1247 /* EU takes the value from the flag register and pushes it onto some
1248 * sort of a stack (presumably merging with any flag value already on
1249 * the stack). Within an if block, the flags at the top of the stack
1250 * control execution on each channel of the unit, eg. on each of the
1251 * 16 pixel values in our wm programs.
1253 * When the matching 'else' instruction is reached (presumably by
1254 * countdown of the instruction count patched in by our ELSE/ENDIF
1255 * functions), the relevant flags are inverted.
1257 * When the matching 'endif' instruction is reached, the flags are
1258 * popped off. If the stack is now empty, normal execution resumes.
1261 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1263 const struct gen_device_info
*devinfo
= p
->devinfo
;
1266 insn
= next_insn(p
, BRW_OPCODE_IF
);
1268 /* Override the defaults for this instruction:
1270 if (devinfo
->gen
< 6) {
1271 brw_set_dest(p
, insn
, brw_ip_reg());
1272 brw_set_src0(p
, insn
, brw_ip_reg());
1273 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1274 } else if (devinfo
->gen
== 6) {
1275 brw_set_dest(p
, insn
, brw_imm_w(0));
1276 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1277 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1278 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1279 } else if (devinfo
->gen
== 7) {
1280 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1281 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1282 brw_set_src1(p
, insn
, brw_imm_w(0));
1283 brw_inst_set_jip(devinfo
, insn
, 0);
1284 brw_inst_set_uip(devinfo
, insn
, 0);
1286 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1287 brw_set_src0(p
, insn
, brw_imm_d(0));
1288 brw_inst_set_jip(devinfo
, insn
, 0);
1289 brw_inst_set_uip(devinfo
, insn
, 0);
1292 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1293 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1294 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1295 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1296 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1297 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1299 push_if_stack(p
, insn
);
1300 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1304 /* This function is only used for gen6-style IF instructions with an
1305 * embedded comparison (conditional modifier). It is not used on gen7.
1308 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1309 struct brw_reg src0
, struct brw_reg src1
)
1311 const struct gen_device_info
*devinfo
= p
->devinfo
;
1314 insn
= next_insn(p
, BRW_OPCODE_IF
);
1316 brw_set_dest(p
, insn
, brw_imm_w(0));
1317 brw_inst_set_exec_size(devinfo
, insn
,
1318 brw_inst_exec_size(devinfo
, p
->current
));
1319 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1320 brw_set_src0(p
, insn
, src0
);
1321 brw_set_src1(p
, insn
, src1
);
1323 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1324 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1325 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1327 push_if_stack(p
, insn
);
1332 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1335 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1336 brw_inst
*if_inst
, brw_inst
*else_inst
)
1338 const struct gen_device_info
*devinfo
= p
->devinfo
;
1340 /* The next instruction (where the ENDIF would be, if it existed) */
1341 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1343 assert(p
->single_program_flow
);
1344 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1345 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1346 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1348 /* Convert IF to an ADD instruction that moves the instruction pointer
1349 * to the first instruction of the ELSE block. If there is no ELSE
1350 * block, point to where ENDIF would be. Reverse the predicate.
1352 * There's no need to execute an ENDIF since we don't need to do any
1353 * stack operations, and if we're currently executing, we just want to
1354 * continue normally.
1356 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1357 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1359 if (else_inst
!= NULL
) {
1360 /* Convert ELSE to an ADD instruction that points where the ENDIF
1363 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1365 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1366 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1368 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1373 * Patch IF and ELSE instructions with appropriate jump targets.
1376 patch_IF_ELSE(struct brw_codegen
*p
,
1377 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1379 const struct gen_device_info
*devinfo
= p
->devinfo
;
1381 /* We shouldn't be patching IF and ELSE instructions in single program flow
1382 * mode when gen < 6, because in single program flow mode on those
1383 * platforms, we convert flow control instructions to conditional ADDs that
1384 * operate on IP (see brw_ENDIF).
1386 * However, on Gen6, writing to IP doesn't work in single program flow mode
1387 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1388 * not be updated by non-flow control instructions."). And on later
1389 * platforms, there is no significant benefit to converting control flow
1390 * instructions to conditional ADDs. So we do patch IF and ELSE
1391 * instructions in single program flow mode on those platforms.
1393 if (devinfo
->gen
< 6)
1394 assert(!p
->single_program_flow
);
1396 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1397 assert(endif_inst
!= NULL
);
1398 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1400 unsigned br
= brw_jump_scale(devinfo
);
1402 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1403 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1405 if (else_inst
== NULL
) {
1406 /* Patch IF -> ENDIF */
1407 if (devinfo
->gen
< 6) {
1408 /* Turn it into an IFF, which means no mask stack operations for
1409 * all-false and jumping past the ENDIF.
1411 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1412 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1413 br
* (endif_inst
- if_inst
+ 1));
1414 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1415 } else if (devinfo
->gen
== 6) {
1416 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1417 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1419 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1420 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1423 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1425 /* Patch IF -> ELSE */
1426 if (devinfo
->gen
< 6) {
1427 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1428 br
* (else_inst
- if_inst
));
1429 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1430 } else if (devinfo
->gen
== 6) {
1431 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1432 br
* (else_inst
- if_inst
+ 1));
1435 /* Patch ELSE -> ENDIF */
1436 if (devinfo
->gen
< 6) {
1437 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1440 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1441 br
* (endif_inst
- else_inst
+ 1));
1442 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1443 } else if (devinfo
->gen
== 6) {
1444 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1445 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1446 br
* (endif_inst
- else_inst
));
1448 /* The IF instruction's JIP should point just past the ELSE */
1449 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1450 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1451 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1452 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1453 if (devinfo
->gen
>= 8) {
1454 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1455 * should point to ENDIF.
1457 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1464 brw_ELSE(struct brw_codegen
*p
)
1466 const struct gen_device_info
*devinfo
= p
->devinfo
;
1469 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1471 if (devinfo
->gen
< 6) {
1472 brw_set_dest(p
, insn
, brw_ip_reg());
1473 brw_set_src0(p
, insn
, brw_ip_reg());
1474 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1475 } else if (devinfo
->gen
== 6) {
1476 brw_set_dest(p
, insn
, brw_imm_w(0));
1477 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1478 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1479 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1480 } else if (devinfo
->gen
== 7) {
1481 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1482 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1483 brw_set_src1(p
, insn
, brw_imm_w(0));
1484 brw_inst_set_jip(devinfo
, insn
, 0);
1485 brw_inst_set_uip(devinfo
, insn
, 0);
1487 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1488 brw_set_src0(p
, insn
, brw_imm_d(0));
1489 brw_inst_set_jip(devinfo
, insn
, 0);
1490 brw_inst_set_uip(devinfo
, insn
, 0);
1493 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1494 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1495 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1496 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1498 push_if_stack(p
, insn
);
1502 brw_ENDIF(struct brw_codegen
*p
)
1504 const struct gen_device_info
*devinfo
= p
->devinfo
;
1505 brw_inst
*insn
= NULL
;
1506 brw_inst
*else_inst
= NULL
;
1507 brw_inst
*if_inst
= NULL
;
1509 bool emit_endif
= true;
1511 /* In single program flow mode, we can express IF and ELSE instructions
1512 * equivalently as ADD instructions that operate on IP. On platforms prior
1513 * to Gen6, flow control instructions cause an implied thread switch, so
1514 * this is a significant savings.
1516 * However, on Gen6, writing to IP doesn't work in single program flow mode
1517 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1518 * not be updated by non-flow control instructions."). And on later
1519 * platforms, there is no significant benefit to converting control flow
1520 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1523 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1527 * A single next_insn() may change the base address of instruction store
1528 * memory(p->store), so call it first before referencing the instruction
1529 * store pointer from an index
1532 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1534 /* Pop the IF and (optional) ELSE instructions from the stack */
1535 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1536 tmp
= pop_if_stack(p
);
1537 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1539 tmp
= pop_if_stack(p
);
1544 /* ENDIF is useless; don't bother emitting it. */
1545 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1549 if (devinfo
->gen
< 6) {
1550 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1551 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1552 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1553 } else if (devinfo
->gen
== 6) {
1554 brw_set_dest(p
, insn
, brw_imm_w(0));
1555 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1556 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1557 } else if (devinfo
->gen
== 7) {
1558 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1559 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1560 brw_set_src1(p
, insn
, brw_imm_w(0));
1562 brw_set_src0(p
, insn
, brw_imm_d(0));
1565 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1566 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1567 if (devinfo
->gen
< 6)
1568 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1570 /* Also pop item off the stack in the endif instruction: */
1571 if (devinfo
->gen
< 6) {
1572 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1573 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1574 } else if (devinfo
->gen
== 6) {
1575 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1577 brw_inst_set_jip(devinfo
, insn
, 2);
1579 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1583 brw_BREAK(struct brw_codegen
*p
)
1585 const struct gen_device_info
*devinfo
= p
->devinfo
;
1588 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1589 if (devinfo
->gen
>= 8) {
1590 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1591 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1592 } else if (devinfo
->gen
>= 6) {
1593 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1594 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1595 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1597 brw_set_dest(p
, insn
, brw_ip_reg());
1598 brw_set_src0(p
, insn
, brw_ip_reg());
1599 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1600 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1601 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1603 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1604 brw_inst_set_exec_size(devinfo
, insn
,
1605 brw_inst_exec_size(devinfo
, p
->current
));
1611 brw_CONT(struct brw_codegen
*p
)
1613 const struct gen_device_info
*devinfo
= p
->devinfo
;
1616 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1617 brw_set_dest(p
, insn
, brw_ip_reg());
1618 if (devinfo
->gen
>= 8) {
1619 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1621 brw_set_src0(p
, insn
, brw_ip_reg());
1622 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1625 if (devinfo
->gen
< 6) {
1626 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1627 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1629 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1630 brw_inst_set_exec_size(devinfo
, insn
,
1631 brw_inst_exec_size(devinfo
, p
->current
));
1636 gen6_HALT(struct brw_codegen
*p
)
1638 const struct gen_device_info
*devinfo
= p
->devinfo
;
1641 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1642 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1643 if (devinfo
->gen
>= 8) {
1644 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1646 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1647 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1650 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1651 brw_inst_set_exec_size(devinfo
, insn
,
1652 brw_inst_exec_size(devinfo
, p
->current
));
1658 * The DO/WHILE is just an unterminated loop -- break or continue are
1659 * used for control within the loop. We have a few ways they can be
1662 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1663 * jip and no DO instruction.
1665 * For non-uniform control flow pre-gen6, there's a DO instruction to
1666 * push the mask, and a WHILE to jump back, and BREAK to get out and
1669 * For gen6, there's no more mask stack, so no need for DO. WHILE
1670 * just points back to the first instruction of the loop.
1673 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1675 const struct gen_device_info
*devinfo
= p
->devinfo
;
1677 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1678 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1679 return &p
->store
[p
->nr_insn
];
1681 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1683 push_loop_stack(p
, insn
);
1685 /* Override the defaults for this instruction:
1687 brw_set_dest(p
, insn
, brw_null_reg());
1688 brw_set_src0(p
, insn
, brw_null_reg());
1689 brw_set_src1(p
, insn
, brw_null_reg());
1691 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1692 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1693 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1700 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1703 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1704 * nesting, since it can always just point to the end of the block/current loop.
1707 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1709 const struct gen_device_info
*devinfo
= p
->devinfo
;
1710 brw_inst
*do_inst
= get_inner_do_insn(p
);
1712 unsigned br
= brw_jump_scale(devinfo
);
1714 assert(devinfo
->gen
< 6);
1716 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1717 /* If the jump count is != 0, that means that this instruction has already
1718 * been patched because it's part of a loop inside of the one we're
1721 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1722 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1723 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1724 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1725 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1726 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1732 brw_WHILE(struct brw_codegen
*p
)
1734 const struct gen_device_info
*devinfo
= p
->devinfo
;
1735 brw_inst
*insn
, *do_insn
;
1736 unsigned br
= brw_jump_scale(devinfo
);
1738 if (devinfo
->gen
>= 6) {
1739 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1740 do_insn
= get_inner_do_insn(p
);
1742 if (devinfo
->gen
>= 8) {
1743 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1744 brw_set_src0(p
, insn
, brw_imm_d(0));
1745 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1746 } else if (devinfo
->gen
== 7) {
1747 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1748 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1749 brw_set_src1(p
, insn
, brw_imm_w(0));
1750 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1752 brw_set_dest(p
, insn
, brw_imm_w(0));
1753 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1754 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1755 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1758 brw_inst_set_exec_size(devinfo
, insn
,
1759 brw_inst_exec_size(devinfo
, p
->current
));
1762 if (p
->single_program_flow
) {
1763 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1764 do_insn
= get_inner_do_insn(p
);
1766 brw_set_dest(p
, insn
, brw_ip_reg());
1767 brw_set_src0(p
, insn
, brw_ip_reg());
1768 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1769 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1771 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1772 do_insn
= get_inner_do_insn(p
);
1774 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1776 brw_set_dest(p
, insn
, brw_ip_reg());
1777 brw_set_src0(p
, insn
, brw_ip_reg());
1778 brw_set_src1(p
, insn
, brw_imm_d(0));
1780 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1781 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1782 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1784 brw_patch_break_cont(p
, insn
);
1787 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1789 p
->loop_stack_depth
--;
1796 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1798 const struct gen_device_info
*devinfo
= p
->devinfo
;
1799 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1802 if (devinfo
->gen
>= 5)
1805 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1806 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1808 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1809 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1812 /* To integrate with the above, it makes sense that the comparison
1813 * instruction should populate the flag register. It might be simpler
1814 * just to use the flag reg for most WM tasks?
1816 void brw_CMP(struct brw_codegen
*p
,
1817 struct brw_reg dest
,
1818 unsigned conditional
,
1819 struct brw_reg src0
,
1820 struct brw_reg src1
)
1822 const struct gen_device_info
*devinfo
= p
->devinfo
;
1823 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1825 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1826 brw_set_dest(p
, insn
, dest
);
1827 brw_set_src0(p
, insn
, src0
);
1828 brw_set_src1(p
, insn
, src1
);
1830 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1832 * "Any CMP instruction with a null destination must use a {switch}."
1834 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1835 * mentioned on their work-arounds pages.
1837 if (devinfo
->gen
== 7) {
1838 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1839 dest
.nr
== BRW_ARF_NULL
) {
1840 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1845 /***********************************************************************
1846 * Helpers for the various SEND message types:
1849 /** Extended math function, float[8].
1851 void gen4_math(struct brw_codegen
*p
,
1852 struct brw_reg dest
,
1854 unsigned msg_reg_nr
,
1856 unsigned precision
)
1858 const struct gen_device_info
*devinfo
= p
->devinfo
;
1859 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1861 if (has_scalar_region(src
)) {
1862 data_type
= BRW_MATH_DATA_SCALAR
;
1864 data_type
= BRW_MATH_DATA_VECTOR
;
1867 assert(devinfo
->gen
< 6);
1869 /* Example code doesn't set predicate_control for send
1872 brw_inst_set_pred_control(devinfo
, insn
, 0);
1873 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1875 brw_set_dest(p
, insn
, dest
);
1876 brw_set_src0(p
, insn
, src
);
1877 brw_set_math_message(p
,
1880 src
.type
== BRW_REGISTER_TYPE_D
,
1885 void gen6_math(struct brw_codegen
*p
,
1886 struct brw_reg dest
,
1888 struct brw_reg src0
,
1889 struct brw_reg src1
)
1891 const struct gen_device_info
*devinfo
= p
->devinfo
;
1892 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1894 assert(devinfo
->gen
>= 6);
1896 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1897 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1899 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1900 if (devinfo
->gen
== 6) {
1901 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1902 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1905 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1906 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1907 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1908 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1909 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1910 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1911 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1913 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1914 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1917 /* Source modifiers are ignored for extended math instructions on Gen6. */
1918 if (devinfo
->gen
== 6) {
1919 assert(!src0
.negate
);
1921 assert(!src1
.negate
);
1925 brw_inst_set_math_function(devinfo
, insn
, function
);
1927 brw_set_dest(p
, insn
, dest
);
1928 brw_set_src0(p
, insn
, src0
);
1929 brw_set_src1(p
, insn
, src1
);
1933 * Return the right surface index to access the thread scratch space using
1934 * stateless dataport messages.
1937 brw_scratch_surface_idx(const struct brw_codegen
*p
)
1939 /* The scratch space is thread-local so IA coherency is unnecessary. */
1940 if (p
->devinfo
->gen
>= 8)
1941 return GEN8_BTI_STATELESS_NON_COHERENT
;
1943 return BRW_BTI_STATELESS
;
1947 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1948 * using a constant offset per channel.
1950 * The offset must be aligned to oword size (16 bytes). Used for
1951 * register spilling.
1953 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1958 const struct gen_device_info
*devinfo
= p
->devinfo
;
1959 const unsigned target_cache
=
1960 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
1961 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
1962 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
1965 if (devinfo
->gen
>= 6)
1968 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1970 const unsigned mlen
= 1 + num_regs
;
1972 /* Set up the message header. This is g0, with g0.2 filled with
1973 * the offset. We don't want to leave our offset around in g0 or
1974 * it'll screw up texture samples, so set it up inside the message
1978 brw_push_insn_state(p
);
1979 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
1980 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
1981 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
1983 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
1985 /* set message header global offset field (reg 0, element 2) */
1986 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
1988 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
1990 2), BRW_REGISTER_TYPE_UD
),
1991 brw_imm_ud(offset
));
1993 brw_pop_insn_state(p
);
1997 struct brw_reg dest
;
1998 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1999 int send_commit_msg
;
2000 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2001 BRW_REGISTER_TYPE_UW
);
2003 brw_inst_set_compression(devinfo
, insn
, false);
2005 if (brw_inst_exec_size(devinfo
, insn
) >= 16)
2006 src_header
= vec16(src_header
);
2008 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2009 if (devinfo
->gen
< 6)
2010 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2012 /* Until gen6, writes followed by reads from the same location
2013 * are not guaranteed to be ordered unless write_commit is set.
2014 * If set, then a no-op write is issued to the destination
2015 * register to set a dependency, and a read from the destination
2016 * can be used to ensure the ordering.
2018 * For gen6, only writes between different threads need ordering
2019 * protection. Our use of DP writes is all about register
2020 * spilling within a thread.
2022 if (devinfo
->gen
>= 6) {
2023 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2024 send_commit_msg
= 0;
2027 send_commit_msg
= 1;
2030 brw_set_dest(p
, insn
, dest
);
2031 if (devinfo
->gen
>= 6) {
2032 brw_set_src0(p
, insn
, mrf
);
2034 brw_set_src0(p
, insn
, brw_null_reg());
2037 if (devinfo
->gen
>= 6)
2038 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2040 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2042 brw_set_dp_write_message(p
,
2044 brw_scratch_surface_idx(p
),
2045 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2049 true, /* header_present */
2050 0, /* not a render target */
2051 send_commit_msg
, /* response_length */
2059 * Read a block of owords (half a GRF each) from the scratch buffer
2060 * using a constant index per channel.
2062 * Offset must be aligned to oword size (16 bytes). Used for register
2066 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2067 struct brw_reg dest
,
2072 const struct gen_device_info
*devinfo
= p
->devinfo
;
2074 if (devinfo
->gen
>= 6)
2077 if (p
->devinfo
->gen
>= 7) {
2078 /* On gen 7 and above, we no longer have message registers and we can
2079 * send from any register we want. By using the destination register
2080 * for the message, we guarantee that the implied message write won't
2081 * accidentally overwrite anything. This has been a problem because
2082 * the MRF registers and source for the final FB write are both fixed
2085 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2087 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2089 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2091 const unsigned rlen
= num_regs
;
2092 const unsigned target_cache
=
2093 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2094 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2095 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2098 brw_push_insn_state(p
);
2099 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2100 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2101 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2103 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2105 /* set message header global offset field (reg 0, element 2) */
2106 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2107 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2109 brw_pop_insn_state(p
);
2113 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2115 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2116 brw_inst_set_compression(devinfo
, insn
, false);
2118 brw_set_dest(p
, insn
, dest
); /* UW? */
2119 if (devinfo
->gen
>= 6) {
2120 brw_set_src0(p
, insn
, mrf
);
2122 brw_set_src0(p
, insn
, brw_null_reg());
2123 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2126 brw_set_dp_read_message(p
,
2128 brw_scratch_surface_idx(p
),
2129 BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs
* 8),
2130 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2133 true, /* header_present */
2139 gen7_block_read_scratch(struct brw_codegen
*p
,
2140 struct brw_reg dest
,
2144 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2145 assert(brw_inst_pred_control(p
->devinfo
, insn
) == BRW_PREDICATE_NONE
);
2147 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2149 /* The HW requires that the header is present; this is to get the g0.5
2152 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2154 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2155 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2156 * is 32 bytes, which happens to be the size of a register.
2159 assert(offset
< (1 << 12));
2161 gen7_set_dp_scratch_message(p
, insn
,
2162 false, /* scratch read */
2164 false, /* invalidate after read */
2167 1, /* mlen: just g0 */
2168 num_regs
, /* rlen */
2169 true); /* header present */
2173 * Read float[4] vectors from the data port constant cache.
2174 * Location (in buffer) should be a multiple of 16.
2175 * Used for fetching shader constants.
2177 void brw_oword_block_read(struct brw_codegen
*p
,
2178 struct brw_reg dest
,
2181 uint32_t bind_table_index
)
2183 const struct gen_device_info
*devinfo
= p
->devinfo
;
2184 const unsigned target_cache
=
2185 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE
:
2186 BRW_DATAPORT_READ_TARGET_DATA_CACHE
);
2187 const unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, p
->current
);
2189 /* On newer hardware, offset is in units of owords. */
2190 if (devinfo
->gen
>= 6)
2193 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2195 brw_push_insn_state(p
);
2196 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2197 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2198 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2200 brw_push_insn_state(p
);
2201 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2202 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2204 /* set message header global offset field (reg 0, element 2) */
2205 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2207 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2209 2), BRW_REGISTER_TYPE_UD
),
2210 brw_imm_ud(offset
));
2211 brw_pop_insn_state(p
);
2213 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2215 /* cast dest to a uword[8] vector */
2216 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2218 brw_set_dest(p
, insn
, dest
);
2219 if (devinfo
->gen
>= 6) {
2220 brw_set_src0(p
, insn
, mrf
);
2222 brw_set_src0(p
, insn
, brw_null_reg());
2223 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2226 brw_set_dp_read_message(p
, insn
, bind_table_index
,
2227 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size
),
2228 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2231 true, /* header_present */
2232 DIV_ROUND_UP(exec_size
, 8)); /* response_length */
2234 brw_pop_insn_state(p
);
2238 void brw_fb_WRITE(struct brw_codegen
*p
,
2239 struct brw_reg payload
,
2240 struct brw_reg implied_header
,
2241 unsigned msg_control
,
2242 unsigned binding_table_index
,
2243 unsigned msg_length
,
2244 unsigned response_length
,
2246 bool last_render_target
,
2247 bool header_present
)
2249 const struct gen_device_info
*devinfo
= p
->devinfo
;
2250 const unsigned target_cache
=
2251 (devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2252 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2255 struct brw_reg dest
, src0
;
2257 if (brw_inst_exec_size(devinfo
, p
->current
) >= BRW_EXECUTE_16
)
2258 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2260 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2262 if (devinfo
->gen
>= 6) {
2263 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2265 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2267 brw_inst_set_compression(devinfo
, insn
, false);
2269 if (devinfo
->gen
>= 6) {
2270 /* headerless version, just submit color payload */
2273 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2275 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2276 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2277 src0
= implied_header
;
2279 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2282 brw_set_dest(p
, insn
, dest
);
2283 brw_set_src0(p
, insn
, src0
);
2284 brw_set_dp_write_message(p
,
2286 binding_table_index
,
2295 0 /* send_commit_msg */);
2299 gen9_fb_READ(struct brw_codegen
*p
,
2301 struct brw_reg payload
,
2302 unsigned binding_table_index
,
2303 unsigned msg_length
,
2304 unsigned response_length
,
2307 const struct gen_device_info
*devinfo
= p
->devinfo
;
2308 assert(devinfo
->gen
>= 9);
2309 const unsigned msg_subtype
=
2310 brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
? 0 : 1;
2311 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2313 brw_set_dest(p
, insn
, dst
);
2314 brw_set_src0(p
, insn
, payload
);
2315 brw_set_dp_read_message(p
, insn
, binding_table_index
,
2316 per_sample
<< 5 | msg_subtype
,
2317 GEN9_DATAPORT_RC_RENDER_TARGET_READ
,
2318 GEN6_SFID_DATAPORT_RENDER_CACHE
,
2319 msg_length
, true /* header_present */,
2321 brw_inst_set_rt_slot_group(devinfo
, insn
,
2322 brw_inst_qtr_control(devinfo
, p
->current
) / 2);
2328 * Texture sample instruction.
2329 * Note: the msg_type plus msg_length values determine exactly what kind
2330 * of sampling operation is performed. See volume 4, page 161 of docs.
2332 void brw_SAMPLE(struct brw_codegen
*p
,
2333 struct brw_reg dest
,
2334 unsigned msg_reg_nr
,
2335 struct brw_reg src0
,
2336 unsigned binding_table_index
,
2339 unsigned response_length
,
2340 unsigned msg_length
,
2341 unsigned header_present
,
2343 unsigned return_format
)
2345 const struct gen_device_info
*devinfo
= p
->devinfo
;
2348 if (msg_reg_nr
!= -1)
2349 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2351 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2352 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2354 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2356 * "Instruction compression is not allowed for this instruction (that
2357 * is, send). The hardware behavior is undefined if this instruction is
2358 * set as compressed. However, compress control can be set to "SecHalf"
2359 * to affect the EMask generation."
2361 * No similar wording is found in later PRMs, but there are examples
2362 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2363 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2364 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2366 brw_inst_set_compression(devinfo
, insn
, false);
2368 if (devinfo
->gen
< 6)
2369 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2371 brw_set_dest(p
, insn
, dest
);
2372 brw_set_src0(p
, insn
, src0
);
2373 brw_set_sampler_message(p
, insn
,
2374 binding_table_index
,
2384 /* Adjust the message header's sampler state pointer to
2385 * select the correct group of 16 samplers.
2387 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2388 struct brw_reg header
,
2389 struct brw_reg sampler_index
)
2391 /* The "Sampler Index" field can only store values between 0 and 15.
2392 * However, we can add an offset to the "Sampler State Pointer"
2393 * field, effectively selecting a different set of 16 samplers.
2395 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2396 * offset, and each sampler state is only 16-bytes, so we can't
2397 * exclusively use the offset - we have to use both.
2400 const struct gen_device_info
*devinfo
= p
->devinfo
;
2402 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2403 const int sampler_state_size
= 16; /* 16 bytes */
2404 uint32_t sampler
= sampler_index
.ud
;
2406 if (sampler
>= 16) {
2407 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2409 get_element_ud(header
, 3),
2410 get_element_ud(brw_vec8_grf(0, 0), 3),
2411 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2414 /* Non-const sampler array indexing case */
2415 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2419 struct brw_reg temp
= get_element_ud(header
, 3);
2421 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2422 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2424 get_element_ud(header
, 3),
2425 get_element_ud(brw_vec8_grf(0, 0), 3),
2430 /* All these variables are pretty confusing - we might be better off
2431 * using bitmasks and macros for this, in the old style. Or perhaps
2432 * just having the caller instantiate the fields in dword3 itself.
2434 void brw_urb_WRITE(struct brw_codegen
*p
,
2435 struct brw_reg dest
,
2436 unsigned msg_reg_nr
,
2437 struct brw_reg src0
,
2438 enum brw_urb_write_flags flags
,
2439 unsigned msg_length
,
2440 unsigned response_length
,
2444 const struct gen_device_info
*devinfo
= p
->devinfo
;
2447 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2449 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2450 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2451 brw_push_insn_state(p
);
2452 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2453 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2454 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2455 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2456 BRW_REGISTER_TYPE_UD
),
2457 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2458 brw_imm_ud(0xff00));
2459 brw_pop_insn_state(p
);
2462 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2464 assert(msg_length
< BRW_MAX_MRF(devinfo
->gen
));
2466 brw_set_dest(p
, insn
, dest
);
2467 brw_set_src0(p
, insn
, src0
);
2468 brw_set_src1(p
, insn
, brw_imm_d(0));
2470 if (devinfo
->gen
< 6)
2471 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2473 brw_set_urb_message(p
,
2483 brw_send_indirect_message(struct brw_codegen
*p
,
2486 struct brw_reg payload
,
2487 struct brw_reg desc
)
2489 const struct gen_device_info
*devinfo
= p
->devinfo
;
2490 struct brw_inst
*send
;
2493 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
2495 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2497 /* We hold on to the setup instruction (the SEND in the direct case, the OR
2498 * in the indirect case) by its index in the instruction store. The
2499 * pointer returned by next_insn() may become invalid if emitting the SEND
2500 * in the indirect case reallocs the store.
2503 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2505 send
= next_insn(p
, BRW_OPCODE_SEND
);
2506 brw_set_src1(p
, send
, desc
);
2509 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2511 brw_push_insn_state(p
);
2512 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2513 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2514 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2515 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2517 /* Load the indirect descriptor to an address register using OR so the
2518 * caller can specify additional descriptor bits with the usual
2519 * brw_set_*_message() helper functions.
2522 brw_OR(p
, addr
, desc
, brw_imm_ud(0));
2524 brw_pop_insn_state(p
);
2526 send
= next_insn(p
, BRW_OPCODE_SEND
);
2527 brw_set_src1(p
, send
, addr
);
2530 if (dst
.width
< BRW_EXECUTE_8
)
2531 brw_inst_set_exec_size(devinfo
, send
, dst
.width
);
2533 brw_set_dest(p
, send
, dst
);
2534 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2535 brw_inst_set_sfid(devinfo
, send
, sfid
);
2537 return &p
->store
[setup
];
2540 static struct brw_inst
*
2541 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2544 struct brw_reg payload
,
2545 struct brw_reg surface
,
2546 unsigned message_len
,
2547 unsigned response_len
,
2548 bool header_present
)
2550 const struct gen_device_info
*devinfo
= p
->devinfo
;
2551 struct brw_inst
*insn
;
2553 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2554 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2556 brw_push_insn_state(p
);
2557 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2558 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2559 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
2560 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2562 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2563 * some surface array is accessed out of bounds.
2565 insn
= brw_AND(p
, addr
,
2566 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2567 BRW_GET_SWZ(surface
.swizzle
, 0)),
2570 brw_pop_insn_state(p
);
2575 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
);
2576 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2577 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2578 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2584 while_jumps_before_offset(const struct gen_device_info
*devinfo
,
2585 brw_inst
*insn
, int while_offset
, int start_offset
)
2587 int scale
= 16 / brw_jump_scale(devinfo
);
2588 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2589 : brw_inst_jip(devinfo
, insn
);
2591 return while_offset
+ jip
* scale
<= start_offset
;
2596 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2599 void *store
= p
->store
;
2600 const struct gen_device_info
*devinfo
= p
->devinfo
;
2604 for (offset
= next_offset(devinfo
, store
, start_offset
);
2605 offset
< p
->next_insn_offset
;
2606 offset
= next_offset(devinfo
, store
, offset
)) {
2607 brw_inst
*insn
= store
+ offset
;
2609 switch (brw_inst_opcode(devinfo
, insn
)) {
2613 case BRW_OPCODE_ENDIF
:
2618 case BRW_OPCODE_WHILE
:
2619 /* If the while doesn't jump before our instruction, it's the end
2620 * of a sibling do...while loop. Ignore it.
2622 if (!while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2625 case BRW_OPCODE_ELSE
:
2626 case BRW_OPCODE_HALT
:
2635 /* There is no DO instruction on gen6, so to find the end of the loop
2636 * we have to see if the loop is jumping back before our start
2640 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2642 const struct gen_device_info
*devinfo
= p
->devinfo
;
2644 void *store
= p
->store
;
2646 assert(devinfo
->gen
>= 6);
2648 /* Always start after the instruction (such as a WHILE) we're trying to fix
2651 for (offset
= next_offset(devinfo
, store
, start_offset
);
2652 offset
< p
->next_insn_offset
;
2653 offset
= next_offset(devinfo
, store
, offset
)) {
2654 brw_inst
*insn
= store
+ offset
;
2656 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2657 if (while_jumps_before_offset(devinfo
, insn
, offset
, start_offset
))
2661 assert(!"not reached");
2662 return start_offset
;
2665 /* After program generation, go back and update the UIP and JIP of
2666 * BREAK, CONT, and HALT instructions to their correct locations.
2669 brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
)
2671 const struct gen_device_info
*devinfo
= p
->devinfo
;
2673 int br
= brw_jump_scale(devinfo
);
2674 int scale
= 16 / br
;
2675 void *store
= p
->store
;
2677 if (devinfo
->gen
< 6)
2680 for (offset
= start_offset
; offset
< p
->next_insn_offset
; offset
+= 16) {
2681 brw_inst
*insn
= store
+ offset
;
2682 assert(brw_inst_cmpt_control(devinfo
, insn
) == 0);
2684 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2685 switch (brw_inst_opcode(devinfo
, insn
)) {
2686 case BRW_OPCODE_BREAK
:
2687 assert(block_end_offset
!= 0);
2688 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2689 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2690 brw_inst_set_uip(devinfo
, insn
,
2691 (brw_find_loop_end(p
, offset
) - offset
+
2692 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2694 case BRW_OPCODE_CONTINUE
:
2695 assert(block_end_offset
!= 0);
2696 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2697 brw_inst_set_uip(devinfo
, insn
,
2698 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2700 assert(brw_inst_uip(devinfo
, insn
) != 0);
2701 assert(brw_inst_jip(devinfo
, insn
) != 0);
2704 case BRW_OPCODE_ENDIF
: {
2705 int32_t jump
= (block_end_offset
== 0) ?
2706 1 * br
: (block_end_offset
- offset
) / scale
;
2707 if (devinfo
->gen
>= 7)
2708 brw_inst_set_jip(devinfo
, insn
, jump
);
2710 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2714 case BRW_OPCODE_HALT
:
2715 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2717 * "In case of the halt instruction not inside any conditional
2718 * code block, the value of <JIP> and <UIP> should be the
2719 * same. In case of the halt instruction inside conditional code
2720 * block, the <UIP> should be the end of the program, and the
2721 * <JIP> should be end of the most inner conditional code block."
2723 * The uip will have already been set by whoever set up the
2726 if (block_end_offset
== 0) {
2727 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2729 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2731 assert(brw_inst_uip(devinfo
, insn
) != 0);
2732 assert(brw_inst_jip(devinfo
, insn
) != 0);
2738 void brw_ff_sync(struct brw_codegen
*p
,
2739 struct brw_reg dest
,
2740 unsigned msg_reg_nr
,
2741 struct brw_reg src0
,
2743 unsigned response_length
,
2746 const struct gen_device_info
*devinfo
= p
->devinfo
;
2749 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2751 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2752 brw_set_dest(p
, insn
, dest
);
2753 brw_set_src0(p
, insn
, src0
);
2754 brw_set_src1(p
, insn
, brw_imm_d(0));
2756 if (devinfo
->gen
< 6)
2757 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2759 brw_set_ff_sync_message(p
,
2767 * Emit the SEND instruction necessary to generate stream output data on Gen6
2768 * (for transform feedback).
2770 * If send_commit_msg is true, this is the last piece of stream output data
2771 * from this thread, so send the data as a committed write. According to the
2772 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2774 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2775 * writes are complete by sending the final write as a committed write."
2778 brw_svb_write(struct brw_codegen
*p
,
2779 struct brw_reg dest
,
2780 unsigned msg_reg_nr
,
2781 struct brw_reg src0
,
2782 unsigned binding_table_index
,
2783 bool send_commit_msg
)
2785 const struct gen_device_info
*devinfo
= p
->devinfo
;
2786 const unsigned target_cache
=
2787 (devinfo
->gen
>= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE
:
2788 devinfo
->gen
>= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE
:
2789 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
);
2792 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2794 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2795 brw_set_dest(p
, insn
, dest
);
2796 brw_set_src0(p
, insn
, src0
);
2797 brw_set_src1(p
, insn
, brw_imm_d(0));
2798 brw_set_dp_write_message(p
, insn
,
2799 binding_table_index
,
2800 0, /* msg_control: ignored */
2801 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2804 true, /* header_present */
2805 0, /* last_render_target: ignored */
2806 send_commit_msg
, /* response_length */
2807 0, /* end_of_thread */
2808 send_commit_msg
); /* send_commit_msg */
2812 brw_surface_payload_size(struct brw_codegen
*p
,
2813 unsigned num_channels
,
2818 brw_inst_access_mode(p
->devinfo
, p
->current
) == BRW_ALIGN_16
)
2820 else if (has_simd16
&&
2821 brw_inst_exec_size(p
->devinfo
, p
->current
) == BRW_EXECUTE_16
)
2822 return 2 * num_channels
;
2824 return num_channels
;
2828 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2831 bool response_expected
)
2833 const struct gen_device_info
*devinfo
= p
->devinfo
;
2834 unsigned msg_control
=
2835 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2836 (response_expected
? 1 << 5 : 0); /* Return data expected */
2838 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2839 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2840 if (brw_inst_exec_size(devinfo
, p
->current
) != BRW_EXECUTE_16
)
2841 msg_control
|= 1 << 4; /* SIMD8 mode */
2843 brw_inst_set_dp_msg_type(devinfo
, insn
,
2844 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2846 brw_inst_set_dp_msg_type(devinfo
, insn
,
2847 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2850 brw_inst_set_dp_msg_type(devinfo
, insn
,
2851 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2853 if (brw_inst_exec_size(devinfo
, p
->current
) != BRW_EXECUTE_16
)
2854 msg_control
|= 1 << 4; /* SIMD8 mode */
2857 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2861 brw_untyped_atomic(struct brw_codegen
*p
,
2863 struct brw_reg payload
,
2864 struct brw_reg surface
,
2866 unsigned msg_length
,
2867 bool response_expected
)
2869 const struct gen_device_info
*devinfo
= p
->devinfo
;
2870 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2871 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2872 GEN7_SFID_DATAPORT_DATA_CACHE
);
2873 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2874 /* Mask out unused components -- This is especially important in Align16
2875 * mode on generations that don't have native support for SIMD4x2 atomics,
2876 * because unused but enabled components will cause the dataport to perform
2877 * additional atomic operations on the addresses that happen to be in the
2878 * uninitialized Y, Z and W coordinates of the payload.
2880 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2881 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2882 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2883 brw_surface_payload_size(p
, response_expected
,
2884 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2887 brw_set_dp_untyped_atomic_message(
2888 p
, insn
, atomic_op
, response_expected
);
2892 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
2893 struct brw_inst
*insn
,
2894 unsigned num_channels
)
2896 const struct gen_device_info
*devinfo
= p
->devinfo
;
2897 /* Set mask of 32-bit channels to drop. */
2898 unsigned msg_control
= 0xf & (0xf << num_channels
);
2900 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2901 if (brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
)
2902 msg_control
|= 1 << 4; /* SIMD16 mode */
2904 msg_control
|= 2 << 4; /* SIMD8 mode */
2907 brw_inst_set_dp_msg_type(devinfo
, insn
,
2908 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2909 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2910 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
2911 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2915 brw_untyped_surface_read(struct brw_codegen
*p
,
2917 struct brw_reg payload
,
2918 struct brw_reg surface
,
2919 unsigned msg_length
,
2920 unsigned num_channels
)
2922 const struct gen_device_info
*devinfo
= p
->devinfo
;
2923 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2924 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2925 GEN7_SFID_DATAPORT_DATA_CACHE
);
2926 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2927 p
, sfid
, dst
, payload
, surface
, msg_length
,
2928 brw_surface_payload_size(p
, num_channels
, true, true),
2931 brw_set_dp_untyped_surface_read_message(
2932 p
, insn
, num_channels
);
2936 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
2937 struct brw_inst
*insn
,
2938 unsigned num_channels
)
2940 const struct gen_device_info
*devinfo
= p
->devinfo
;
2941 /* Set mask of 32-bit channels to drop. */
2942 unsigned msg_control
= 0xf & (0xf << num_channels
);
2944 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2945 if (brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
)
2946 msg_control
|= 1 << 4; /* SIMD16 mode */
2948 msg_control
|= 2 << 4; /* SIMD8 mode */
2950 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
2951 msg_control
|= 0 << 4; /* SIMD4x2 mode */
2953 msg_control
|= 2 << 4; /* SIMD8 mode */
2956 brw_inst_set_dp_msg_type(devinfo
, insn
,
2957 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2958 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
2959 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
2960 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2964 brw_untyped_surface_write(struct brw_codegen
*p
,
2965 struct brw_reg payload
,
2966 struct brw_reg surface
,
2967 unsigned msg_length
,
2968 unsigned num_channels
)
2970 const struct gen_device_info
*devinfo
= p
->devinfo
;
2971 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2972 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2973 GEN7_SFID_DATAPORT_DATA_CACHE
);
2974 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2975 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2976 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
2977 WRITEMASK_X
: WRITEMASK_XYZW
;
2978 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2979 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2980 payload
, surface
, msg_length
, 0, align1
);
2982 brw_set_dp_untyped_surface_write_message(
2983 p
, insn
, num_channels
);
2987 brw_byte_scattered_data_element_from_bit_size(unsigned bit_size
)
2991 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE
;
2993 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD
;
2995 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD
;
2997 unreachable("Unsupported bit_size for byte scattered messages");
3003 brw_byte_scattered_read(struct brw_codegen
*p
,
3005 struct brw_reg payload
,
3006 struct brw_reg surface
,
3007 unsigned msg_length
,
3010 const struct gen_device_info
*devinfo
= p
->devinfo
;
3011 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
3012 assert(brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3013 const unsigned sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
3015 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3016 p
, sfid
, dst
, payload
, surface
, msg_length
,
3017 brw_surface_payload_size(p
, 1, true, true),
3020 unsigned msg_control
=
3021 brw_byte_scattered_data_element_from_bit_size(bit_size
) << 2;
3023 if (brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
)
3024 msg_control
|= 1; /* SIMD16 mode */
3026 msg_control
|= 0; /* SIMD8 mode */
3028 brw_inst_set_dp_msg_type(devinfo
, insn
,
3029 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ
);
3030 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3034 brw_byte_scattered_write(struct brw_codegen
*p
,
3035 struct brw_reg payload
,
3036 struct brw_reg surface
,
3037 unsigned msg_length
,
3040 const struct gen_device_info
*devinfo
= p
->devinfo
;
3041 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
3042 assert(brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3043 const unsigned sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
3045 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3046 p
, sfid
, brw_writemask(brw_null_reg(), WRITEMASK_XYZW
),
3047 payload
, surface
, msg_length
, 0, true);
3049 unsigned msg_control
=
3050 brw_byte_scattered_data_element_from_bit_size(bit_size
) << 2;
3052 if (brw_inst_exec_size(devinfo
, p
->current
) == BRW_EXECUTE_16
)
3057 brw_inst_set_dp_msg_type(devinfo
, insn
,
3058 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE
);
3059 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3063 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
3064 struct brw_inst
*insn
,
3066 bool response_expected
)
3068 const struct gen_device_info
*devinfo
= p
->devinfo
;
3069 unsigned msg_control
=
3070 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
3071 (response_expected
? 1 << 5 : 0); /* Return data expected */
3073 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3074 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3075 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3076 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3078 brw_inst_set_dp_msg_type(devinfo
, insn
,
3079 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
3081 brw_inst_set_dp_msg_type(devinfo
, insn
,
3082 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
3086 brw_inst_set_dp_msg_type(devinfo
, insn
,
3087 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
3089 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3090 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
3093 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3097 brw_typed_atomic(struct brw_codegen
*p
,
3099 struct brw_reg payload
,
3100 struct brw_reg surface
,
3102 unsigned msg_length
,
3103 bool response_expected
) {
3104 const struct gen_device_info
*devinfo
= p
->devinfo
;
3105 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3106 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3107 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3108 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3109 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3110 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
3111 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3112 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
3113 brw_surface_payload_size(p
, response_expected
,
3114 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3117 brw_set_dp_typed_atomic_message(
3118 p
, insn
, atomic_op
, response_expected
);
3122 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3123 struct brw_inst
*insn
,
3124 unsigned num_channels
)
3126 const struct gen_device_info
*devinfo
= p
->devinfo
;
3127 /* Set mask of unused channels. */
3128 unsigned msg_control
= 0xf & (0xf << num_channels
);
3130 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3131 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3132 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3133 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3135 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3138 brw_inst_set_dp_msg_type(devinfo
, insn
,
3139 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3141 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3142 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3143 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3146 brw_inst_set_dp_msg_type(devinfo
, insn
,
3147 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3150 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3154 brw_typed_surface_read(struct brw_codegen
*p
,
3156 struct brw_reg payload
,
3157 struct brw_reg surface
,
3158 unsigned msg_length
,
3159 unsigned num_channels
)
3161 const struct gen_device_info
*devinfo
= p
->devinfo
;
3162 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3163 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3164 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3165 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3166 p
, sfid
, dst
, payload
, surface
, msg_length
,
3167 brw_surface_payload_size(p
, num_channels
,
3168 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3171 brw_set_dp_typed_surface_read_message(
3172 p
, insn
, num_channels
);
3176 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3177 struct brw_inst
*insn
,
3178 unsigned num_channels
)
3180 const struct gen_device_info
*devinfo
= p
->devinfo
;
3181 /* Set mask of unused channels. */
3182 unsigned msg_control
= 0xf & (0xf << num_channels
);
3184 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3185 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3186 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3187 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3189 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3192 brw_inst_set_dp_msg_type(devinfo
, insn
,
3193 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3196 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3197 if (brw_inst_qtr_control(devinfo
, p
->current
) % 2 == 1)
3198 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3201 brw_inst_set_dp_msg_type(devinfo
, insn
,
3202 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3205 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3209 brw_typed_surface_write(struct brw_codegen
*p
,
3210 struct brw_reg payload
,
3211 struct brw_reg surface
,
3212 unsigned msg_length
,
3213 unsigned num_channels
)
3215 const struct gen_device_info
*devinfo
= p
->devinfo
;
3216 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3217 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3218 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3219 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3220 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3221 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3222 WRITEMASK_X
: WRITEMASK_XYZW
);
3223 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3224 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3225 payload
, surface
, msg_length
, 0, true);
3227 brw_set_dp_typed_surface_write_message(
3228 p
, insn
, num_channels
);
3232 brw_set_memory_fence_message(struct brw_codegen
*p
,
3233 struct brw_inst
*insn
,
3234 enum brw_message_target sfid
,
3237 const struct gen_device_info
*devinfo
= p
->devinfo
;
3239 brw_set_message_descriptor(p
, insn
, sfid
,
3240 1 /* message length */,
3241 (commit_enable
? 1 : 0) /* response length */,
3242 true /* header present */,
3246 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3247 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3249 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3250 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3253 unreachable("Not reached");
3257 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3261 brw_memory_fence(struct brw_codegen
*p
,
3264 const struct gen_device_info
*devinfo
= p
->devinfo
;
3265 const bool commit_enable
= devinfo
->gen
== 7 && !devinfo
->is_haswell
;
3266 struct brw_inst
*insn
;
3268 brw_push_insn_state(p
);
3269 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3270 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3273 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3274 * message doesn't write anything back.
3276 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3277 dst
= retype(dst
, BRW_REGISTER_TYPE_UW
);
3278 brw_set_dest(p
, insn
, dst
);
3279 brw_set_src0(p
, insn
, dst
);
3280 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3283 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3284 /* IVB does typed surface access through the render cache, so we need to
3285 * flush it too. Use a different register so both flushes can be
3286 * pipelined by the hardware.
3288 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3289 brw_set_dest(p
, insn
, offset(dst
, 1));
3290 brw_set_src0(p
, insn
, offset(dst
, 1));
3291 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3294 /* Now write the response of the second message into the response of the
3295 * first to trigger a pipeline stall -- This way future render and data
3296 * cache messages will be properly ordered with respect to past data and
3297 * render cache messages.
3299 brw_MOV(p
, dst
, offset(dst
, 1));
3302 brw_pop_insn_state(p
);
3306 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3307 struct brw_reg dest
,
3311 struct brw_reg data
,
3312 unsigned msg_length
,
3313 unsigned response_length
)
3315 const struct gen_device_info
*devinfo
= p
->devinfo
;
3316 struct brw_inst
*insn
;
3317 const uint16_t exec_size
= brw_inst_exec_size(devinfo
, p
->current
);
3319 /* brw_send_indirect_message will automatically use a direct send message
3320 * if data is actually immediate.
3322 insn
= brw_send_indirect_message(p
,
3323 GEN7_SFID_PIXEL_INTERPOLATOR
,
3327 brw_inst_set_mlen(devinfo
, insn
, msg_length
);
3328 brw_inst_set_rlen(devinfo
, insn
, response_length
);
3330 brw_inst_set_pi_simd_mode(devinfo
, insn
, exec_size
== BRW_EXECUTE_16
);
3331 brw_inst_set_pi_slot_group(devinfo
, insn
, 0); /* zero unless 32/64px dispatch */
3332 brw_inst_set_pi_nopersp(devinfo
, insn
, noperspective
);
3333 brw_inst_set_pi_message_type(devinfo
, insn
, mode
);
3337 brw_find_live_channel(struct brw_codegen
*p
, struct brw_reg dst
,
3338 struct brw_reg mask
)
3340 const struct gen_device_info
*devinfo
= p
->devinfo
;
3341 const unsigned exec_size
= 1 << brw_inst_exec_size(devinfo
, p
->current
);
3342 const unsigned qtr_control
= brw_inst_qtr_control(devinfo
, p
->current
);
3345 assert(devinfo
->gen
>= 7);
3346 assert(mask
.type
== BRW_REGISTER_TYPE_UD
);
3348 brw_push_insn_state(p
);
3350 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3351 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3353 if (devinfo
->gen
>= 8) {
3354 /* Getting the first active channel index is easy on Gen8: Just find
3355 * the first bit set in the execution mask. The register exists on
3356 * HSW already but it reads back as all ones when the current
3357 * instruction has execution masking disabled, so it's kind of
3360 struct brw_reg exec_mask
=
3361 retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
);
3363 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3364 if (mask
.file
!= BRW_IMMEDIATE_VALUE
|| mask
.ud
!= 0xffffffff) {
3365 /* Unfortunately, ce0 does not take into account the thread
3366 * dispatch mask, which may be a problem in cases where it's not
3367 * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3368 * some n). Combine ce0 with the given dispatch (or vector) mask
3369 * to mask off those channels which were never dispatched by the
3372 brw_SHR(p
, vec1(dst
), mask
, brw_imm_ud(qtr_control
* 8));
3373 brw_AND(p
, vec1(dst
), exec_mask
, vec1(dst
));
3374 exec_mask
= vec1(dst
);
3377 /* Quarter control has the effect of magically shifting the value of
3378 * ce0 so you'll get the first active channel relative to the
3379 * specified quarter control as result.
3381 inst
= brw_FBL(p
, vec1(dst
), exec_mask
);
3383 const struct brw_reg flag
= brw_flag_reg(1, 0);
3385 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3386 brw_MOV(p
, retype(flag
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0));
3388 /* Run enough instructions returning zero with execution masking and
3389 * a conditional modifier enabled in order to get the full execution
3390 * mask in f1.0. We could use a single 32-wide move here if it
3391 * weren't because of the hardware bug that causes channel enables to
3392 * be applied incorrectly to the second half of 32-wide instructions
3395 const unsigned lower_size
= MIN2(16, exec_size
);
3396 for (unsigned i
= 0; i
< exec_size
/ lower_size
; i
++) {
3397 inst
= brw_MOV(p
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
),
3399 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3400 brw_inst_set_group(devinfo
, inst
, lower_size
* i
+ 8 * qtr_control
);
3401 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_Z
);
3402 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3403 brw_inst_set_exec_size(devinfo
, inst
, cvt(lower_size
) - 1);
3406 /* Find the first bit set in the exec_size-wide portion of the flag
3407 * register that was updated by the last sequence of MOV
3410 const enum brw_reg_type type
= brw_int_type(exec_size
/ 8, false);
3411 brw_set_default_exec_size(p
, BRW_EXECUTE_1
);
3412 brw_FBL(p
, vec1(dst
), byte_offset(retype(flag
, type
), qtr_control
));
3415 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3417 if (devinfo
->gen
>= 8 &&
3418 mask
.file
== BRW_IMMEDIATE_VALUE
&& mask
.ud
== 0xffffffff) {
3419 /* In SIMD4x2 mode the first active channel index is just the
3420 * negation of the first bit of the mask register. Note that ce0
3421 * doesn't take into account the dispatch mask, so the Gen7 path
3422 * should be used instead unless you have the guarantee that the
3423 * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3426 inst
= brw_AND(p
, brw_writemask(dst
, WRITEMASK_X
),
3427 negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD
)),
3431 /* Overwrite the destination without and with execution masking to
3432 * find out which of the channels is active.
3434 brw_push_insn_state(p
);
3435 brw_set_default_exec_size(p
, BRW_EXECUTE_4
);
3436 brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3439 inst
= brw_MOV(p
, brw_writemask(vec4(dst
), WRITEMASK_X
),
3441 brw_pop_insn_state(p
);
3442 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_ENABLE
);
3446 brw_pop_insn_state(p
);
3450 brw_broadcast(struct brw_codegen
*p
,
3455 const struct gen_device_info
*devinfo
= p
->devinfo
;
3456 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
3459 brw_push_insn_state(p
);
3460 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3461 brw_set_default_exec_size(p
, align1
? BRW_EXECUTE_1
: BRW_EXECUTE_4
);
3463 assert(src
.file
== BRW_GENERAL_REGISTER_FILE
&&
3464 src
.address_mode
== BRW_ADDRESS_DIRECT
);
3465 assert(!src
.abs
&& !src
.negate
);
3466 assert(src
.type
== dst
.type
);
3468 if ((src
.vstride
== 0 && (src
.hstride
== 0 || !align1
)) ||
3469 idx
.file
== BRW_IMMEDIATE_VALUE
) {
3470 /* Trivial, the source is already uniform or the index is a constant.
3471 * We will typically not get here if the optimizer is doing its job, but
3472 * asserting would be mean.
3474 const unsigned i
= idx
.file
== BRW_IMMEDIATE_VALUE
? idx
.ud
: 0;
3476 (align1
? stride(suboffset(src
, i
), 0, 1, 0) :
3477 stride(suboffset(src
, 4 * i
), 0, 4, 1)));
3479 /* From the Haswell PRM section "Register Region Restrictions":
3481 * "The lower bits of the AddressImmediate must not overflow to
3482 * change the register address. The lower 5 bits of Address
3483 * Immediate when added to lower 5 bits of address register gives
3484 * the sub-register offset. The upper bits of Address Immediate
3485 * when added to upper bits of address register gives the register
3486 * address. Any overflow from sub-register offset is dropped."
3488 * Fortunately, for broadcast, we never have a sub-register offset so
3489 * this isn't an issue.
3491 assert(src
.subnr
== 0);
3494 const struct brw_reg addr
=
3495 retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
3496 unsigned offset
= src
.nr
* REG_SIZE
+ src
.subnr
;
3497 /* Limit in bytes of the signed indirect addressing immediate. */
3498 const unsigned limit
= 512;
3500 brw_push_insn_state(p
);
3501 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3502 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
3504 /* Take into account the component size and horizontal stride. */
3505 assert(src
.vstride
== src
.hstride
+ src
.width
);
3506 brw_SHL(p
, addr
, vec1(idx
),
3507 brw_imm_ud(_mesa_logbase2(type_sz(src
.type
)) +
3510 /* We can only address up to limit bytes using the indirect
3511 * addressing immediate, account for the difference if the source
3512 * register is above this limit.
3514 if (offset
>= limit
) {
3515 brw_ADD(p
, addr
, addr
, brw_imm_ud(offset
- offset
% limit
));
3516 offset
= offset
% limit
;
3519 brw_pop_insn_state(p
);
3521 /* Use indirect addressing to fetch the specified component. */
3522 if (type_sz(src
.type
) > 4 &&
3523 (devinfo
->is_cherryview
|| gen_device_info_is_9lp(devinfo
))) {
3524 /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
3526 * "When source or destination datatype is 64b or operation is
3527 * integer DWord multiply, indirect addressing must not be
3530 * To work around both of this issue, we do two integer MOVs
3531 * insead of one 64-bit MOV. Because no double value should ever
3532 * cross a register boundary, it's safe to use the immediate
3533 * offset in the indirect here to handle adding 4 bytes to the
3534 * offset and avoid the extra ADD to the register file.
3536 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 0),
3537 retype(brw_vec1_indirect(addr
.subnr
, offset
),
3538 BRW_REGISTER_TYPE_D
));
3539 brw_MOV(p
, subscript(dst
, BRW_REGISTER_TYPE_D
, 1),
3540 retype(brw_vec1_indirect(addr
.subnr
, offset
+ 4),
3541 BRW_REGISTER_TYPE_D
));
3544 retype(brw_vec1_indirect(addr
.subnr
, offset
), src
.type
));
3547 /* In SIMD4x2 mode the index can be either zero or one, replicate it
3548 * to all bits of a flag register,
3552 stride(brw_swizzle(idx
, BRW_SWIZZLE_XXXX
), 4, 4, 1));
3553 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NONE
);
3554 brw_inst_set_cond_modifier(devinfo
, inst
, BRW_CONDITIONAL_NZ
);
3555 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3557 /* and use predicated SEL to pick the right channel. */
3558 inst
= brw_SEL(p
, dst
,
3559 stride(suboffset(src
, 4), 4, 4, 1),
3560 stride(src
, 4, 4, 1));
3561 brw_inst_set_pred_control(devinfo
, inst
, BRW_PREDICATE_NORMAL
);
3562 brw_inst_set_flag_reg_nr(devinfo
, inst
, 1);
3566 brw_pop_insn_state(p
);
3570 * This instruction is generated as a single-channel align1 instruction by
3571 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3573 * We can't use the typed atomic op in the FS because that has the execution
3574 * mask ANDed with the pixel mask, but we just want to write the one dword for
3577 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3578 * one u32. So we use the same untyped atomic write message as the pixel
3581 * The untyped atomic operation requires a BUFFER surface type with RAW
3582 * format, and is only accessible through the legacy DATA_CACHE dataport
3585 void brw_shader_time_add(struct brw_codegen
*p
,
3586 struct brw_reg payload
,
3587 uint32_t surf_index
)
3589 const unsigned sfid
= (p
->devinfo
->gen
>= 8 || p
->devinfo
->is_haswell
?
3590 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3591 GEN7_SFID_DATAPORT_DATA_CACHE
);
3592 assert(p
->devinfo
->gen
>= 7);
3594 brw_push_insn_state(p
);
3595 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3596 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3597 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3598 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3600 /* We use brw_vec1_reg and unmasked because we want to increment the given
3603 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3605 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3607 brw_set_src1(p
, send
, brw_imm_ud(0));
3608 brw_set_message_descriptor(p
, send
, sfid
, 2, 0, false, false);
3609 brw_inst_set_binding_table_index(p
->devinfo
, send
, surf_index
);
3610 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3612 brw_pop_insn_state(p
);
3617 * Emit the SEND message for a barrier
3620 brw_barrier(struct brw_codegen
*p
, struct brw_reg src
)
3622 const struct gen_device_info
*devinfo
= p
->devinfo
;
3623 struct brw_inst
*inst
;
3625 assert(devinfo
->gen
>= 7);
3627 brw_push_insn_state(p
);
3628 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3629 inst
= next_insn(p
, BRW_OPCODE_SEND
);
3630 brw_set_dest(p
, inst
, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW
));
3631 brw_set_src0(p
, inst
, src
);
3632 brw_set_src1(p
, inst
, brw_null_reg());
3634 brw_set_message_descriptor(p
, inst
, BRW_SFID_MESSAGE_GATEWAY
,
3636 0 /* response_length */,
3637 false /* header_present */,
3638 false /* end_of_thread */);
3640 brw_inst_set_gateway_notify(devinfo
, inst
, 1);
3641 brw_inst_set_gateway_subfuncid(devinfo
, inst
,
3642 BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG
);
3644 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
3645 brw_pop_insn_state(p
);
3650 * Emit the wait instruction for a barrier
3653 brw_WAIT(struct brw_codegen
*p
)
3655 const struct gen_device_info
*devinfo
= p
->devinfo
;
3656 struct brw_inst
*insn
;
3658 struct brw_reg src
= brw_notification_reg();
3660 insn
= next_insn(p
, BRW_OPCODE_WAIT
);
3661 brw_set_dest(p
, insn
, src
);
3662 brw_set_src0(p
, insn
, src
);
3663 brw_set_src1(p
, insn
, brw_null_reg());
3665 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
3666 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_DISABLE
);
3670 * Changes the floating point rounding mode updating the control register
3671 * field defined at cr0.0[5-6] bits. This function supports the changes to
3672 * RTNE (00), RU (01), RD (10) and RTZ (11) rounding using bitwise operations.
3673 * Only RTNE and RTZ rounding are enabled at nir.
3676 brw_rounding_mode(struct brw_codegen
*p
,
3677 enum brw_rnd_mode mode
)
3679 const unsigned bits
= mode
<< BRW_CR0_RND_MODE_SHIFT
;
3681 if (bits
!= BRW_CR0_RND_MODE_MASK
) {
3682 brw_inst
*inst
= brw_AND(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3683 brw_imm_ud(~BRW_CR0_RND_MODE_MASK
));
3685 /* From the Skylake PRM, Volume 7, page 760:
3686 * "Implementation Restriction on Register Access: When the control
3687 * register is used as an explicit source and/or destination, hardware
3688 * does not ensure execution pipeline coherency. Software must set the
3689 * thread control field to ‘switch’ for an instruction that uses
3690 * control register as an explicit operand."
3692 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);
3696 brw_inst
*inst
= brw_OR(p
, brw_cr0_reg(0), brw_cr0_reg(0),
3698 brw_inst_set_thread_control(p
->devinfo
, inst
, BRW_THREAD_SWITCH
);