2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_context.h"
34 #include "brw_defines.h"
37 #include "util/ralloc.h"
40 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
41 * registers, implicitly moving the operand to a message register.
43 * On Sandybridge, this is no longer the case. This function performs the
44 * explicit move; it should be called before emitting a SEND instruction.
47 gen6_resolve_implied_move(struct brw_codegen
*p
,
51 const struct brw_device_info
*devinfo
= p
->devinfo
;
55 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
58 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
59 brw_push_insn_state(p
);
60 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
61 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
62 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
63 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
64 retype(*src
, BRW_REGISTER_TYPE_UD
));
65 brw_pop_insn_state(p
);
67 *src
= brw_message_reg(msg_reg_nr
);
71 gen7_convert_mrf_to_grf(struct brw_codegen
*p
, struct brw_reg
*reg
)
73 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74 * "The send with EOT should use register space R112-R127 for <src>. This is
75 * to enable loading of a new thread into the same slot while the message
76 * with EOT for current thread is pending dispatch."
78 * Since we're pretending to have 16 MRFs anyway, we may as well use the
79 * registers required for messages with EOT.
81 const struct brw_device_info
*devinfo
= p
->devinfo
;
82 if (devinfo
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
83 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
84 reg
->nr
+= GEN7_MRF_HACK_START
;
89 * Convert a brw_reg_type enumeration value into the hardware representation.
91 * The hardware encoding may depend on whether the value is an immediate.
94 brw_reg_type_to_hw_type(const struct brw_device_info
*devinfo
,
95 enum brw_reg_type type
, unsigned file
)
97 if (file
== BRW_IMMEDIATE_VALUE
) {
98 const static int imm_hw_types
[] = {
99 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
100 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
101 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
102 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
103 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
104 [BRW_REGISTER_TYPE_UB
] = -1,
105 [BRW_REGISTER_TYPE_B
] = -1,
106 [BRW_REGISTER_TYPE_UV
] = BRW_HW_REG_IMM_TYPE_UV
,
107 [BRW_REGISTER_TYPE_VF
] = BRW_HW_REG_IMM_TYPE_VF
,
108 [BRW_REGISTER_TYPE_V
] = BRW_HW_REG_IMM_TYPE_V
,
109 [BRW_REGISTER_TYPE_DF
] = GEN8_HW_REG_IMM_TYPE_DF
,
110 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_IMM_TYPE_HF
,
111 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
112 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
114 assert(type
< ARRAY_SIZE(imm_hw_types
));
115 assert(imm_hw_types
[type
] != -1);
116 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_DF
);
117 return imm_hw_types
[type
];
119 /* Non-immediate registers */
120 const static int hw_types
[] = {
121 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
122 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
123 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
124 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
125 [BRW_REGISTER_TYPE_UB
] = BRW_HW_REG_NON_IMM_TYPE_UB
,
126 [BRW_REGISTER_TYPE_B
] = BRW_HW_REG_NON_IMM_TYPE_B
,
127 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
128 [BRW_REGISTER_TYPE_UV
] = -1,
129 [BRW_REGISTER_TYPE_VF
] = -1,
130 [BRW_REGISTER_TYPE_V
] = -1,
131 [BRW_REGISTER_TYPE_DF
] = GEN7_HW_REG_NON_IMM_TYPE_DF
,
132 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_NON_IMM_TYPE_HF
,
133 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
134 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
136 assert(type
< ARRAY_SIZE(hw_types
));
137 assert(hw_types
[type
] != -1);
138 assert(devinfo
->gen
>= 7 || type
< BRW_REGISTER_TYPE_DF
);
139 assert(devinfo
->gen
>= 8 || type
< BRW_REGISTER_TYPE_HF
);
140 return hw_types
[type
];
145 brw_set_dest(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg dest
)
147 const struct brw_device_info
*devinfo
= p
->devinfo
;
149 if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
&&
150 dest
.file
!= BRW_MESSAGE_REGISTER_FILE
)
151 assert(dest
.nr
< 128);
153 gen7_convert_mrf_to_grf(p
, &dest
);
155 brw_inst_set_dst_reg_file(devinfo
, inst
, dest
.file
);
156 brw_inst_set_dst_reg_type(devinfo
, inst
,
157 brw_reg_type_to_hw_type(devinfo
, dest
.type
,
159 brw_inst_set_dst_address_mode(devinfo
, inst
, dest
.address_mode
);
161 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
162 brw_inst_set_dst_da_reg_nr(devinfo
, inst
, dest
.nr
);
164 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
165 brw_inst_set_dst_da1_subreg_nr(devinfo
, inst
, dest
.subnr
);
166 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
167 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
168 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
170 brw_inst_set_dst_da16_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
171 brw_inst_set_da16_writemask(devinfo
, inst
, dest
.dw1
.bits
.writemask
);
172 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
173 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
174 assert(dest
.dw1
.bits
.writemask
!= 0);
176 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
177 * Although Dst.HorzStride is a don't care for Align16, HW needs
178 * this to be programmed as "01".
180 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
183 brw_inst_set_dst_ia_subreg_nr(devinfo
, inst
, dest
.subnr
);
185 /* These are different sizes in align1 vs align16:
187 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
188 brw_inst_set_dst_ia1_addr_imm(devinfo
, inst
,
189 dest
.dw1
.bits
.indirect_offset
);
190 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
191 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
192 brw_inst_set_dst_hstride(devinfo
, inst
, dest
.hstride
);
194 brw_inst_set_dst_ia16_addr_imm(devinfo
, inst
,
195 dest
.dw1
.bits
.indirect_offset
);
196 /* even ignored in da16, still need to set as '01' */
197 brw_inst_set_dst_hstride(devinfo
, inst
, 1);
201 /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
202 * or 16 (SIMD16), as that's normally correct. However, when dealing with
203 * small registers, we automatically reduce it to match the register size.
205 if (dest
.width
< BRW_EXECUTE_8
)
206 brw_inst_set_exec_size(devinfo
, inst
, dest
.width
);
209 extern int reg_type_size
[];
212 validate_reg(const struct brw_device_info
*devinfo
,
213 brw_inst
*inst
, struct brw_reg reg
)
215 const int hstride_for_reg
[] = {0, 1, 2, 4};
216 const int vstride_for_reg
[] = {0, 1, 2, 4, 8, 16, 32};
217 const int width_for_reg
[] = {1, 2, 4, 8, 16};
218 const int execsize_for_reg
[] = {1, 2, 4, 8, 16, 32};
219 int width
, hstride
, vstride
, execsize
;
221 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
222 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
223 * mean the destination has to be 128-bit aligned and the
224 * destination horiz stride has to be a word.
226 if (reg
.type
== BRW_REGISTER_TYPE_V
) {
227 assert(hstride_for_reg
[brw_inst_dst_hstride(devinfo
, inst
)] *
228 reg_type_size
[brw_inst_dst_reg_type(devinfo
, inst
)] == 2);
234 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
235 reg
.file
== BRW_ARF_NULL
)
238 assert(reg
.hstride
>= 0 && reg
.hstride
< ARRAY_SIZE(hstride_for_reg
));
239 hstride
= hstride_for_reg
[reg
.hstride
];
241 if (reg
.vstride
== 0xf) {
244 assert(reg
.vstride
>= 0 && reg
.vstride
< ARRAY_SIZE(vstride_for_reg
));
245 vstride
= vstride_for_reg
[reg
.vstride
];
248 assert(reg
.width
>= 0 && reg
.width
< ARRAY_SIZE(width_for_reg
));
249 width
= width_for_reg
[reg
.width
];
251 assert(brw_inst_exec_size(devinfo
, inst
) >= 0 &&
252 brw_inst_exec_size(devinfo
, inst
) < ARRAY_SIZE(execsize_for_reg
));
253 execsize
= execsize_for_reg
[brw_inst_exec_size(devinfo
, inst
)];
255 /* Restrictions from 3.3.10: Register Region Restrictions. */
257 assert(execsize
>= width
);
260 if (execsize
== width
&& hstride
!= 0) {
261 assert(vstride
== -1 || vstride
== width
* hstride
);
265 if (execsize
== width
&& hstride
== 0) {
266 /* no restriction on vstride. */
271 assert(hstride
== 0);
275 if (execsize
== 1 && width
== 1) {
276 assert(hstride
== 0);
277 assert(vstride
== 0);
281 if (vstride
== 0 && hstride
== 0) {
285 /* 10. Check destination issues. */
289 is_compactable_immediate(unsigned imm
)
291 /* We get the low 12 bits as-is. */
294 /* We get one bit replicated through the top 20 bits. */
295 return imm
== 0 || imm
== 0xfffff000;
299 brw_set_src0(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
301 const struct brw_device_info
*devinfo
= p
->devinfo
;
303 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
304 assert(reg
.nr
< 128);
306 gen7_convert_mrf_to_grf(p
, ®
);
308 if (devinfo
->gen
>= 6 && (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SEND
||
309 brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_SENDC
)) {
310 /* Any source modifiers or regions will be ignored, since this just
311 * identifies the MRF/GRF to start reading the message contents from.
312 * Check for some likely failures.
316 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
319 validate_reg(devinfo
, inst
, reg
);
321 brw_inst_set_src0_reg_file(devinfo
, inst
, reg
.file
);
322 brw_inst_set_src0_reg_type(devinfo
, inst
,
323 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
324 brw_inst_set_src0_abs(devinfo
, inst
, reg
.abs
);
325 brw_inst_set_src0_negate(devinfo
, inst
, reg
.negate
);
326 brw_inst_set_src0_address_mode(devinfo
, inst
, reg
.address_mode
);
328 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
329 brw_inst_set_imm_ud(devinfo
, inst
, reg
.dw1
.ud
);
331 /* The Bspec's section titled "Non-present Operands" claims that if src0
332 * is an immediate that src1's type must be the same as that of src0.
334 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
335 * that do not follow this rule. E.g., from the IVB/HSW table:
337 * DataTypeIndex 18-Bit Mapping Mapped Meaning
338 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
340 * And from the SNB table:
342 * DataTypeIndex 18-Bit Mapping Mapped Meaning
343 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
345 * Neither of these cause warnings from the simulator when used,
346 * compacted or otherwise. In fact, all compaction mappings that have an
347 * immediate in src0 use a:ud for src1.
349 * The GM45 instruction compaction tables do not contain mapped meanings
350 * so it's not clear whether it has the restriction. We'll assume it was
351 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
353 brw_inst_set_src1_reg_file(devinfo
, inst
, BRW_ARCHITECTURE_REGISTER_FILE
);
354 if (devinfo
->gen
< 6) {
355 brw_inst_set_src1_reg_type(devinfo
, inst
,
356 brw_inst_src0_reg_type(devinfo
, inst
));
358 brw_inst_set_src1_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
361 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
362 * for immediate values. Presumably the hardware engineers realized
363 * that the only useful floating-point value that could be represented
364 * in this format is 0.0, which can also be represented as a VF-typed
365 * immediate, so they gave us the previously mentioned mapping on IVB+.
367 * Strangely, we do have a mapping for imm:f in src1, so we don't need
370 * If we see a 0.0:F, change the type to VF so that it can be compacted.
372 if (brw_inst_imm_ud(devinfo
, inst
) == 0x0 &&
373 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_F
) {
374 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_IMM_TYPE_VF
);
377 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
378 * set the types to :UD so the instruction can be compacted.
380 if (is_compactable_immediate(brw_inst_imm_ud(devinfo
, inst
)) &&
381 brw_inst_cond_modifier(devinfo
, inst
) == BRW_CONDITIONAL_NONE
&&
382 brw_inst_src0_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
&&
383 brw_inst_dst_reg_type(devinfo
, inst
) == BRW_HW_REG_TYPE_D
) {
384 brw_inst_set_src0_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
385 brw_inst_set_dst_reg_type(devinfo
, inst
, BRW_HW_REG_TYPE_UD
);
388 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
389 brw_inst_set_src0_da_reg_nr(devinfo
, inst
, reg
.nr
);
390 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
391 brw_inst_set_src0_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
393 brw_inst_set_src0_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
396 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.subnr
);
398 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
399 brw_inst_set_src0_ia1_addr_imm(devinfo
, inst
, reg
.dw1
.bits
.indirect_offset
);
401 brw_inst_set_src0_ia_subreg_nr(devinfo
, inst
, reg
.dw1
.bits
.indirect_offset
);
405 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
406 if (reg
.width
== BRW_WIDTH_1
&&
407 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
408 brw_inst_set_src0_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
409 brw_inst_set_src0_width(devinfo
, inst
, BRW_WIDTH_1
);
410 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
412 brw_inst_set_src0_hstride(devinfo
, inst
, reg
.hstride
);
413 brw_inst_set_src0_width(devinfo
, inst
, reg
.width
);
414 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
417 brw_inst_set_src0_da16_swiz_x(devinfo
, inst
,
418 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_X
));
419 brw_inst_set_src0_da16_swiz_y(devinfo
, inst
,
420 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Y
));
421 brw_inst_set_src0_da16_swiz_z(devinfo
, inst
,
422 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Z
));
423 brw_inst_set_src0_da16_swiz_w(devinfo
, inst
,
424 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_W
));
426 /* This is an oddity of the fact we're using the same
427 * descriptions for registers in align_16 as align_1:
429 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
430 brw_inst_set_src0_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
432 brw_inst_set_src0_vstride(devinfo
, inst
, reg
.vstride
);
439 brw_set_src1(struct brw_codegen
*p
, brw_inst
*inst
, struct brw_reg reg
)
441 const struct brw_device_info
*devinfo
= p
->devinfo
;
443 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
444 assert(reg
.nr
< 128);
446 gen7_convert_mrf_to_grf(p
, ®
);
447 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
449 validate_reg(devinfo
, inst
, reg
);
451 brw_inst_set_src1_reg_file(devinfo
, inst
, reg
.file
);
452 brw_inst_set_src1_reg_type(devinfo
, inst
,
453 brw_reg_type_to_hw_type(devinfo
, reg
.type
, reg
.file
));
454 brw_inst_set_src1_abs(devinfo
, inst
, reg
.abs
);
455 brw_inst_set_src1_negate(devinfo
, inst
, reg
.negate
);
457 /* Only src1 can be immediate in two-argument instructions.
459 assert(brw_inst_src0_reg_file(devinfo
, inst
) != BRW_IMMEDIATE_VALUE
);
461 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
462 brw_inst_set_imm_ud(devinfo
, inst
, reg
.dw1
.ud
);
464 /* This is a hardware restriction, which may or may not be lifted
467 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
468 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
470 brw_inst_set_src1_da_reg_nr(devinfo
, inst
, reg
.nr
);
471 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
472 brw_inst_set_src1_da1_subreg_nr(devinfo
, inst
, reg
.subnr
);
474 brw_inst_set_src1_da16_subreg_nr(devinfo
, inst
, reg
.subnr
/ 16);
477 if (brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_1
) {
478 if (reg
.width
== BRW_WIDTH_1
&&
479 brw_inst_exec_size(devinfo
, inst
) == BRW_EXECUTE_1
) {
480 brw_inst_set_src1_hstride(devinfo
, inst
, BRW_HORIZONTAL_STRIDE_0
);
481 brw_inst_set_src1_width(devinfo
, inst
, BRW_WIDTH_1
);
482 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_0
);
484 brw_inst_set_src1_hstride(devinfo
, inst
, reg
.hstride
);
485 brw_inst_set_src1_width(devinfo
, inst
, reg
.width
);
486 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
489 brw_inst_set_src1_da16_swiz_x(devinfo
, inst
,
490 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_X
));
491 brw_inst_set_src1_da16_swiz_y(devinfo
, inst
,
492 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Y
));
493 brw_inst_set_src1_da16_swiz_z(devinfo
, inst
,
494 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Z
));
495 brw_inst_set_src1_da16_swiz_w(devinfo
, inst
,
496 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_W
));
498 /* This is an oddity of the fact we're using the same
499 * descriptions for registers in align_16 as align_1:
501 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
502 brw_inst_set_src1_vstride(devinfo
, inst
, BRW_VERTICAL_STRIDE_4
);
504 brw_inst_set_src1_vstride(devinfo
, inst
, reg
.vstride
);
510 * Set the Message Descriptor and Extended Message Descriptor fields
513 * \note This zeroes out the Function Control bits, so it must be called
514 * \b before filling out any message-specific data. Callers can
515 * choose not to fill in irrelevant bits; they will be zero.
518 brw_set_message_descriptor(struct brw_codegen
*p
,
520 enum brw_message_target sfid
,
522 unsigned response_length
,
526 const struct brw_device_info
*devinfo
= p
->devinfo
;
528 brw_set_src1(p
, inst
, brw_imm_d(0));
530 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
531 * itself; instead, it will be a MOV/OR into the address register.
533 * In this case, we avoid setting the extended message descriptor bits,
534 * since they go on the later SEND/SENDC instead and if set here would
535 * instead clobber the conditionalmod bits.
537 unsigned opcode
= brw_inst_opcode(devinfo
, inst
);
538 if (opcode
== BRW_OPCODE_SEND
|| opcode
== BRW_OPCODE_SENDC
) {
539 brw_inst_set_sfid(devinfo
, inst
, sfid
);
542 brw_inst_set_mlen(devinfo
, inst
, msg_length
);
543 brw_inst_set_rlen(devinfo
, inst
, response_length
);
544 brw_inst_set_eot(devinfo
, inst
, end_of_thread
);
546 if (devinfo
->gen
>= 5) {
547 brw_inst_set_header_present(devinfo
, inst
, header_present
);
551 static void brw_set_math_message( struct brw_codegen
*p
,
554 unsigned integer_type
,
558 const struct brw_device_info
*devinfo
= p
->devinfo
;
560 unsigned response_length
;
562 /* Infer message length from the function */
564 case BRW_MATH_FUNCTION_POW
:
565 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
566 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
567 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
575 /* Infer response length from the function */
577 case BRW_MATH_FUNCTION_SINCOS
:
578 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
587 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
588 msg_length
, response_length
, false, false);
589 brw_inst_set_math_msg_function(devinfo
, inst
, function
);
590 brw_inst_set_math_msg_signed_int(devinfo
, inst
, integer_type
);
591 brw_inst_set_math_msg_precision(devinfo
, inst
, low_precision
);
592 brw_inst_set_math_msg_saturate(devinfo
, inst
, brw_inst_saturate(devinfo
, inst
));
593 brw_inst_set_math_msg_data_type(devinfo
, inst
, dataType
);
594 brw_inst_set_saturate(devinfo
, inst
, 0);
598 static void brw_set_ff_sync_message(struct brw_codegen
*p
,
601 unsigned response_length
,
604 const struct brw_device_info
*devinfo
= p
->devinfo
;
606 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
607 1, response_length
, true, end_of_thread
);
608 brw_inst_set_urb_opcode(devinfo
, insn
, 1); /* FF_SYNC */
609 brw_inst_set_urb_allocate(devinfo
, insn
, allocate
);
610 /* The following fields are not used by FF_SYNC: */
611 brw_inst_set_urb_global_offset(devinfo
, insn
, 0);
612 brw_inst_set_urb_swizzle_control(devinfo
, insn
, 0);
613 brw_inst_set_urb_used(devinfo
, insn
, 0);
614 brw_inst_set_urb_complete(devinfo
, insn
, 0);
617 static void brw_set_urb_message( struct brw_codegen
*p
,
619 enum brw_urb_write_flags flags
,
621 unsigned response_length
,
623 unsigned swizzle_control
)
625 const struct brw_device_info
*devinfo
= p
->devinfo
;
627 assert(devinfo
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
628 assert(devinfo
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
629 assert(devinfo
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
631 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
632 msg_length
, response_length
, true,
633 flags
& BRW_URB_WRITE_EOT
);
635 if (flags
& BRW_URB_WRITE_OWORD
) {
636 assert(msg_length
== 2); /* header + one OWORD of data */
637 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
639 brw_inst_set_urb_opcode(devinfo
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
642 brw_inst_set_urb_global_offset(devinfo
, insn
, offset
);
643 brw_inst_set_urb_swizzle_control(devinfo
, insn
, swizzle_control
);
645 if (devinfo
->gen
< 8) {
646 brw_inst_set_urb_complete(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
649 if (devinfo
->gen
< 7) {
650 brw_inst_set_urb_allocate(devinfo
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
651 brw_inst_set_urb_used(devinfo
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
653 brw_inst_set_urb_per_slot_offset(devinfo
, insn
,
654 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
659 brw_set_dp_write_message(struct brw_codegen
*p
,
661 unsigned binding_table_index
,
662 unsigned msg_control
,
666 unsigned last_render_target
,
667 unsigned response_length
,
668 unsigned end_of_thread
,
669 unsigned send_commit_msg
)
671 const struct brw_device_info
*devinfo
= p
->devinfo
;
674 if (devinfo
->gen
>= 7) {
675 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
676 if (msg_type
== GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
)
677 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
679 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
680 } else if (devinfo
->gen
== 6) {
681 /* Use the render cache for all write messages. */
682 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
684 sfid
= BRW_SFID_DATAPORT_WRITE
;
687 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
688 header_present
, end_of_thread
);
690 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
691 brw_inst_set_dp_write_msg_type(devinfo
, insn
, msg_type
);
692 brw_inst_set_dp_write_msg_control(devinfo
, insn
, msg_control
);
693 brw_inst_set_rt_last(devinfo
, insn
, last_render_target
);
694 if (devinfo
->gen
< 7) {
695 brw_inst_set_dp_write_commit(devinfo
, insn
, send_commit_msg
);
700 brw_set_dp_read_message(struct brw_codegen
*p
,
702 unsigned binding_table_index
,
703 unsigned msg_control
,
705 unsigned target_cache
,
708 unsigned response_length
)
710 const struct brw_device_info
*devinfo
= p
->devinfo
;
713 if (devinfo
->gen
>= 7) {
714 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
715 } else if (devinfo
->gen
== 6) {
716 if (target_cache
== BRW_DATAPORT_READ_TARGET_RENDER_CACHE
)
717 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
719 sfid
= GEN6_SFID_DATAPORT_SAMPLER_CACHE
;
721 sfid
= BRW_SFID_DATAPORT_READ
;
724 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
725 header_present
, false);
727 brw_inst_set_binding_table_index(devinfo
, insn
, binding_table_index
);
728 brw_inst_set_dp_read_msg_type(devinfo
, insn
, msg_type
);
729 brw_inst_set_dp_read_msg_control(devinfo
, insn
, msg_control
);
730 if (devinfo
->gen
< 6)
731 brw_inst_set_dp_read_target_cache(devinfo
, insn
, target_cache
);
735 brw_set_sampler_message(struct brw_codegen
*p
,
737 unsigned binding_table_index
,
740 unsigned response_length
,
742 unsigned header_present
,
744 unsigned return_format
)
746 const struct brw_device_info
*devinfo
= p
->devinfo
;
748 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
749 response_length
, header_present
, false);
751 brw_inst_set_binding_table_index(devinfo
, inst
, binding_table_index
);
752 brw_inst_set_sampler(devinfo
, inst
, sampler
);
753 brw_inst_set_sampler_msg_type(devinfo
, inst
, msg_type
);
754 if (devinfo
->gen
>= 5) {
755 brw_inst_set_sampler_simd_mode(devinfo
, inst
, simd_mode
);
756 } else if (devinfo
->gen
== 4 && !devinfo
->is_g4x
) {
757 brw_inst_set_sampler_return_format(devinfo
, inst
, return_format
);
762 gen7_set_dp_scratch_message(struct brw_codegen
*p
,
766 bool invalidate_after_read
,
768 unsigned addr_offset
,
773 const struct brw_device_info
*devinfo
= p
->devinfo
;
774 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
775 (devinfo
->gen
>= 8 && num_regs
== 8));
776 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
777 mlen
, rlen
, header_present
, false);
778 brw_inst_set_dp_category(devinfo
, inst
, 1); /* Scratch Block Read/Write msgs */
779 brw_inst_set_scratch_read_write(devinfo
, inst
, write
);
780 brw_inst_set_scratch_type(devinfo
, inst
, dword
);
781 brw_inst_set_scratch_invalidate_after_read(devinfo
, inst
, invalidate_after_read
);
782 brw_inst_set_scratch_block_size(devinfo
, inst
, ffs(num_regs
) - 1);
783 brw_inst_set_scratch_addr_offset(devinfo
, inst
, addr_offset
);
786 #define next_insn brw_next_insn
788 brw_next_insn(struct brw_codegen
*p
, unsigned opcode
)
790 const struct brw_device_info
*devinfo
= p
->devinfo
;
793 if (p
->nr_insn
+ 1 > p
->store_size
) {
795 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
798 p
->next_insn_offset
+= 16;
799 insn
= &p
->store
[p
->nr_insn
++];
800 memcpy(insn
, p
->current
, sizeof(*insn
));
802 brw_inst_set_opcode(devinfo
, insn
, opcode
);
807 brw_alu1(struct brw_codegen
*p
, unsigned opcode
,
808 struct brw_reg dest
, struct brw_reg src
)
810 brw_inst
*insn
= next_insn(p
, opcode
);
811 brw_set_dest(p
, insn
, dest
);
812 brw_set_src0(p
, insn
, src
);
817 brw_alu2(struct brw_codegen
*p
, unsigned opcode
,
818 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
820 brw_inst
*insn
= next_insn(p
, opcode
);
821 brw_set_dest(p
, insn
, dest
);
822 brw_set_src0(p
, insn
, src0
);
823 brw_set_src1(p
, insn
, src1
);
828 get_3src_subreg_nr(struct brw_reg reg
)
830 if (reg
.vstride
== BRW_VERTICAL_STRIDE_0
) {
831 assert(brw_is_single_value_swizzle(reg
.dw1
.bits
.swizzle
));
832 return reg
.subnr
/ 4 + BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, 0);
834 return reg
.subnr
/ 4;
839 brw_alu3(struct brw_codegen
*p
, unsigned opcode
, struct brw_reg dest
,
840 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
842 const struct brw_device_info
*devinfo
= p
->devinfo
;
843 brw_inst
*inst
= next_insn(p
, opcode
);
845 gen7_convert_mrf_to_grf(p
, &dest
);
847 assert(brw_inst_access_mode(devinfo
, inst
) == BRW_ALIGN_16
);
849 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
850 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
851 assert(dest
.nr
< 128);
852 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
853 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
854 dest
.type
== BRW_REGISTER_TYPE_D
||
855 dest
.type
== BRW_REGISTER_TYPE_UD
);
856 if (devinfo
->gen
== 6) {
857 brw_inst_set_3src_dst_reg_file(devinfo
, inst
,
858 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
860 brw_inst_set_3src_dst_reg_nr(devinfo
, inst
, dest
.nr
);
861 brw_inst_set_3src_dst_subreg_nr(devinfo
, inst
, dest
.subnr
/ 16);
862 brw_inst_set_3src_dst_writemask(devinfo
, inst
, dest
.dw1
.bits
.writemask
);
864 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
865 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
866 assert(src0
.nr
< 128);
867 brw_inst_set_3src_src0_swizzle(devinfo
, inst
, src0
.dw1
.bits
.swizzle
);
868 brw_inst_set_3src_src0_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src0
));
869 brw_inst_set_3src_src0_reg_nr(devinfo
, inst
, src0
.nr
);
870 brw_inst_set_3src_src0_abs(devinfo
, inst
, src0
.abs
);
871 brw_inst_set_3src_src0_negate(devinfo
, inst
, src0
.negate
);
872 brw_inst_set_3src_src0_rep_ctrl(devinfo
, inst
,
873 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
875 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
876 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
877 assert(src1
.nr
< 128);
878 brw_inst_set_3src_src1_swizzle(devinfo
, inst
, src1
.dw1
.bits
.swizzle
);
879 brw_inst_set_3src_src1_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src1
));
880 brw_inst_set_3src_src1_reg_nr(devinfo
, inst
, src1
.nr
);
881 brw_inst_set_3src_src1_abs(devinfo
, inst
, src1
.abs
);
882 brw_inst_set_3src_src1_negate(devinfo
, inst
, src1
.negate
);
883 brw_inst_set_3src_src1_rep_ctrl(devinfo
, inst
,
884 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
886 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
887 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
888 assert(src2
.nr
< 128);
889 brw_inst_set_3src_src2_swizzle(devinfo
, inst
, src2
.dw1
.bits
.swizzle
);
890 brw_inst_set_3src_src2_subreg_nr(devinfo
, inst
, get_3src_subreg_nr(src2
));
891 brw_inst_set_3src_src2_reg_nr(devinfo
, inst
, src2
.nr
);
892 brw_inst_set_3src_src2_abs(devinfo
, inst
, src2
.abs
);
893 brw_inst_set_3src_src2_negate(devinfo
, inst
, src2
.negate
);
894 brw_inst_set_3src_src2_rep_ctrl(devinfo
, inst
,
895 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
897 if (devinfo
->gen
>= 7) {
898 /* Set both the source and destination types based on dest.type,
899 * ignoring the source register types. The MAD and LRP emitters ensure
900 * that all four types are float. The BFE and BFI2 emitters, however,
901 * may send us mixed D and UD types and want us to ignore that and use
902 * the destination type.
905 case BRW_REGISTER_TYPE_F
:
906 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
907 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_F
);
909 case BRW_REGISTER_TYPE_D
:
910 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
911 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_D
);
913 case BRW_REGISTER_TYPE_UD
:
914 brw_inst_set_3src_src_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
915 brw_inst_set_3src_dst_type(devinfo
, inst
, BRW_3SRC_TYPE_UD
);
924 /***********************************************************************
925 * Convenience routines.
928 brw_inst *brw_##OP(struct brw_codegen *p, \
929 struct brw_reg dest, \
930 struct brw_reg src0) \
932 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
936 brw_inst *brw_##OP(struct brw_codegen *p, \
937 struct brw_reg dest, \
938 struct brw_reg src0, \
939 struct brw_reg src1) \
941 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
945 brw_inst *brw_##OP(struct brw_codegen *p, \
946 struct brw_reg dest, \
947 struct brw_reg src0, \
948 struct brw_reg src1, \
949 struct brw_reg src2) \
951 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
955 brw_inst *brw_##OP(struct brw_codegen *p, \
956 struct brw_reg dest, \
957 struct brw_reg src0, \
958 struct brw_reg src1, \
959 struct brw_reg src2) \
961 assert(dest.type == BRW_REGISTER_TYPE_F); \
962 assert(src0.type == BRW_REGISTER_TYPE_F); \
963 assert(src1.type == BRW_REGISTER_TYPE_F); \
964 assert(src2.type == BRW_REGISTER_TYPE_F); \
965 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
968 /* Rounding operations (other than RNDD) require two instructions - the first
969 * stores a rounded value (possibly the wrong way) in the dest register, but
970 * also sets a per-channel "increment bit" in the flag register. A predicated
971 * add of 1.0 fixes dest to contain the desired result.
973 * Sandybridge and later appear to round correctly without an ADD.
976 void brw_##OP(struct brw_codegen *p, \
977 struct brw_reg dest, \
978 struct brw_reg src) \
980 const struct brw_device_info *devinfo = p->devinfo; \
981 brw_inst *rnd, *add; \
982 rnd = next_insn(p, BRW_OPCODE_##OP); \
983 brw_set_dest(p, rnd, dest); \
984 brw_set_src0(p, rnd, src); \
986 if (devinfo->gen < 6) { \
987 /* turn on round-increments */ \
988 brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R); \
989 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
990 brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL); \
1030 brw_ADD(struct brw_codegen
*p
, struct brw_reg dest
,
1031 struct brw_reg src0
, struct brw_reg src1
)
1034 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1035 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1036 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1037 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1038 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1041 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1042 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1043 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1044 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1045 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1048 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1052 brw_AVG(struct brw_codegen
*p
, struct brw_reg dest
,
1053 struct brw_reg src0
, struct brw_reg src1
)
1055 assert(dest
.type
== src0
.type
);
1056 assert(src0
.type
== src1
.type
);
1057 switch (src0
.type
) {
1058 case BRW_REGISTER_TYPE_B
:
1059 case BRW_REGISTER_TYPE_UB
:
1060 case BRW_REGISTER_TYPE_W
:
1061 case BRW_REGISTER_TYPE_UW
:
1062 case BRW_REGISTER_TYPE_D
:
1063 case BRW_REGISTER_TYPE_UD
:
1066 unreachable("Bad type for brw_AVG");
1069 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1073 brw_MUL(struct brw_codegen
*p
, struct brw_reg dest
,
1074 struct brw_reg src0
, struct brw_reg src1
)
1077 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1078 src0
.type
== BRW_REGISTER_TYPE_UD
||
1079 src1
.type
== BRW_REGISTER_TYPE_D
||
1080 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1081 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1084 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1085 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1086 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1087 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1088 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1091 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1092 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1093 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1094 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1095 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1098 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1099 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1100 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1101 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1103 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1107 brw_LINE(struct brw_codegen
*p
, struct brw_reg dest
,
1108 struct brw_reg src0
, struct brw_reg src1
)
1110 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1111 src0
.width
= BRW_WIDTH_1
;
1112 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1113 return brw_alu2(p
, BRW_OPCODE_LINE
, dest
, src0
, src1
);
1117 brw_PLN(struct brw_codegen
*p
, struct brw_reg dest
,
1118 struct brw_reg src0
, struct brw_reg src1
)
1120 src0
.vstride
= BRW_VERTICAL_STRIDE_0
;
1121 src0
.width
= BRW_WIDTH_1
;
1122 src0
.hstride
= BRW_HORIZONTAL_STRIDE_0
;
1123 src1
.vstride
= BRW_VERTICAL_STRIDE_8
;
1124 src1
.width
= BRW_WIDTH_8
;
1125 src1
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
1126 return brw_alu2(p
, BRW_OPCODE_PLN
, dest
, src0
, src1
);
1130 brw_F32TO16(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1132 const struct brw_device_info
*devinfo
= p
->devinfo
;
1133 const bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1134 /* The F32TO16 instruction doesn't support 32-bit destination types in
1135 * Align1 mode, and neither does the Gen8 implementation in terms of a
1136 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1137 * an undocumented feature.
1139 const bool needs_zero_fill
= (dst
.type
== BRW_REGISTER_TYPE_UD
&&
1140 (!align16
|| devinfo
->gen
>= 8));
1144 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1146 assert(dst
.type
== BRW_REGISTER_TYPE_UD
||
1147 dst
.type
== BRW_REGISTER_TYPE_W
||
1148 dst
.type
== BRW_REGISTER_TYPE_UW
||
1149 dst
.type
== BRW_REGISTER_TYPE_HF
);
1152 brw_push_insn_state(p
);
1154 if (needs_zero_fill
) {
1155 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
1156 dst
= spread(retype(dst
, BRW_REGISTER_TYPE_W
), 2);
1159 if (devinfo
->gen
>= 8) {
1160 inst
= brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1162 assert(devinfo
->gen
== 7);
1163 inst
= brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1166 if (needs_zero_fill
) {
1167 brw_inst_set_no_dd_clear(devinfo
, inst
, true);
1168 inst
= brw_MOV(p
, suboffset(dst
, 1), brw_imm_ud(0u));
1169 brw_inst_set_no_dd_check(devinfo
, inst
, true);
1172 brw_pop_insn_state(p
);
1177 brw_F16TO32(struct brw_codegen
*p
, struct brw_reg dst
, struct brw_reg src
)
1179 const struct brw_device_info
*devinfo
= p
->devinfo
;
1180 bool align16
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_16
;
1183 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1185 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1187 * Because this instruction does not have a 16-bit floating-point
1188 * type, the source data type must be Word (W). The destination type
1189 * must be F (Float).
1191 if (src
.type
== BRW_REGISTER_TYPE_UD
)
1192 src
= spread(retype(src
, BRW_REGISTER_TYPE_W
), 2);
1194 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1195 src
.type
== BRW_REGISTER_TYPE_UW
||
1196 src
.type
== BRW_REGISTER_TYPE_HF
);
1199 if (devinfo
->gen
>= 8) {
1200 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1202 assert(devinfo
->gen
== 7);
1203 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1208 void brw_NOP(struct brw_codegen
*p
)
1210 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1211 brw_set_dest(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1212 brw_set_src0(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1213 brw_set_src1(p
, insn
, brw_imm_ud(0x0));
1220 /***********************************************************************
1221 * Comparisons, if/else/endif
1225 brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1226 unsigned predicate_control
)
1228 const struct brw_device_info
*devinfo
= p
->devinfo
;
1229 struct brw_reg ip
= brw_ip_reg();
1230 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1232 brw_inst_set_exec_size(devinfo
, inst
, BRW_EXECUTE_2
);
1233 brw_inst_set_qtr_control(devinfo
, inst
, BRW_COMPRESSION_NONE
);
1234 brw_inst_set_mask_control(devinfo
, inst
, BRW_MASK_DISABLE
);
1235 brw_inst_set_pred_control(devinfo
, inst
, predicate_control
);
1241 push_if_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1243 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1245 p
->if_stack_depth
++;
1246 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1247 p
->if_stack_array_size
*= 2;
1248 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1249 p
->if_stack_array_size
);
1254 pop_if_stack(struct brw_codegen
*p
)
1256 p
->if_stack_depth
--;
1257 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1261 push_loop_stack(struct brw_codegen
*p
, brw_inst
*inst
)
1263 if (p
->loop_stack_array_size
< p
->loop_stack_depth
) {
1264 p
->loop_stack_array_size
*= 2;
1265 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1266 p
->loop_stack_array_size
);
1267 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1268 p
->loop_stack_array_size
);
1271 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1272 p
->loop_stack_depth
++;
1273 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1277 get_inner_do_insn(struct brw_codegen
*p
)
1279 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1282 /* EU takes the value from the flag register and pushes it onto some
1283 * sort of a stack (presumably merging with any flag value already on
1284 * the stack). Within an if block, the flags at the top of the stack
1285 * control execution on each channel of the unit, eg. on each of the
1286 * 16 pixel values in our wm programs.
1288 * When the matching 'else' instruction is reached (presumably by
1289 * countdown of the instruction count patched in by our ELSE/ENDIF
1290 * functions), the relevant flags are inverted.
1292 * When the matching 'endif' instruction is reached, the flags are
1293 * popped off. If the stack is now empty, normal execution resumes.
1296 brw_IF(struct brw_codegen
*p
, unsigned execute_size
)
1298 const struct brw_device_info
*devinfo
= p
->devinfo
;
1301 insn
= next_insn(p
, BRW_OPCODE_IF
);
1303 /* Override the defaults for this instruction:
1305 if (devinfo
->gen
< 6) {
1306 brw_set_dest(p
, insn
, brw_ip_reg());
1307 brw_set_src0(p
, insn
, brw_ip_reg());
1308 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1309 } else if (devinfo
->gen
== 6) {
1310 brw_set_dest(p
, insn
, brw_imm_w(0));
1311 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1312 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1313 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1314 } else if (devinfo
->gen
== 7) {
1315 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1316 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1317 brw_set_src1(p
, insn
, brw_imm_w(0));
1318 brw_inst_set_jip(devinfo
, insn
, 0);
1319 brw_inst_set_uip(devinfo
, insn
, 0);
1321 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1322 brw_set_src0(p
, insn
, brw_imm_d(0));
1323 brw_inst_set_jip(devinfo
, insn
, 0);
1324 brw_inst_set_uip(devinfo
, insn
, 0);
1327 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1328 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1329 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NORMAL
);
1330 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1331 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1332 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1334 push_if_stack(p
, insn
);
1335 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1339 /* This function is only used for gen6-style IF instructions with an
1340 * embedded comparison (conditional modifier). It is not used on gen7.
1343 gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1344 struct brw_reg src0
, struct brw_reg src1
)
1346 const struct brw_device_info
*devinfo
= p
->devinfo
;
1349 insn
= next_insn(p
, BRW_OPCODE_IF
);
1351 brw_set_dest(p
, insn
, brw_imm_w(0));
1352 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1354 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1355 brw_set_src0(p
, insn
, src0
);
1356 brw_set_src1(p
, insn
, src1
);
1358 assert(brw_inst_qtr_control(devinfo
, insn
) == BRW_COMPRESSION_NONE
);
1359 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
1360 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1362 push_if_stack(p
, insn
);
1367 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1370 convert_IF_ELSE_to_ADD(struct brw_codegen
*p
,
1371 brw_inst
*if_inst
, brw_inst
*else_inst
)
1373 const struct brw_device_info
*devinfo
= p
->devinfo
;
1375 /* The next instruction (where the ENDIF would be, if it existed) */
1376 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1378 assert(p
->single_program_flow
);
1379 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1380 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1381 assert(brw_inst_exec_size(devinfo
, if_inst
) == BRW_EXECUTE_1
);
1383 /* Convert IF to an ADD instruction that moves the instruction pointer
1384 * to the first instruction of the ELSE block. If there is no ELSE
1385 * block, point to where ENDIF would be. Reverse the predicate.
1387 * There's no need to execute an ENDIF since we don't need to do any
1388 * stack operations, and if we're currently executing, we just want to
1389 * continue normally.
1391 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_ADD
);
1392 brw_inst_set_pred_inv(devinfo
, if_inst
, true);
1394 if (else_inst
!= NULL
) {
1395 /* Convert ELSE to an ADD instruction that points where the ENDIF
1398 brw_inst_set_opcode(devinfo
, else_inst
, BRW_OPCODE_ADD
);
1400 brw_inst_set_imm_ud(devinfo
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1401 brw_inst_set_imm_ud(devinfo
, else_inst
, (next_inst
- else_inst
) * 16);
1403 brw_inst_set_imm_ud(devinfo
, if_inst
, (next_inst
- if_inst
) * 16);
1408 * Patch IF and ELSE instructions with appropriate jump targets.
1411 patch_IF_ELSE(struct brw_codegen
*p
,
1412 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1414 const struct brw_device_info
*devinfo
= p
->devinfo
;
1416 /* We shouldn't be patching IF and ELSE instructions in single program flow
1417 * mode when gen < 6, because in single program flow mode on those
1418 * platforms, we convert flow control instructions to conditional ADDs that
1419 * operate on IP (see brw_ENDIF).
1421 * However, on Gen6, writing to IP doesn't work in single program flow mode
1422 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1423 * not be updated by non-flow control instructions."). And on later
1424 * platforms, there is no significant benefit to converting control flow
1425 * instructions to conditional ADDs. So we do patch IF and ELSE
1426 * instructions in single program flow mode on those platforms.
1428 if (devinfo
->gen
< 6)
1429 assert(!p
->single_program_flow
);
1431 assert(if_inst
!= NULL
&& brw_inst_opcode(devinfo
, if_inst
) == BRW_OPCODE_IF
);
1432 assert(endif_inst
!= NULL
);
1433 assert(else_inst
== NULL
|| brw_inst_opcode(devinfo
, else_inst
) == BRW_OPCODE_ELSE
);
1435 unsigned br
= brw_jump_scale(devinfo
);
1437 assert(brw_inst_opcode(devinfo
, endif_inst
) == BRW_OPCODE_ENDIF
);
1438 brw_inst_set_exec_size(devinfo
, endif_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1440 if (else_inst
== NULL
) {
1441 /* Patch IF -> ENDIF */
1442 if (devinfo
->gen
< 6) {
1443 /* Turn it into an IFF, which means no mask stack operations for
1444 * all-false and jumping past the ENDIF.
1446 brw_inst_set_opcode(devinfo
, if_inst
, BRW_OPCODE_IFF
);
1447 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1448 br
* (endif_inst
- if_inst
+ 1));
1449 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1450 } else if (devinfo
->gen
== 6) {
1451 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1452 brw_inst_set_gen6_jump_count(devinfo
, if_inst
, br
*(endif_inst
- if_inst
));
1454 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1455 brw_inst_set_jip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1458 brw_inst_set_exec_size(devinfo
, else_inst
, brw_inst_exec_size(devinfo
, if_inst
));
1460 /* Patch IF -> ELSE */
1461 if (devinfo
->gen
< 6) {
1462 brw_inst_set_gen4_jump_count(devinfo
, if_inst
,
1463 br
* (else_inst
- if_inst
));
1464 brw_inst_set_gen4_pop_count(devinfo
, if_inst
, 0);
1465 } else if (devinfo
->gen
== 6) {
1466 brw_inst_set_gen6_jump_count(devinfo
, if_inst
,
1467 br
* (else_inst
- if_inst
+ 1));
1470 /* Patch ELSE -> ENDIF */
1471 if (devinfo
->gen
< 6) {
1472 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1475 brw_inst_set_gen4_jump_count(devinfo
, else_inst
,
1476 br
* (endif_inst
- else_inst
+ 1));
1477 brw_inst_set_gen4_pop_count(devinfo
, else_inst
, 1);
1478 } else if (devinfo
->gen
== 6) {
1479 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1480 brw_inst_set_gen6_jump_count(devinfo
, else_inst
,
1481 br
* (endif_inst
- else_inst
));
1483 /* The IF instruction's JIP should point just past the ELSE */
1484 brw_inst_set_jip(devinfo
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1485 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1486 brw_inst_set_uip(devinfo
, if_inst
, br
* (endif_inst
- if_inst
));
1487 brw_inst_set_jip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1488 if (devinfo
->gen
>= 8) {
1489 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1490 * should point to ENDIF.
1492 brw_inst_set_uip(devinfo
, else_inst
, br
* (endif_inst
- else_inst
));
1499 brw_ELSE(struct brw_codegen
*p
)
1501 const struct brw_device_info
*devinfo
= p
->devinfo
;
1504 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1506 if (devinfo
->gen
< 6) {
1507 brw_set_dest(p
, insn
, brw_ip_reg());
1508 brw_set_src0(p
, insn
, brw_ip_reg());
1509 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1510 } else if (devinfo
->gen
== 6) {
1511 brw_set_dest(p
, insn
, brw_imm_w(0));
1512 brw_inst_set_gen6_jump_count(devinfo
, insn
, 0);
1513 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1514 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1515 } else if (devinfo
->gen
== 7) {
1516 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1517 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1518 brw_set_src1(p
, insn
, brw_imm_w(0));
1519 brw_inst_set_jip(devinfo
, insn
, 0);
1520 brw_inst_set_uip(devinfo
, insn
, 0);
1522 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1523 brw_set_src0(p
, insn
, brw_imm_d(0));
1524 brw_inst_set_jip(devinfo
, insn
, 0);
1525 brw_inst_set_uip(devinfo
, insn
, 0);
1528 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1529 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1530 if (!p
->single_program_flow
&& devinfo
->gen
< 6)
1531 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1533 push_if_stack(p
, insn
);
1537 brw_ENDIF(struct brw_codegen
*p
)
1539 const struct brw_device_info
*devinfo
= p
->devinfo
;
1540 brw_inst
*insn
= NULL
;
1541 brw_inst
*else_inst
= NULL
;
1542 brw_inst
*if_inst
= NULL
;
1544 bool emit_endif
= true;
1546 /* In single program flow mode, we can express IF and ELSE instructions
1547 * equivalently as ADD instructions that operate on IP. On platforms prior
1548 * to Gen6, flow control instructions cause an implied thread switch, so
1549 * this is a significant savings.
1551 * However, on Gen6, writing to IP doesn't work in single program flow mode
1552 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1553 * not be updated by non-flow control instructions."). And on later
1554 * platforms, there is no significant benefit to converting control flow
1555 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1558 if (devinfo
->gen
< 6 && p
->single_program_flow
)
1562 * A single next_insn() may change the base address of instruction store
1563 * memory(p->store), so call it first before referencing the instruction
1564 * store pointer from an index
1567 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1569 /* Pop the IF and (optional) ELSE instructions from the stack */
1570 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1571 tmp
= pop_if_stack(p
);
1572 if (brw_inst_opcode(devinfo
, tmp
) == BRW_OPCODE_ELSE
) {
1574 tmp
= pop_if_stack(p
);
1579 /* ENDIF is useless; don't bother emitting it. */
1580 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1584 if (devinfo
->gen
< 6) {
1585 brw_set_dest(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1586 brw_set_src0(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1587 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1588 } else if (devinfo
->gen
== 6) {
1589 brw_set_dest(p
, insn
, brw_imm_w(0));
1590 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1591 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1592 } else if (devinfo
->gen
== 7) {
1593 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1594 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1595 brw_set_src1(p
, insn
, brw_imm_w(0));
1597 brw_set_src0(p
, insn
, brw_imm_d(0));
1600 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1601 brw_inst_set_mask_control(devinfo
, insn
, BRW_MASK_ENABLE
);
1602 if (devinfo
->gen
< 6)
1603 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1605 /* Also pop item off the stack in the endif instruction: */
1606 if (devinfo
->gen
< 6) {
1607 brw_inst_set_gen4_jump_count(devinfo
, insn
, 0);
1608 brw_inst_set_gen4_pop_count(devinfo
, insn
, 1);
1609 } else if (devinfo
->gen
== 6) {
1610 brw_inst_set_gen6_jump_count(devinfo
, insn
, 2);
1612 brw_inst_set_jip(devinfo
, insn
, 2);
1614 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1618 brw_BREAK(struct brw_codegen
*p
)
1620 const struct brw_device_info
*devinfo
= p
->devinfo
;
1623 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1624 if (devinfo
->gen
>= 8) {
1625 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1626 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1627 } else if (devinfo
->gen
>= 6) {
1628 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1629 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1630 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1632 brw_set_dest(p
, insn
, brw_ip_reg());
1633 brw_set_src0(p
, insn
, brw_ip_reg());
1634 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1635 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1636 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1638 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1639 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1646 brw_CONT(struct brw_codegen
*p
)
1648 const struct brw_device_info
*devinfo
= p
->devinfo
;
1651 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1652 brw_set_dest(p
, insn
, brw_ip_reg());
1653 if (devinfo
->gen
>= 8) {
1654 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1656 brw_set_src0(p
, insn
, brw_ip_reg());
1657 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1660 if (devinfo
->gen
< 6) {
1661 brw_inst_set_gen4_pop_count(devinfo
, insn
,
1662 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1664 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1665 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1671 gen6_HALT(struct brw_codegen
*p
)
1673 const struct brw_device_info
*devinfo
= p
->devinfo
;
1676 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1677 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1678 if (devinfo
->gen
>= 8) {
1679 brw_set_src0(p
, insn
, brw_imm_d(0x0));
1681 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1682 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1685 if (p
->compressed
) {
1686 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_16
);
1688 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1689 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_8
);
1696 * The DO/WHILE is just an unterminated loop -- break or continue are
1697 * used for control within the loop. We have a few ways they can be
1700 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1701 * jip and no DO instruction.
1703 * For non-uniform control flow pre-gen6, there's a DO instruction to
1704 * push the mask, and a WHILE to jump back, and BREAK to get out and
1707 * For gen6, there's no more mask stack, so no need for DO. WHILE
1708 * just points back to the first instruction of the loop.
1711 brw_DO(struct brw_codegen
*p
, unsigned execute_size
)
1713 const struct brw_device_info
*devinfo
= p
->devinfo
;
1715 if (devinfo
->gen
>= 6 || p
->single_program_flow
) {
1716 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1717 return &p
->store
[p
->nr_insn
];
1719 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1721 push_loop_stack(p
, insn
);
1723 /* Override the defaults for this instruction:
1725 brw_set_dest(p
, insn
, brw_null_reg());
1726 brw_set_src0(p
, insn
, brw_null_reg());
1727 brw_set_src1(p
, insn
, brw_null_reg());
1729 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1730 brw_inst_set_exec_size(devinfo
, insn
, execute_size
);
1731 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
);
1738 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1741 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1742 * nesting, since it can always just point to the end of the block/current loop.
1745 brw_patch_break_cont(struct brw_codegen
*p
, brw_inst
*while_inst
)
1747 const struct brw_device_info
*devinfo
= p
->devinfo
;
1748 brw_inst
*do_inst
= get_inner_do_insn(p
);
1750 unsigned br
= brw_jump_scale(devinfo
);
1752 assert(devinfo
->gen
< 6);
1754 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1755 /* If the jump count is != 0, that means that this instruction has already
1756 * been patched because it's part of a loop inside of the one we're
1759 if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_BREAK
&&
1760 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1761 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
*((while_inst
- inst
) + 1));
1762 } else if (brw_inst_opcode(devinfo
, inst
) == BRW_OPCODE_CONTINUE
&&
1763 brw_inst_gen4_jump_count(devinfo
, inst
) == 0) {
1764 brw_inst_set_gen4_jump_count(devinfo
, inst
, br
* (while_inst
- inst
));
1770 brw_WHILE(struct brw_codegen
*p
)
1772 const struct brw_device_info
*devinfo
= p
->devinfo
;
1773 brw_inst
*insn
, *do_insn
;
1774 unsigned br
= brw_jump_scale(devinfo
);
1776 if (devinfo
->gen
>= 6) {
1777 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1778 do_insn
= get_inner_do_insn(p
);
1780 if (devinfo
->gen
>= 8) {
1781 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1782 brw_set_src0(p
, insn
, brw_imm_d(0));
1783 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1784 } else if (devinfo
->gen
== 7) {
1785 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1786 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1787 brw_set_src1(p
, insn
, brw_imm_w(0));
1788 brw_inst_set_jip(devinfo
, insn
, br
* (do_insn
- insn
));
1790 brw_set_dest(p
, insn
, brw_imm_w(0));
1791 brw_inst_set_gen6_jump_count(devinfo
, insn
, br
* (do_insn
- insn
));
1792 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1793 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1796 brw_inst_set_exec_size(devinfo
, insn
, p
->compressed
? BRW_EXECUTE_16
1799 if (p
->single_program_flow
) {
1800 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1801 do_insn
= get_inner_do_insn(p
);
1803 brw_set_dest(p
, insn
, brw_ip_reg());
1804 brw_set_src0(p
, insn
, brw_ip_reg());
1805 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1806 brw_inst_set_exec_size(devinfo
, insn
, BRW_EXECUTE_1
);
1808 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1809 do_insn
= get_inner_do_insn(p
);
1811 assert(brw_inst_opcode(devinfo
, do_insn
) == BRW_OPCODE_DO
);
1813 brw_set_dest(p
, insn
, brw_ip_reg());
1814 brw_set_src0(p
, insn
, brw_ip_reg());
1815 brw_set_src1(p
, insn
, brw_imm_d(0));
1817 brw_inst_set_exec_size(devinfo
, insn
, brw_inst_exec_size(devinfo
, do_insn
));
1818 brw_inst_set_gen4_jump_count(devinfo
, insn
, br
* (do_insn
- insn
+ 1));
1819 brw_inst_set_gen4_pop_count(devinfo
, insn
, 0);
1821 brw_patch_break_cont(p
, insn
);
1824 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
1826 p
->loop_stack_depth
--;
1833 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
)
1835 const struct brw_device_info
*devinfo
= p
->devinfo
;
1836 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1839 if (devinfo
->gen
>= 5)
1842 assert(brw_inst_opcode(devinfo
, jmp_insn
) == BRW_OPCODE_JMPI
);
1843 assert(brw_inst_src1_reg_file(devinfo
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1845 brw_inst_set_gen4_jump_count(devinfo
, jmp_insn
,
1846 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1849 /* To integrate with the above, it makes sense that the comparison
1850 * instruction should populate the flag register. It might be simpler
1851 * just to use the flag reg for most WM tasks?
1853 void brw_CMP(struct brw_codegen
*p
,
1854 struct brw_reg dest
,
1855 unsigned conditional
,
1856 struct brw_reg src0
,
1857 struct brw_reg src1
)
1859 const struct brw_device_info
*devinfo
= p
->devinfo
;
1860 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1862 brw_inst_set_cond_modifier(devinfo
, insn
, conditional
);
1863 brw_set_dest(p
, insn
, dest
);
1864 brw_set_src0(p
, insn
, src0
);
1865 brw_set_src1(p
, insn
, src1
);
1867 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1869 * "Any CMP instruction with a null destination must use a {switch}."
1871 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1872 * mentioned on their work-arounds pages.
1874 if (devinfo
->gen
== 7) {
1875 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1876 dest
.nr
== BRW_ARF_NULL
) {
1877 brw_inst_set_thread_control(devinfo
, insn
, BRW_THREAD_SWITCH
);
1882 /***********************************************************************
1883 * Helpers for the various SEND message types:
1886 /** Extended math function, float[8].
1888 void gen4_math(struct brw_codegen
*p
,
1889 struct brw_reg dest
,
1891 unsigned msg_reg_nr
,
1893 unsigned precision
)
1895 const struct brw_device_info
*devinfo
= p
->devinfo
;
1896 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1898 if (has_scalar_region(src
)) {
1899 data_type
= BRW_MATH_DATA_SCALAR
;
1901 data_type
= BRW_MATH_DATA_VECTOR
;
1904 assert(devinfo
->gen
< 6);
1906 /* Example code doesn't set predicate_control for send
1909 brw_inst_set_pred_control(devinfo
, insn
, 0);
1910 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
1912 brw_set_dest(p
, insn
, dest
);
1913 brw_set_src0(p
, insn
, src
);
1914 brw_set_math_message(p
,
1917 src
.type
== BRW_REGISTER_TYPE_D
,
1922 void gen6_math(struct brw_codegen
*p
,
1923 struct brw_reg dest
,
1925 struct brw_reg src0
,
1926 struct brw_reg src1
)
1928 const struct brw_device_info
*devinfo
= p
->devinfo
;
1929 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1931 assert(devinfo
->gen
>= 6);
1933 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1934 (devinfo
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1935 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
1936 (devinfo
->gen
>= 8 && src0
.file
== BRW_IMMEDIATE_VALUE
));
1938 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1939 if (devinfo
->gen
== 6) {
1940 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1941 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1944 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1945 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1946 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1947 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1948 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1949 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1950 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1952 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1953 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1954 if (function
== BRW_MATH_FUNCTION_POW
) {
1955 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1956 (devinfo
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1958 assert(src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1959 src1
.nr
== BRW_ARF_NULL
);
1963 /* Source modifiers are ignored for extended math instructions on Gen6. */
1964 if (devinfo
->gen
== 6) {
1965 assert(!src0
.negate
);
1967 assert(!src1
.negate
);
1971 brw_inst_set_math_function(devinfo
, insn
, function
);
1973 brw_set_dest(p
, insn
, dest
);
1974 brw_set_src0(p
, insn
, src0
);
1975 brw_set_src1(p
, insn
, src1
);
1980 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1981 * using a constant offset per channel.
1983 * The offset must be aligned to oword size (16 bytes). Used for
1984 * register spilling.
1986 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1991 const struct brw_device_info
*devinfo
= p
->devinfo
;
1992 uint32_t msg_control
, msg_type
;
1995 if (devinfo
->gen
>= 6)
1998 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2000 if (num_regs
== 1) {
2001 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2004 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2008 /* Set up the message header. This is g0, with g0.2 filled with
2009 * the offset. We don't want to leave our offset around in g0 or
2010 * it'll screw up texture samples, so set it up inside the message
2014 brw_push_insn_state(p
);
2015 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2016 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2017 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2019 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2021 /* set message header global offset field (reg 0, element 2) */
2023 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2025 2), BRW_REGISTER_TYPE_UD
),
2026 brw_imm_ud(offset
));
2028 brw_pop_insn_state(p
);
2032 struct brw_reg dest
;
2033 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2034 int send_commit_msg
;
2035 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
2036 BRW_REGISTER_TYPE_UW
);
2038 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_NONE
) {
2039 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2040 src_header
= vec16(src_header
);
2042 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2043 if (devinfo
->gen
< 6)
2044 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2046 /* Until gen6, writes followed by reads from the same location
2047 * are not guaranteed to be ordered unless write_commit is set.
2048 * If set, then a no-op write is issued to the destination
2049 * register to set a dependency, and a read from the destination
2050 * can be used to ensure the ordering.
2052 * For gen6, only writes between different threads need ordering
2053 * protection. Our use of DP writes is all about register
2054 * spilling within a thread.
2056 if (devinfo
->gen
>= 6) {
2057 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2058 send_commit_msg
= 0;
2061 send_commit_msg
= 1;
2064 brw_set_dest(p
, insn
, dest
);
2065 if (devinfo
->gen
>= 6) {
2066 brw_set_src0(p
, insn
, mrf
);
2068 brw_set_src0(p
, insn
, brw_null_reg());
2071 if (devinfo
->gen
>= 6)
2072 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2074 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2076 brw_set_dp_write_message(p
,
2078 255, /* binding table index (255=stateless) */
2082 true, /* header_present */
2083 0, /* not a render target */
2084 send_commit_msg
, /* response_length */
2092 * Read a block of owords (half a GRF each) from the scratch buffer
2093 * using a constant index per channel.
2095 * Offset must be aligned to oword size (16 bytes). Used for register
2099 brw_oword_block_read_scratch(struct brw_codegen
*p
,
2100 struct brw_reg dest
,
2105 const struct brw_device_info
*devinfo
= p
->devinfo
;
2106 uint32_t msg_control
;
2109 if (devinfo
->gen
>= 6)
2112 if (p
->devinfo
->gen
>= 7) {
2113 /* On gen 7 and above, we no longer have message registers and we can
2114 * send from any register we want. By using the destination register
2115 * for the message, we guarantee that the implied message write won't
2116 * accidentally overwrite anything. This has been a problem because
2117 * the MRF registers and source for the final FB write are both fixed
2120 mrf
= retype(dest
, BRW_REGISTER_TYPE_UD
);
2122 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2124 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2126 if (num_regs
== 1) {
2127 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2130 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2135 brw_push_insn_state(p
);
2136 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2137 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2138 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2140 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2142 /* set message header global offset field (reg 0, element 2) */
2143 brw_MOV(p
, get_element_ud(mrf
, 2), brw_imm_ud(offset
));
2145 brw_pop_insn_state(p
);
2149 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2151 assert(brw_inst_pred_control(devinfo
, insn
) == 0);
2152 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2154 brw_set_dest(p
, insn
, dest
); /* UW? */
2155 if (devinfo
->gen
>= 6) {
2156 brw_set_src0(p
, insn
, mrf
);
2158 brw_set_src0(p
, insn
, brw_null_reg());
2159 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2162 brw_set_dp_read_message(p
,
2164 255, /* binding table index (255=stateless) */
2166 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2167 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
,
2169 true, /* header_present */
2175 gen7_block_read_scratch(struct brw_codegen
*p
,
2176 struct brw_reg dest
,
2180 const struct brw_device_info
*devinfo
= p
->devinfo
;
2181 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2182 assert(brw_inst_pred_control(devinfo
, insn
) == BRW_PREDICATE_NONE
);
2184 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2185 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2187 /* The HW requires that the header is present; this is to get the g0.5
2190 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2192 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2193 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2194 * is 32 bytes, which happens to be the size of a register.
2197 assert(offset
< (1 << 12));
2199 gen7_set_dp_scratch_message(p
, insn
,
2200 false, /* scratch read */
2202 false, /* invalidate after read */
2205 1, /* mlen: just g0 */
2206 num_regs
, /* rlen */
2207 true); /* header present */
2211 * Read a float[4] vector from the data port Data Cache (const buffer).
2212 * Location (in buffer) should be a multiple of 16.
2213 * Used for fetching shader constants.
2215 void brw_oword_block_read(struct brw_codegen
*p
,
2216 struct brw_reg dest
,
2219 uint32_t bind_table_index
)
2221 const struct brw_device_info
*devinfo
= p
->devinfo
;
2223 /* On newer hardware, offset is in units of owords. */
2224 if (devinfo
->gen
>= 6)
2227 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2229 brw_push_insn_state(p
);
2230 brw_set_default_exec_size(p
, BRW_EXECUTE_8
);
2231 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2232 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2233 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2235 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2237 /* set message header global offset field (reg 0, element 2) */
2239 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2241 2), BRW_REGISTER_TYPE_UD
),
2242 brw_imm_ud(offset
));
2244 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2246 /* cast dest to a uword[8] vector */
2247 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2249 brw_set_dest(p
, insn
, dest
);
2250 if (devinfo
->gen
>= 6) {
2251 brw_set_src0(p
, insn
, mrf
);
2253 brw_set_src0(p
, insn
, brw_null_reg());
2254 brw_inst_set_base_mrf(devinfo
, insn
, mrf
.nr
);
2257 brw_set_dp_read_message(p
,
2260 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW
,
2261 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2262 BRW_DATAPORT_READ_TARGET_DATA_CACHE
,
2264 true, /* header_present */
2265 1); /* response_length (1 reg, 2 owords!) */
2267 brw_pop_insn_state(p
);
2271 void brw_fb_WRITE(struct brw_codegen
*p
,
2273 struct brw_reg payload
,
2274 struct brw_reg implied_header
,
2275 unsigned msg_control
,
2276 unsigned binding_table_index
,
2277 unsigned msg_length
,
2278 unsigned response_length
,
2280 bool last_render_target
,
2281 bool header_present
)
2283 const struct brw_device_info
*devinfo
= p
->devinfo
;
2286 struct brw_reg dest
, src0
;
2288 if (dispatch_width
== 16)
2289 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2291 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2293 if (devinfo
->gen
>= 6) {
2294 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2296 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2298 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2300 if (devinfo
->gen
>= 6) {
2301 /* headerless version, just submit color payload */
2304 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2306 assert(payload
.file
== BRW_MESSAGE_REGISTER_FILE
);
2307 brw_inst_set_base_mrf(devinfo
, insn
, payload
.nr
);
2308 src0
= implied_header
;
2310 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2313 brw_set_dest(p
, insn
, dest
);
2314 brw_set_src0(p
, insn
, src0
);
2315 brw_set_dp_write_message(p
,
2317 binding_table_index
,
2325 0 /* send_commit_msg */);
2330 * Texture sample instruction.
2331 * Note: the msg_type plus msg_length values determine exactly what kind
2332 * of sampling operation is performed. See volume 4, page 161 of docs.
2334 void brw_SAMPLE(struct brw_codegen
*p
,
2335 struct brw_reg dest
,
2336 unsigned msg_reg_nr
,
2337 struct brw_reg src0
,
2338 unsigned binding_table_index
,
2341 unsigned response_length
,
2342 unsigned msg_length
,
2343 unsigned header_present
,
2345 unsigned return_format
)
2347 const struct brw_device_info
*devinfo
= p
->devinfo
;
2350 if (msg_reg_nr
!= -1)
2351 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2353 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2354 brw_inst_set_pred_control(devinfo
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2356 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2358 * "Instruction compression is not allowed for this instruction (that
2359 * is, send). The hardware behavior is undefined if this instruction is
2360 * set as compressed. However, compress control can be set to "SecHalf"
2361 * to affect the EMask generation."
2363 * No similar wording is found in later PRMs, but there are examples
2364 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2365 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2366 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2368 if (brw_inst_qtr_control(devinfo
, insn
) != BRW_COMPRESSION_2NDHALF
)
2369 brw_inst_set_qtr_control(devinfo
, insn
, BRW_COMPRESSION_NONE
);
2371 if (devinfo
->gen
< 6)
2372 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2374 brw_set_dest(p
, insn
, dest
);
2375 brw_set_src0(p
, insn
, src0
);
2376 brw_set_sampler_message(p
, insn
,
2377 binding_table_index
,
2387 /* Adjust the message header's sampler state pointer to
2388 * select the correct group of 16 samplers.
2390 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
2391 struct brw_reg header
,
2392 struct brw_reg sampler_index
)
2394 /* The "Sampler Index" field can only store values between 0 and 15.
2395 * However, we can add an offset to the "Sampler State Pointer"
2396 * field, effectively selecting a different set of 16 samplers.
2398 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2399 * offset, and each sampler state is only 16-bytes, so we can't
2400 * exclusively use the offset - we have to use both.
2403 const struct brw_device_info
*devinfo
= p
->devinfo
;
2405 if (sampler_index
.file
== BRW_IMMEDIATE_VALUE
) {
2406 const int sampler_state_size
= 16; /* 16 bytes */
2407 uint32_t sampler
= sampler_index
.dw1
.ud
;
2409 if (sampler
>= 16) {
2410 assert(devinfo
->is_haswell
|| devinfo
->gen
>= 8);
2412 get_element_ud(header
, 3),
2413 get_element_ud(brw_vec8_grf(0, 0), 3),
2414 brw_imm_ud(16 * (sampler
/ 16) * sampler_state_size
));
2417 /* Non-const sampler array indexing case */
2418 if (devinfo
->gen
< 8 && !devinfo
->is_haswell
) {
2422 struct brw_reg temp
= get_element_ud(header
, 3);
2424 brw_AND(p
, temp
, get_element_ud(sampler_index
, 0), brw_imm_ud(0x0f0));
2425 brw_SHL(p
, temp
, temp
, brw_imm_ud(4));
2427 get_element_ud(header
, 3),
2428 get_element_ud(brw_vec8_grf(0, 0), 3),
2433 /* All these variables are pretty confusing - we might be better off
2434 * using bitmasks and macros for this, in the old style. Or perhaps
2435 * just having the caller instantiate the fields in dword3 itself.
2437 void brw_urb_WRITE(struct brw_codegen
*p
,
2438 struct brw_reg dest
,
2439 unsigned msg_reg_nr
,
2440 struct brw_reg src0
,
2441 enum brw_urb_write_flags flags
,
2442 unsigned msg_length
,
2443 unsigned response_length
,
2447 const struct brw_device_info
*devinfo
= p
->devinfo
;
2450 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2452 if (devinfo
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2453 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2454 brw_push_insn_state(p
);
2455 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2456 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2457 brw_OR(p
, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
, msg_reg_nr
, 5),
2458 BRW_REGISTER_TYPE_UD
),
2459 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD
),
2460 brw_imm_ud(0xff00));
2461 brw_pop_insn_state(p
);
2464 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2466 assert(msg_length
< BRW_MAX_MRF
);
2468 brw_set_dest(p
, insn
, dest
);
2469 brw_set_src0(p
, insn
, src0
);
2470 brw_set_src1(p
, insn
, brw_imm_d(0));
2472 if (devinfo
->gen
< 6)
2473 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2475 brw_set_urb_message(p
,
2485 brw_send_indirect_message(struct brw_codegen
*p
,
2488 struct brw_reg payload
,
2489 struct brw_reg desc
)
2491 const struct brw_device_info
*devinfo
= p
->devinfo
;
2492 struct brw_inst
*send
, *setup
;
2494 assert(desc
.type
== BRW_REGISTER_TYPE_UD
);
2496 if (desc
.file
== BRW_IMMEDIATE_VALUE
) {
2497 setup
= send
= next_insn(p
, BRW_OPCODE_SEND
);
2498 brw_set_src1(p
, send
, desc
);
2501 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2503 brw_push_insn_state(p
);
2504 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2505 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2506 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2508 /* Load the indirect descriptor to an address register using OR so the
2509 * caller can specify additional descriptor bits with the usual
2510 * brw_set_*_message() helper functions.
2512 setup
= brw_OR(p
, addr
, desc
, brw_imm_ud(0));
2514 brw_pop_insn_state(p
);
2516 send
= next_insn(p
, BRW_OPCODE_SEND
);
2517 brw_set_src1(p
, send
, addr
);
2520 brw_set_dest(p
, send
, dst
);
2521 brw_set_src0(p
, send
, retype(payload
, BRW_REGISTER_TYPE_UD
));
2522 brw_inst_set_sfid(devinfo
, send
, sfid
);
2527 static struct brw_inst
*
2528 brw_send_indirect_surface_message(struct brw_codegen
*p
,
2531 struct brw_reg payload
,
2532 struct brw_reg surface
,
2533 unsigned message_len
,
2534 unsigned response_len
,
2535 bool header_present
)
2537 const struct brw_device_info
*devinfo
= p
->devinfo
;
2538 struct brw_inst
*insn
;
2540 if (surface
.file
!= BRW_IMMEDIATE_VALUE
) {
2541 struct brw_reg addr
= retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD
);
2543 brw_push_insn_state(p
);
2544 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2545 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2546 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2548 /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2549 * some surface array is accessed out of bounds.
2551 insn
= brw_AND(p
, addr
,
2552 suboffset(vec1(retype(surface
, BRW_REGISTER_TYPE_UD
)),
2553 BRW_GET_SWZ(surface
.dw1
.bits
.swizzle
, 0)),
2556 brw_pop_insn_state(p
);
2561 insn
= brw_send_indirect_message(p
, sfid
, dst
, payload
, surface
);
2562 brw_inst_set_mlen(devinfo
, insn
, message_len
);
2563 brw_inst_set_rlen(devinfo
, insn
, response_len
);
2564 brw_inst_set_header_present(devinfo
, insn
, header_present
);
2570 brw_find_next_block_end(struct brw_codegen
*p
, int start_offset
)
2573 void *store
= p
->store
;
2574 const struct brw_device_info
*devinfo
= p
->devinfo
;
2576 for (offset
= next_offset(devinfo
, store
, start_offset
);
2577 offset
< p
->next_insn_offset
;
2578 offset
= next_offset(devinfo
, store
, offset
)) {
2579 brw_inst
*insn
= store
+ offset
;
2581 switch (brw_inst_opcode(devinfo
, insn
)) {
2582 case BRW_OPCODE_ENDIF
:
2583 case BRW_OPCODE_ELSE
:
2584 case BRW_OPCODE_WHILE
:
2585 case BRW_OPCODE_HALT
:
2593 /* There is no DO instruction on gen6, so to find the end of the loop
2594 * we have to see if the loop is jumping back before our start
2598 brw_find_loop_end(struct brw_codegen
*p
, int start_offset
)
2600 const struct brw_device_info
*devinfo
= p
->devinfo
;
2602 int scale
= 16 / brw_jump_scale(devinfo
);
2603 void *store
= p
->store
;
2605 assert(devinfo
->gen
>= 6);
2607 /* Always start after the instruction (such as a WHILE) we're trying to fix
2610 for (offset
= next_offset(devinfo
, store
, start_offset
);
2611 offset
< p
->next_insn_offset
;
2612 offset
= next_offset(devinfo
, store
, offset
)) {
2613 brw_inst
*insn
= store
+ offset
;
2615 if (brw_inst_opcode(devinfo
, insn
) == BRW_OPCODE_WHILE
) {
2616 int jip
= devinfo
->gen
== 6 ? brw_inst_gen6_jump_count(devinfo
, insn
)
2617 : brw_inst_jip(devinfo
, insn
);
2618 if (offset
+ jip
* scale
<= start_offset
)
2622 assert(!"not reached");
2623 return start_offset
;
2626 /* After program generation, go back and update the UIP and JIP of
2627 * BREAK, CONT, and HALT instructions to their correct locations.
2630 brw_set_uip_jip(struct brw_codegen
*p
)
2632 const struct brw_device_info
*devinfo
= p
->devinfo
;
2634 int br
= brw_jump_scale(devinfo
);
2635 int scale
= 16 / br
;
2636 void *store
= p
->store
;
2638 if (devinfo
->gen
< 6)
2641 for (offset
= 0; offset
< p
->next_insn_offset
;
2642 offset
= next_offset(devinfo
, store
, offset
)) {
2643 brw_inst
*insn
= store
+ offset
;
2645 if (brw_inst_cmpt_control(devinfo
, insn
)) {
2646 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2647 assert(brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_BREAK
&&
2648 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_CONTINUE
&&
2649 brw_inst_opcode(devinfo
, insn
) != BRW_OPCODE_HALT
);
2653 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2654 switch (brw_inst_opcode(devinfo
, insn
)) {
2655 case BRW_OPCODE_BREAK
:
2656 assert(block_end_offset
!= 0);
2657 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2658 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2659 brw_inst_set_uip(devinfo
, insn
,
2660 (brw_find_loop_end(p
, offset
) - offset
+
2661 (devinfo
->gen
== 6 ? 16 : 0)) / scale
);
2663 case BRW_OPCODE_CONTINUE
:
2664 assert(block_end_offset
!= 0);
2665 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2666 brw_inst_set_uip(devinfo
, insn
,
2667 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2669 assert(brw_inst_uip(devinfo
, insn
) != 0);
2670 assert(brw_inst_jip(devinfo
, insn
) != 0);
2673 case BRW_OPCODE_ENDIF
: {
2674 int32_t jump
= (block_end_offset
== 0) ?
2675 1 * br
: (block_end_offset
- offset
) / scale
;
2676 if (devinfo
->gen
>= 7)
2677 brw_inst_set_jip(devinfo
, insn
, jump
);
2679 brw_inst_set_gen6_jump_count(devinfo
, insn
, jump
);
2683 case BRW_OPCODE_HALT
:
2684 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2686 * "In case of the halt instruction not inside any conditional
2687 * code block, the value of <JIP> and <UIP> should be the
2688 * same. In case of the halt instruction inside conditional code
2689 * block, the <UIP> should be the end of the program, and the
2690 * <JIP> should be end of the most inner conditional code block."
2692 * The uip will have already been set by whoever set up the
2695 if (block_end_offset
== 0) {
2696 brw_inst_set_jip(devinfo
, insn
, brw_inst_uip(devinfo
, insn
));
2698 brw_inst_set_jip(devinfo
, insn
, (block_end_offset
- offset
) / scale
);
2700 assert(brw_inst_uip(devinfo
, insn
) != 0);
2701 assert(brw_inst_jip(devinfo
, insn
) != 0);
2707 void brw_ff_sync(struct brw_codegen
*p
,
2708 struct brw_reg dest
,
2709 unsigned msg_reg_nr
,
2710 struct brw_reg src0
,
2712 unsigned response_length
,
2715 const struct brw_device_info
*devinfo
= p
->devinfo
;
2718 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2720 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2721 brw_set_dest(p
, insn
, dest
);
2722 brw_set_src0(p
, insn
, src0
);
2723 brw_set_src1(p
, insn
, brw_imm_d(0));
2725 if (devinfo
->gen
< 6)
2726 brw_inst_set_base_mrf(devinfo
, insn
, msg_reg_nr
);
2728 brw_set_ff_sync_message(p
,
2736 * Emit the SEND instruction necessary to generate stream output data on Gen6
2737 * (for transform feedback).
2739 * If send_commit_msg is true, this is the last piece of stream output data
2740 * from this thread, so send the data as a committed write. According to the
2741 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2743 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2744 * writes are complete by sending the final write as a committed write."
2747 brw_svb_write(struct brw_codegen
*p
,
2748 struct brw_reg dest
,
2749 unsigned msg_reg_nr
,
2750 struct brw_reg src0
,
2751 unsigned binding_table_index
,
2752 bool send_commit_msg
)
2756 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2758 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2759 brw_set_dest(p
, insn
, dest
);
2760 brw_set_src0(p
, insn
, src0
);
2761 brw_set_src1(p
, insn
, brw_imm_d(0));
2762 brw_set_dp_write_message(p
, insn
,
2763 binding_table_index
,
2764 0, /* msg_control: ignored */
2765 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2767 true, /* header_present */
2768 0, /* last_render_target: ignored */
2769 send_commit_msg
, /* response_length */
2770 0, /* end_of_thread */
2771 send_commit_msg
); /* send_commit_msg */
2775 brw_surface_payload_size(struct brw_codegen
*p
,
2776 unsigned num_channels
,
2780 if (has_simd4x2
&& brw_inst_access_mode(p
->devinfo
, p
->current
) == BRW_ALIGN_16
)
2782 else if (has_simd16
&& p
->compressed
)
2783 return 2 * num_channels
;
2785 return num_channels
;
2789 brw_set_dp_untyped_atomic_message(struct brw_codegen
*p
,
2792 bool response_expected
)
2794 const struct brw_device_info
*devinfo
= p
->devinfo
;
2795 unsigned msg_control
=
2796 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2797 (response_expected
? 1 << 5 : 0); /* Return data expected */
2799 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2800 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2802 msg_control
|= 1 << 4; /* SIMD8 mode */
2804 brw_inst_set_dp_msg_type(devinfo
, insn
,
2805 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2807 brw_inst_set_dp_msg_type(devinfo
, insn
,
2808 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2811 brw_inst_set_dp_msg_type(devinfo
, insn
,
2812 GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2815 msg_control
|= 1 << 4; /* SIMD8 mode */
2818 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2822 brw_untyped_atomic(struct brw_codegen
*p
,
2824 struct brw_reg payload
,
2825 struct brw_reg surface
,
2827 unsigned msg_length
,
2828 bool response_expected
)
2830 const struct brw_device_info
*devinfo
= p
->devinfo
;
2831 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2832 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2833 GEN7_SFID_DATAPORT_DATA_CACHE
);
2834 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2835 /* Mask out unused components -- This is especially important in Align16
2836 * mode on generations that don't have native support for SIMD4x2 atomics,
2837 * because unused but enabled components will cause the dataport to perform
2838 * additional atomic operations on the addresses that happen to be in the
2839 * uninitialized Y, Z and W coordinates of the payload.
2841 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2842 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2843 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2844 brw_surface_payload_size(p
, response_expected
,
2845 devinfo
->gen
>= 8 || devinfo
->is_haswell
, true),
2848 brw_set_dp_untyped_atomic_message(
2849 p
, insn
, atomic_op
, response_expected
);
2853 brw_set_dp_untyped_surface_read_message(struct brw_codegen
*p
,
2854 struct brw_inst
*insn
,
2855 unsigned num_channels
)
2857 const struct brw_device_info
*devinfo
= p
->devinfo
;
2858 /* Set mask of 32-bit channels to drop. */
2859 unsigned msg_control
= 0xf & (0xf << num_channels
);
2861 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2863 msg_control
|= 1 << 4; /* SIMD16 mode */
2865 msg_control
|= 2 << 4; /* SIMD8 mode */
2868 brw_inst_set_dp_msg_type(devinfo
, insn
,
2869 (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2870 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
:
2871 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
));
2872 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2876 brw_untyped_surface_read(struct brw_codegen
*p
,
2878 struct brw_reg payload
,
2879 struct brw_reg surface
,
2880 unsigned msg_length
,
2881 unsigned num_channels
)
2883 const struct brw_device_info
*devinfo
= p
->devinfo
;
2884 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2885 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2886 GEN7_SFID_DATAPORT_DATA_CACHE
);
2887 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
2888 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2889 p
, sfid
, dst
, payload
, surface
, msg_length
,
2890 brw_surface_payload_size(p
, num_channels
, true, true),
2893 brw_set_dp_untyped_surface_read_message(
2894 p
, insn
, num_channels
);
2898 brw_set_dp_untyped_surface_write_message(struct brw_codegen
*p
,
2899 struct brw_inst
*insn
,
2900 unsigned num_channels
)
2902 const struct brw_device_info
*devinfo
= p
->devinfo
;
2903 /* Set mask of 32-bit channels to drop. */
2904 unsigned msg_control
= 0xf & (0xf << num_channels
);
2906 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2908 msg_control
|= 1 << 4; /* SIMD16 mode */
2910 msg_control
|= 2 << 4; /* SIMD8 mode */
2912 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
)
2913 msg_control
|= 0 << 4; /* SIMD4x2 mode */
2915 msg_control
|= 2 << 4; /* SIMD8 mode */
2918 brw_inst_set_dp_msg_type(devinfo
, insn
,
2919 devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2920 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
:
2921 GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
);
2922 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2926 brw_untyped_surface_write(struct brw_codegen
*p
,
2927 struct brw_reg payload
,
2928 struct brw_reg surface
,
2929 unsigned msg_length
,
2930 unsigned num_channels
)
2932 const struct brw_device_info
*devinfo
= p
->devinfo
;
2933 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2934 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2935 GEN7_SFID_DATAPORT_DATA_CACHE
);
2936 const bool align1
= brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
;
2937 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2938 const unsigned mask
= devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
2939 WRITEMASK_X
: WRITEMASK_XYZW
;
2940 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2941 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
2942 payload
, surface
, msg_length
, 0, align1
);
2944 brw_set_dp_untyped_surface_write_message(
2945 p
, insn
, num_channels
);
2949 brw_set_dp_typed_atomic_message(struct brw_codegen
*p
,
2950 struct brw_inst
*insn
,
2952 bool response_expected
)
2954 const struct brw_device_info
*devinfo
= p
->devinfo
;
2955 unsigned msg_control
=
2956 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2957 (response_expected
? 1 << 5 : 0); /* Return data expected */
2959 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
2960 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
2961 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
2962 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
2964 brw_inst_set_dp_msg_type(devinfo
, insn
,
2965 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
);
2967 brw_inst_set_dp_msg_type(devinfo
, insn
,
2968 HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
);
2972 brw_inst_set_dp_msg_type(devinfo
, insn
,
2973 GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
);
2975 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
2976 msg_control
|= 1 << 4; /* Use high 8 slots of the sample mask */
2979 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
2983 brw_typed_atomic(struct brw_codegen
*p
,
2985 struct brw_reg payload
,
2986 struct brw_reg surface
,
2988 unsigned msg_length
,
2989 bool response_expected
) {
2990 const struct brw_device_info
*devinfo
= p
->devinfo
;
2991 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
2992 HSW_SFID_DATAPORT_DATA_CACHE_1
:
2993 GEN6_SFID_DATAPORT_RENDER_CACHE
);
2994 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
2995 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
2996 const unsigned mask
= align1
? WRITEMASK_XYZW
: WRITEMASK_X
;
2997 struct brw_inst
*insn
= brw_send_indirect_surface_message(
2998 p
, sfid
, brw_writemask(dst
, mask
), payload
, surface
, msg_length
,
2999 brw_surface_payload_size(p
, response_expected
,
3000 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3003 brw_set_dp_typed_atomic_message(
3004 p
, insn
, atomic_op
, response_expected
);
3008 brw_set_dp_typed_surface_read_message(struct brw_codegen
*p
,
3009 struct brw_inst
*insn
,
3010 unsigned num_channels
)
3012 const struct brw_device_info
*devinfo
= p
->devinfo
;
3013 /* Set mask of unused channels. */
3014 unsigned msg_control
= 0xf & (0xf << num_channels
);
3016 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3017 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3018 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3019 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3021 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3024 brw_inst_set_dp_msg_type(devinfo
, insn
,
3025 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
);
3027 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3028 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3029 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3032 brw_inst_set_dp_msg_type(devinfo
, insn
,
3033 GEN7_DATAPORT_RC_TYPED_SURFACE_READ
);
3036 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3040 brw_typed_surface_read(struct brw_codegen
*p
,
3042 struct brw_reg payload
,
3043 struct brw_reg surface
,
3044 unsigned msg_length
,
3045 unsigned num_channels
)
3047 const struct brw_device_info
*devinfo
= p
->devinfo
;
3048 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3049 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3050 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3051 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3052 p
, sfid
, dst
, payload
, surface
, msg_length
,
3053 brw_surface_payload_size(p
, num_channels
,
3054 devinfo
->gen
>= 8 || devinfo
->is_haswell
, false),
3057 brw_set_dp_typed_surface_read_message(
3058 p
, insn
, num_channels
);
3062 brw_set_dp_typed_surface_write_message(struct brw_codegen
*p
,
3063 struct brw_inst
*insn
,
3064 unsigned num_channels
)
3066 const struct brw_device_info
*devinfo
= p
->devinfo
;
3067 /* Set mask of unused channels. */
3068 unsigned msg_control
= 0xf & (0xf << num_channels
);
3070 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
3071 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3072 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3073 msg_control
|= 2 << 4; /* Use high 8 slots of the sample mask */
3075 msg_control
|= 1 << 4; /* Use low 8 slots of the sample mask */
3078 brw_inst_set_dp_msg_type(devinfo
, insn
,
3079 HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
);
3082 if (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
) {
3083 if (brw_inst_qtr_control(devinfo
, p
->current
) == GEN6_COMPRESSION_2Q
)
3084 msg_control
|= 1 << 5; /* Use high 8 slots of the sample mask */
3087 brw_inst_set_dp_msg_type(devinfo
, insn
,
3088 GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
);
3091 brw_inst_set_dp_msg_control(devinfo
, insn
, msg_control
);
3095 brw_typed_surface_write(struct brw_codegen
*p
,
3096 struct brw_reg payload
,
3097 struct brw_reg surface
,
3098 unsigned msg_length
,
3099 unsigned num_channels
)
3101 const struct brw_device_info
*devinfo
= p
->devinfo
;
3102 const unsigned sfid
= (devinfo
->gen
>= 8 || devinfo
->is_haswell
?
3103 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3104 GEN6_SFID_DATAPORT_RENDER_CACHE
);
3105 const bool align1
= (brw_inst_access_mode(devinfo
, p
->current
) == BRW_ALIGN_1
);
3106 /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3107 const unsigned mask
= (devinfo
->gen
== 7 && !devinfo
->is_haswell
&& !align1
?
3108 WRITEMASK_X
: WRITEMASK_XYZW
);
3109 struct brw_inst
*insn
= brw_send_indirect_surface_message(
3110 p
, sfid
, brw_writemask(brw_null_reg(), mask
),
3111 payload
, surface
, msg_length
, 0, true);
3113 brw_set_dp_typed_surface_write_message(
3114 p
, insn
, num_channels
);
3118 brw_set_memory_fence_message(struct brw_codegen
*p
,
3119 struct brw_inst
*insn
,
3120 enum brw_message_target sfid
,
3123 const struct brw_device_info
*devinfo
= p
->devinfo
;
3125 brw_set_message_descriptor(p
, insn
, sfid
,
3126 1 /* message length */,
3127 (commit_enable
? 1 : 0) /* response length */,
3128 true /* header present */,
3132 case GEN6_SFID_DATAPORT_RENDER_CACHE
:
3133 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_RC_MEMORY_FENCE
);
3135 case GEN7_SFID_DATAPORT_DATA_CACHE
:
3136 brw_inst_set_dp_msg_type(devinfo
, insn
, GEN7_DATAPORT_DC_MEMORY_FENCE
);
3139 unreachable("Not reached");
3143 brw_inst_set_dp_msg_control(devinfo
, insn
, 1 << 5);
3147 brw_memory_fence(struct brw_codegen
*p
,
3150 const struct brw_device_info
*devinfo
= p
->devinfo
;
3151 const bool commit_enable
= devinfo
->gen
== 7 && !devinfo
->is_haswell
;
3152 struct brw_inst
*insn
;
3154 /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3155 * message doesn't write anything back.
3157 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3158 brw_set_dest(p
, insn
, dst
);
3159 brw_set_src0(p
, insn
, dst
);
3160 brw_set_memory_fence_message(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
3163 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
3164 /* IVB does typed surface access through the render cache, so we need to
3165 * flush it too. Use a different register so both flushes can be
3166 * pipelined by the hardware.
3168 insn
= next_insn(p
, BRW_OPCODE_SEND
);
3169 brw_set_dest(p
, insn
, offset(dst
, 1));
3170 brw_set_src0(p
, insn
, offset(dst
, 1));
3171 brw_set_memory_fence_message(p
, insn
, GEN6_SFID_DATAPORT_RENDER_CACHE
,
3174 /* Now write the response of the second message into the response of the
3175 * first to trigger a pipeline stall -- This way future render and data
3176 * cache messages will be properly ordered with respect to past data and
3177 * render cache messages.
3179 brw_push_insn_state(p
);
3180 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3181 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3182 brw_MOV(p
, dst
, offset(dst
, 1));
3183 brw_pop_insn_state(p
);
3188 brw_pixel_interpolator_query(struct brw_codegen
*p
,
3189 struct brw_reg dest
,
3194 unsigned msg_length
,
3195 unsigned response_length
)
3197 const struct brw_device_info
*devinfo
= p
->devinfo
;
3198 struct brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
3200 brw_set_dest(p
, insn
, dest
);
3201 brw_set_src0(p
, insn
, mrf
);
3202 brw_set_message_descriptor(p
, insn
, GEN7_SFID_PIXEL_INTERPOLATOR
,
3203 msg_length
, response_length
,
3204 false /* header is never present for PI */,
3207 brw_inst_set_pi_simd_mode(
3208 devinfo
, insn
, brw_inst_exec_size(devinfo
, insn
) == BRW_EXECUTE_16
);
3209 brw_inst_set_pi_slot_group(devinfo
, insn
, 0); /* zero unless 32/64px dispatch */
3210 brw_inst_set_pi_nopersp(devinfo
, insn
, noperspective
);
3211 brw_inst_set_pi_message_type(devinfo
, insn
, mode
);
3212 brw_inst_set_pi_message_data(devinfo
, insn
, data
);
3216 * This instruction is generated as a single-channel align1 instruction by
3217 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3219 * We can't use the typed atomic op in the FS because that has the execution
3220 * mask ANDed with the pixel mask, but we just want to write the one dword for
3223 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3224 * one u32. So we use the same untyped atomic write message as the pixel
3227 * The untyped atomic operation requires a BUFFER surface type with RAW
3228 * format, and is only accessible through the legacy DATA_CACHE dataport
3231 void brw_shader_time_add(struct brw_codegen
*p
,
3232 struct brw_reg payload
,
3233 uint32_t surf_index
)
3235 const unsigned sfid
= (p
->devinfo
->gen
>= 8 || p
->devinfo
->is_haswell
?
3236 HSW_SFID_DATAPORT_DATA_CACHE_1
:
3237 GEN7_SFID_DATAPORT_DATA_CACHE
);
3238 assert(p
->devinfo
->gen
>= 7);
3240 brw_push_insn_state(p
);
3241 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
3242 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
3243 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
3244 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
3246 /* We use brw_vec1_reg and unmasked because we want to increment the given
3249 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
3251 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
3253 brw_set_src1(p
, send
, brw_imm_ud(0));
3254 brw_set_message_descriptor(p
, send
, sfid
, 2, 0, false, false);
3255 brw_inst_set_binding_table_index(p
->devinfo
, send
, surf_index
);
3256 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, false);
3258 brw_pop_insn_state(p
);