2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
33 #include "brw_context.h"
34 #include "brw_defines.h"
37 #include "util/ralloc.h"
39 /***********************************************************************
40 * Internal helper for constructing instructions
43 static void guess_execution_size(struct brw_compile
*p
,
47 const struct brw_context
*brw
= p
->brw
;
49 if (reg
.width
== BRW_WIDTH_8
&& p
->compressed
) {
50 brw_inst_set_exec_size(brw
, insn
, BRW_EXECUTE_16
);
52 /* Register width definitions are compatible with BRW_EXECUTE_* enums. */
53 brw_inst_set_exec_size(brw
, insn
, reg
.width
);
59 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
60 * registers, implicitly moving the operand to a message register.
62 * On Sandybridge, this is no longer the case. This function performs the
63 * explicit move; it should be called before emitting a SEND instruction.
66 gen6_resolve_implied_move(struct brw_compile
*p
,
70 struct brw_context
*brw
= p
->brw
;
74 if (src
->file
== BRW_MESSAGE_REGISTER_FILE
)
77 if (src
->file
!= BRW_ARCHITECTURE_REGISTER_FILE
|| src
->nr
!= BRW_ARF_NULL
) {
78 brw_push_insn_state(p
);
79 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
80 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
81 brw_MOV(p
, retype(brw_message_reg(msg_reg_nr
), BRW_REGISTER_TYPE_UD
),
82 retype(*src
, BRW_REGISTER_TYPE_UD
));
83 brw_pop_insn_state(p
);
85 *src
= brw_message_reg(msg_reg_nr
);
89 gen7_convert_mrf_to_grf(struct brw_compile
*p
, struct brw_reg
*reg
)
91 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
92 * "The send with EOT should use register space R112-R127 for <src>. This is
93 * to enable loading of a new thread into the same slot while the message
94 * with EOT for current thread is pending dispatch."
96 * Since we're pretending to have 16 MRFs anyway, we may as well use the
97 * registers required for messages with EOT.
99 struct brw_context
*brw
= p
->brw
;
100 if (brw
->gen
>= 7 && reg
->file
== BRW_MESSAGE_REGISTER_FILE
) {
101 reg
->file
= BRW_GENERAL_REGISTER_FILE
;
102 reg
->nr
+= GEN7_MRF_HACK_START
;
107 * Convert a brw_reg_type enumeration value into the hardware representation.
109 * The hardware encoding may depend on whether the value is an immediate.
112 brw_reg_type_to_hw_type(const struct brw_context
*brw
,
113 enum brw_reg_type type
, unsigned file
)
115 if (file
== BRW_IMMEDIATE_VALUE
) {
116 const static int imm_hw_types
[] = {
117 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
118 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
119 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
120 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
121 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
122 [BRW_REGISTER_TYPE_UB
] = -1,
123 [BRW_REGISTER_TYPE_B
] = -1,
124 [BRW_REGISTER_TYPE_UV
] = BRW_HW_REG_IMM_TYPE_UV
,
125 [BRW_REGISTER_TYPE_VF
] = BRW_HW_REG_IMM_TYPE_VF
,
126 [BRW_REGISTER_TYPE_V
] = BRW_HW_REG_IMM_TYPE_V
,
127 [BRW_REGISTER_TYPE_DF
] = GEN8_HW_REG_IMM_TYPE_DF
,
128 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_IMM_TYPE_HF
,
129 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
130 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
132 assert(type
< ARRAY_SIZE(imm_hw_types
));
133 assert(imm_hw_types
[type
] != -1);
134 assert(brw
->gen
>= 8 || type
< BRW_REGISTER_TYPE_DF
);
135 return imm_hw_types
[type
];
137 /* Non-immediate registers */
138 const static int hw_types
[] = {
139 [BRW_REGISTER_TYPE_UD
] = BRW_HW_REG_TYPE_UD
,
140 [BRW_REGISTER_TYPE_D
] = BRW_HW_REG_TYPE_D
,
141 [BRW_REGISTER_TYPE_UW
] = BRW_HW_REG_TYPE_UW
,
142 [BRW_REGISTER_TYPE_W
] = BRW_HW_REG_TYPE_W
,
143 [BRW_REGISTER_TYPE_UB
] = BRW_HW_REG_NON_IMM_TYPE_UB
,
144 [BRW_REGISTER_TYPE_B
] = BRW_HW_REG_NON_IMM_TYPE_B
,
145 [BRW_REGISTER_TYPE_F
] = BRW_HW_REG_TYPE_F
,
146 [BRW_REGISTER_TYPE_UV
] = -1,
147 [BRW_REGISTER_TYPE_VF
] = -1,
148 [BRW_REGISTER_TYPE_V
] = -1,
149 [BRW_REGISTER_TYPE_DF
] = GEN7_HW_REG_NON_IMM_TYPE_DF
,
150 [BRW_REGISTER_TYPE_HF
] = GEN8_HW_REG_NON_IMM_TYPE_HF
,
151 [BRW_REGISTER_TYPE_UQ
] = GEN8_HW_REG_TYPE_UQ
,
152 [BRW_REGISTER_TYPE_Q
] = GEN8_HW_REG_TYPE_Q
,
154 assert(type
< ARRAY_SIZE(hw_types
));
155 assert(hw_types
[type
] != -1);
156 assert(brw
->gen
>= 7 || type
< BRW_REGISTER_TYPE_DF
);
157 assert(brw
->gen
>= 8 || type
< BRW_REGISTER_TYPE_HF
);
158 return hw_types
[type
];
163 brw_set_dest(struct brw_compile
*p
, brw_inst
*inst
, struct brw_reg dest
)
165 const struct brw_context
*brw
= p
->brw
;
167 if (dest
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
&&
168 dest
.file
!= BRW_MESSAGE_REGISTER_FILE
)
169 assert(dest
.nr
< 128);
171 gen7_convert_mrf_to_grf(p
, &dest
);
173 brw_inst_set_dst_reg_file(brw
, inst
, dest
.file
);
174 brw_inst_set_dst_reg_type(brw
, inst
, brw_reg_type_to_hw_type(brw
, dest
.type
,
176 brw_inst_set_dst_address_mode(brw
, inst
, dest
.address_mode
);
178 if (dest
.address_mode
== BRW_ADDRESS_DIRECT
) {
179 brw_inst_set_dst_da_reg_nr(brw
, inst
, dest
.nr
);
181 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
182 brw_inst_set_dst_da1_subreg_nr(brw
, inst
, dest
.subnr
);
183 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
184 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
185 brw_inst_set_dst_hstride(brw
, inst
, dest
.hstride
);
187 brw_inst_set_dst_da16_subreg_nr(brw
, inst
, dest
.subnr
/ 16);
188 brw_inst_set_da16_writemask(brw
, inst
, dest
.dw1
.bits
.writemask
);
189 if (dest
.file
== BRW_GENERAL_REGISTER_FILE
||
190 dest
.file
== BRW_MESSAGE_REGISTER_FILE
) {
191 assert(dest
.dw1
.bits
.writemask
!= 0);
193 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
194 * Although Dst.HorzStride is a don't care for Align16, HW needs
195 * this to be programmed as "01".
197 brw_inst_set_dst_hstride(brw
, inst
, 1);
200 brw_inst_set_dst_ia_subreg_nr(brw
, inst
, dest
.subnr
);
202 /* These are different sizes in align1 vs align16:
204 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
205 brw_inst_set_dst_ia1_addr_imm(brw
, inst
,
206 dest
.dw1
.bits
.indirect_offset
);
207 if (dest
.hstride
== BRW_HORIZONTAL_STRIDE_0
)
208 dest
.hstride
= BRW_HORIZONTAL_STRIDE_1
;
209 brw_inst_set_dst_hstride(brw
, inst
, dest
.hstride
);
211 brw_inst_set_dst_ia16_addr_imm(brw
, inst
,
212 dest
.dw1
.bits
.indirect_offset
);
213 /* even ignored in da16, still need to set as '01' */
214 brw_inst_set_dst_hstride(brw
, inst
, 1);
218 /* NEW: Set the execution size based on dest.width and
219 * inst->compression_control:
221 guess_execution_size(p
, inst
, dest
);
224 extern int reg_type_size
[];
227 validate_reg(const struct brw_context
*brw
, brw_inst
*inst
, struct brw_reg reg
)
229 int hstride_for_reg
[] = {0, 1, 2, 4};
230 int vstride_for_reg
[] = {0, 1, 2, 4, 8, 16, 32, 64, 128, 256};
231 int width_for_reg
[] = {1, 2, 4, 8, 16};
232 int execsize_for_reg
[] = {1, 2, 4, 8, 16};
233 int width
, hstride
, vstride
, execsize
;
235 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
236 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
237 * mean the destination has to be 128-bit aligned and the
238 * destination horiz stride has to be a word.
240 if (reg
.type
== BRW_REGISTER_TYPE_V
) {
241 assert(hstride_for_reg
[brw_inst_dst_hstride(brw
, inst
)] *
242 reg_type_size
[brw_inst_dst_reg_type(brw
, inst
)] == 2);
248 if (reg
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
249 reg
.file
== BRW_ARF_NULL
)
252 assert(reg
.hstride
>= 0 && reg
.hstride
< Elements(hstride_for_reg
));
253 hstride
= hstride_for_reg
[reg
.hstride
];
255 if (reg
.vstride
== 0xf) {
258 assert(reg
.vstride
>= 0 && reg
.vstride
< Elements(vstride_for_reg
));
259 vstride
= vstride_for_reg
[reg
.vstride
];
262 assert(reg
.width
>= 0 && reg
.width
< Elements(width_for_reg
));
263 width
= width_for_reg
[reg
.width
];
265 assert(brw_inst_exec_size(brw
, inst
) >= 0 &&
266 brw_inst_exec_size(brw
, inst
) < Elements(execsize_for_reg
));
267 execsize
= execsize_for_reg
[brw_inst_exec_size(brw
, inst
)];
269 /* Restrictions from 3.3.10: Register Region Restrictions. */
271 assert(execsize
>= width
);
274 if (execsize
== width
&& hstride
!= 0) {
275 assert(vstride
== -1 || vstride
== width
* hstride
);
279 if (execsize
== width
&& hstride
== 0) {
280 /* no restriction on vstride. */
285 assert(hstride
== 0);
289 if (execsize
== 1 && width
== 1) {
290 assert(hstride
== 0);
291 assert(vstride
== 0);
295 if (vstride
== 0 && hstride
== 0) {
299 /* 10. Check destination issues. */
303 is_compactable_immediate(unsigned imm
)
305 /* We get the low 12 bits as-is. */
308 /* We get one bit replicated through the top 20 bits. */
309 return imm
== 0 || imm
== 0xfffff000;
313 brw_set_src0(struct brw_compile
*p
, brw_inst
*inst
, struct brw_reg reg
)
315 struct brw_context
*brw
= p
->brw
;
317 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
318 assert(reg
.nr
< 128);
320 gen7_convert_mrf_to_grf(p
, ®
);
322 if (brw
->gen
>= 6 && (brw_inst_opcode(brw
, inst
) == BRW_OPCODE_SEND
||
323 brw_inst_opcode(brw
, inst
) == BRW_OPCODE_SENDC
)) {
324 /* Any source modifiers or regions will be ignored, since this just
325 * identifies the MRF/GRF to start reading the message contents from.
326 * Check for some likely failures.
330 assert(reg
.address_mode
== BRW_ADDRESS_DIRECT
);
333 validate_reg(brw
, inst
, reg
);
335 brw_inst_set_src0_reg_file(brw
, inst
, reg
.file
);
336 brw_inst_set_src0_reg_type(brw
, inst
,
337 brw_reg_type_to_hw_type(brw
, reg
.type
, reg
.file
));
338 brw_inst_set_src0_abs(brw
, inst
, reg
.abs
);
339 brw_inst_set_src0_negate(brw
, inst
, reg
.negate
);
340 brw_inst_set_src0_address_mode(brw
, inst
, reg
.address_mode
);
342 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
343 brw_inst_set_imm_ud(brw
, inst
, reg
.dw1
.ud
);
345 /* The Bspec's section titled "Non-present Operands" claims that if src0
346 * is an immediate that src1's type must be the same as that of src0.
348 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
349 * that do not follow this rule. E.g., from the IVB/HSW table:
351 * DataTypeIndex 18-Bit Mapping Mapped Meaning
352 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
354 * And from the SNB table:
356 * DataTypeIndex 18-Bit Mapping Mapped Meaning
357 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
359 * Neither of these cause warnings from the simulator when used,
360 * compacted or otherwise. In fact, all compaction mappings that have an
361 * immediate in src0 use a:ud for src1.
363 * The GM45 instruction compaction tables do not contain mapped meanings
364 * so it's not clear whether it has the restriction. We'll assume it was
365 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
367 brw_inst_set_src1_reg_file(brw
, inst
, BRW_ARCHITECTURE_REGISTER_FILE
);
369 brw_inst_set_src1_reg_type(brw
, inst
,
370 brw_inst_src0_reg_type(brw
, inst
));
372 brw_inst_set_src1_reg_type(brw
, inst
, BRW_HW_REG_TYPE_UD
);
375 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
376 * for immediate values. Presumably the hardware engineers realized
377 * that the only useful floating-point value that could be represented
378 * in this format is 0.0, which can also be represented as a VF-typed
379 * immediate, so they gave us the previously mentioned mapping on IVB+.
381 * Strangely, we do have a mapping for imm:f in src1, so we don't need
384 * If we see a 0.0:F, change the type to VF so that it can be compacted.
386 if (brw_inst_imm_ud(brw
, inst
) == 0x0 &&
387 brw_inst_src0_reg_type(brw
, inst
) == BRW_HW_REG_TYPE_F
) {
388 brw_inst_set_src0_reg_type(brw
, inst
, BRW_HW_REG_IMM_TYPE_VF
);
391 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
392 * set the types to :UD so the instruction can be compacted.
394 if (is_compactable_immediate(brw_inst_imm_ud(brw
, inst
)) &&
395 brw_inst_cond_modifier(brw
, inst
) == BRW_CONDITIONAL_NONE
&&
396 brw_inst_src0_reg_type(brw
, inst
) == BRW_HW_REG_TYPE_D
&&
397 brw_inst_dst_reg_type(brw
, inst
) == BRW_HW_REG_TYPE_D
) {
398 brw_inst_set_src0_reg_type(brw
, inst
, BRW_HW_REG_TYPE_UD
);
399 brw_inst_set_dst_reg_type(brw
, inst
, BRW_HW_REG_TYPE_UD
);
402 if (reg
.address_mode
== BRW_ADDRESS_DIRECT
) {
403 brw_inst_set_src0_da_reg_nr(brw
, inst
, reg
.nr
);
404 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
405 brw_inst_set_src0_da1_subreg_nr(brw
, inst
, reg
.subnr
);
407 brw_inst_set_src0_da16_subreg_nr(brw
, inst
, reg
.subnr
/ 16);
410 brw_inst_set_src0_ia_subreg_nr(brw
, inst
, reg
.subnr
);
412 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
413 brw_inst_set_src0_ia1_addr_imm(brw
, inst
, reg
.dw1
.bits
.indirect_offset
);
415 brw_inst_set_src0_ia_subreg_nr(brw
, inst
, reg
.dw1
.bits
.indirect_offset
);
419 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
420 if (reg
.width
== BRW_WIDTH_1
&&
421 brw_inst_exec_size(brw
, inst
) == BRW_EXECUTE_1
) {
422 brw_inst_set_src0_hstride(brw
, inst
, BRW_HORIZONTAL_STRIDE_0
);
423 brw_inst_set_src0_width(brw
, inst
, BRW_WIDTH_1
);
424 brw_inst_set_src0_vstride(brw
, inst
, BRW_VERTICAL_STRIDE_0
);
426 brw_inst_set_src0_hstride(brw
, inst
, reg
.hstride
);
427 brw_inst_set_src0_width(brw
, inst
, reg
.width
);
428 brw_inst_set_src0_vstride(brw
, inst
, reg
.vstride
);
431 brw_inst_set_src0_da16_swiz_x(brw
, inst
,
432 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_X
));
433 brw_inst_set_src0_da16_swiz_y(brw
, inst
,
434 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Y
));
435 brw_inst_set_src0_da16_swiz_z(brw
, inst
,
436 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Z
));
437 brw_inst_set_src0_da16_swiz_w(brw
, inst
,
438 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_W
));
440 /* This is an oddity of the fact we're using the same
441 * descriptions for registers in align_16 as align_1:
443 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
444 brw_inst_set_src0_vstride(brw
, inst
, BRW_VERTICAL_STRIDE_4
);
446 brw_inst_set_src0_vstride(brw
, inst
, reg
.vstride
);
453 brw_set_src1(struct brw_compile
*p
, brw_inst
*inst
, struct brw_reg reg
)
455 const struct brw_context
*brw
= p
->brw
;
456 assert(reg
.file
!= BRW_MESSAGE_REGISTER_FILE
);
458 if (reg
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
)
459 assert(reg
.nr
< 128);
461 gen7_convert_mrf_to_grf(p
, ®
);
463 validate_reg(brw
, inst
, reg
);
465 brw_inst_set_src1_reg_file(brw
, inst
, reg
.file
);
466 brw_inst_set_src1_reg_type(brw
, inst
,
467 brw_reg_type_to_hw_type(brw
, reg
.type
, reg
.file
));
468 brw_inst_set_src1_abs(brw
, inst
, reg
.abs
);
469 brw_inst_set_src1_negate(brw
, inst
, reg
.negate
);
471 /* Only src1 can be immediate in two-argument instructions.
473 assert(brw_inst_src0_reg_file(brw
, inst
) != BRW_IMMEDIATE_VALUE
);
475 if (reg
.file
== BRW_IMMEDIATE_VALUE
) {
476 brw_inst_set_imm_ud(brw
, inst
, reg
.dw1
.ud
);
478 /* This is a hardware restriction, which may or may not be lifted
481 assert (reg
.address_mode
== BRW_ADDRESS_DIRECT
);
482 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
484 brw_inst_set_src1_da_reg_nr(brw
, inst
, reg
.nr
);
485 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
486 brw_inst_set_src1_da1_subreg_nr(brw
, inst
, reg
.subnr
);
488 brw_inst_set_src1_da16_subreg_nr(brw
, inst
, reg
.subnr
/ 16);
491 if (brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_1
) {
492 if (reg
.width
== BRW_WIDTH_1
&&
493 brw_inst_exec_size(brw
, inst
) == BRW_EXECUTE_1
) {
494 brw_inst_set_src1_hstride(brw
, inst
, BRW_HORIZONTAL_STRIDE_0
);
495 brw_inst_set_src1_width(brw
, inst
, BRW_WIDTH_1
);
496 brw_inst_set_src1_vstride(brw
, inst
, BRW_VERTICAL_STRIDE_0
);
498 brw_inst_set_src1_hstride(brw
, inst
, reg
.hstride
);
499 brw_inst_set_src1_width(brw
, inst
, reg
.width
);
500 brw_inst_set_src1_vstride(brw
, inst
, reg
.vstride
);
503 brw_inst_set_src1_da16_swiz_x(brw
, inst
,
504 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_X
));
505 brw_inst_set_src1_da16_swiz_y(brw
, inst
,
506 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Y
));
507 brw_inst_set_src1_da16_swiz_z(brw
, inst
,
508 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_Z
));
509 brw_inst_set_src1_da16_swiz_w(brw
, inst
,
510 BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, BRW_CHANNEL_W
));
512 /* This is an oddity of the fact we're using the same
513 * descriptions for registers in align_16 as align_1:
515 if (reg
.vstride
== BRW_VERTICAL_STRIDE_8
)
516 brw_inst_set_src1_vstride(brw
, inst
, BRW_VERTICAL_STRIDE_4
);
518 brw_inst_set_src1_vstride(brw
, inst
, reg
.vstride
);
524 * Set the Message Descriptor and Extended Message Descriptor fields
527 * \note This zeroes out the Function Control bits, so it must be called
528 * \b before filling out any message-specific data. Callers can
529 * choose not to fill in irrelevant bits; they will be zero.
532 brw_set_message_descriptor(struct brw_compile
*p
,
534 enum brw_message_target sfid
,
536 unsigned response_length
,
540 struct brw_context
*brw
= p
->brw
;
542 brw_set_src1(p
, inst
, brw_imm_d(0));
543 brw_inst_set_sfid(brw
, inst
, sfid
);
544 brw_inst_set_mlen(brw
, inst
, msg_length
);
545 brw_inst_set_rlen(brw
, inst
, response_length
);
546 brw_inst_set_eot(brw
, inst
, end_of_thread
);
549 brw_inst_set_header_present(brw
, inst
, header_present
);
553 static void brw_set_math_message( struct brw_compile
*p
,
556 unsigned integer_type
,
560 struct brw_context
*brw
= p
->brw
;
562 unsigned response_length
;
564 /* Infer message length from the function */
566 case BRW_MATH_FUNCTION_POW
:
567 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
:
568 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER
:
569 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
577 /* Infer response length from the function */
579 case BRW_MATH_FUNCTION_SINCOS
:
580 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
:
589 brw_set_message_descriptor(p
, inst
, BRW_SFID_MATH
,
590 msg_length
, response_length
, false, false);
591 brw_inst_set_math_msg_function(brw
, inst
, function
);
592 brw_inst_set_math_msg_signed_int(brw
, inst
, integer_type
);
593 brw_inst_set_math_msg_precision(brw
, inst
, low_precision
);
594 brw_inst_set_math_msg_saturate(brw
, inst
, brw_inst_saturate(brw
, inst
));
595 brw_inst_set_math_msg_data_type(brw
, inst
, dataType
);
596 brw_inst_set_saturate(brw
, inst
, 0);
600 static void brw_set_ff_sync_message(struct brw_compile
*p
,
603 unsigned response_length
,
606 const struct brw_context
*brw
= p
->brw
;
608 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
609 1, response_length
, true, end_of_thread
);
610 brw_inst_set_urb_opcode(brw
, insn
, 1); /* FF_SYNC */
611 brw_inst_set_urb_allocate(brw
, insn
, allocate
);
612 /* The following fields are not used by FF_SYNC: */
613 brw_inst_set_urb_global_offset(brw
, insn
, 0);
614 brw_inst_set_urb_swizzle_control(brw
, insn
, 0);
615 brw_inst_set_urb_used(brw
, insn
, 0);
616 brw_inst_set_urb_complete(brw
, insn
, 0);
619 static void brw_set_urb_message( struct brw_compile
*p
,
621 enum brw_urb_write_flags flags
,
623 unsigned response_length
,
625 unsigned swizzle_control
)
627 struct brw_context
*brw
= p
->brw
;
629 assert(brw
->gen
< 7 || swizzle_control
!= BRW_URB_SWIZZLE_TRANSPOSE
);
630 assert(brw
->gen
< 7 || !(flags
& BRW_URB_WRITE_ALLOCATE
));
631 assert(brw
->gen
>= 7 || !(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
633 brw_set_message_descriptor(p
, insn
, BRW_SFID_URB
,
634 msg_length
, response_length
, true,
635 flags
& BRW_URB_WRITE_EOT
);
637 if (flags
& BRW_URB_WRITE_OWORD
) {
638 assert(msg_length
== 2); /* header + one OWORD of data */
639 brw_inst_set_urb_opcode(brw
, insn
, BRW_URB_OPCODE_WRITE_OWORD
);
641 brw_inst_set_urb_opcode(brw
, insn
, BRW_URB_OPCODE_WRITE_HWORD
);
644 brw_inst_set_urb_global_offset(brw
, insn
, offset
);
645 brw_inst_set_urb_swizzle_control(brw
, insn
, swizzle_control
);
648 brw_inst_set_urb_complete(brw
, insn
, !!(flags
& BRW_URB_WRITE_COMPLETE
));
652 brw_inst_set_urb_allocate(brw
, insn
, !!(flags
& BRW_URB_WRITE_ALLOCATE
));
653 brw_inst_set_urb_used(brw
, insn
, !(flags
& BRW_URB_WRITE_UNUSED
));
655 brw_inst_set_urb_per_slot_offset(brw
, insn
,
656 !!(flags
& BRW_URB_WRITE_PER_SLOT_OFFSET
));
661 brw_set_dp_write_message(struct brw_compile
*p
,
663 unsigned binding_table_index
,
664 unsigned msg_control
,
668 unsigned last_render_target
,
669 unsigned response_length
,
670 unsigned end_of_thread
,
671 unsigned send_commit_msg
)
673 struct brw_context
*brw
= p
->brw
;
677 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
678 if (msg_type
== GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
)
679 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
681 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
682 } else if (brw
->gen
== 6) {
683 /* Use the render cache for all write messages. */
684 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
686 sfid
= BRW_SFID_DATAPORT_WRITE
;
689 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
690 header_present
, end_of_thread
);
692 brw_inst_set_binding_table_index(brw
, insn
, binding_table_index
);
693 brw_inst_set_dp_write_msg_type(brw
, insn
, msg_type
);
694 brw_inst_set_dp_write_msg_control(brw
, insn
, msg_control
);
695 brw_inst_set_rt_last(brw
, insn
, last_render_target
);
697 brw_inst_set_dp_write_commit(brw
, insn
, send_commit_msg
);
702 brw_set_dp_read_message(struct brw_compile
*p
,
704 unsigned binding_table_index
,
705 unsigned msg_control
,
707 unsigned target_cache
,
710 unsigned response_length
)
712 struct brw_context
*brw
= p
->brw
;
716 sfid
= GEN7_SFID_DATAPORT_DATA_CACHE
;
717 } else if (brw
->gen
== 6) {
718 if (target_cache
== BRW_DATAPORT_READ_TARGET_RENDER_CACHE
)
719 sfid
= GEN6_SFID_DATAPORT_RENDER_CACHE
;
721 sfid
= GEN6_SFID_DATAPORT_SAMPLER_CACHE
;
723 sfid
= BRW_SFID_DATAPORT_READ
;
726 brw_set_message_descriptor(p
, insn
, sfid
, msg_length
, response_length
,
727 header_present
, false);
729 brw_inst_set_binding_table_index(brw
, insn
, binding_table_index
);
730 brw_inst_set_dp_read_msg_type(brw
, insn
, msg_type
);
731 brw_inst_set_dp_read_msg_control(brw
, insn
, msg_control
);
733 brw_inst_set_dp_read_target_cache(brw
, insn
, target_cache
);
737 brw_set_sampler_message(struct brw_compile
*p
,
739 unsigned binding_table_index
,
742 unsigned response_length
,
744 unsigned header_present
,
746 unsigned return_format
)
748 struct brw_context
*brw
= p
->brw
;
750 brw_set_message_descriptor(p
, inst
, BRW_SFID_SAMPLER
, msg_length
,
751 response_length
, header_present
, false);
753 brw_inst_set_binding_table_index(brw
, inst
, binding_table_index
);
754 brw_inst_set_sampler(brw
, inst
, sampler
);
755 brw_inst_set_sampler_msg_type(brw
, inst
, msg_type
);
757 brw_inst_set_sampler_simd_mode(brw
, inst
, simd_mode
);
758 } else if (brw
->gen
== 4 && !brw
->is_g4x
) {
759 brw_inst_set_sampler_return_format(brw
, inst
, return_format
);
764 gen7_set_dp_scratch_message(struct brw_compile
*p
,
768 bool invalidate_after_read
,
770 unsigned addr_offset
,
775 const struct brw_context
*brw
= p
->brw
;
776 assert(num_regs
== 1 || num_regs
== 2 || num_regs
== 4 ||
777 (brw
->gen
>= 8 && num_regs
== 8));
778 brw_set_message_descriptor(p
, inst
, GEN7_SFID_DATAPORT_DATA_CACHE
,
779 mlen
, rlen
, header_present
, false);
780 brw_inst_set_dp_category(brw
, inst
, 1); /* Scratch Block Read/Write msgs */
781 brw_inst_set_scratch_read_write(brw
, inst
, write
);
782 brw_inst_set_scratch_type(brw
, inst
, dword
);
783 brw_inst_set_scratch_invalidate_after_read(brw
, inst
, invalidate_after_read
);
784 brw_inst_set_scratch_block_size(brw
, inst
, ffs(num_regs
) - 1);
785 brw_inst_set_scratch_addr_offset(brw
, inst
, addr_offset
);
788 #define next_insn brw_next_insn
790 brw_next_insn(struct brw_compile
*p
, unsigned opcode
)
792 const struct brw_context
*brw
= p
->brw
;
795 if (p
->nr_insn
+ 1 > p
->store_size
) {
797 p
->store
= reralloc(p
->mem_ctx
, p
->store
, brw_inst
, p
->store_size
);
800 p
->next_insn_offset
+= 16;
801 insn
= &p
->store
[p
->nr_insn
++];
802 memcpy(insn
, p
->current
, sizeof(*insn
));
804 brw_inst_set_opcode(brw
, insn
, opcode
);
809 brw_alu1(struct brw_compile
*p
, unsigned opcode
,
810 struct brw_reg dest
, struct brw_reg src
)
812 brw_inst
*insn
= next_insn(p
, opcode
);
813 brw_set_dest(p
, insn
, dest
);
814 brw_set_src0(p
, insn
, src
);
819 brw_alu2(struct brw_compile
*p
, unsigned opcode
,
820 struct brw_reg dest
, struct brw_reg src0
, struct brw_reg src1
)
822 brw_inst
*insn
= next_insn(p
, opcode
);
823 brw_set_dest(p
, insn
, dest
);
824 brw_set_src0(p
, insn
, src0
);
825 brw_set_src1(p
, insn
, src1
);
830 get_3src_subreg_nr(struct brw_reg reg
)
832 if (reg
.vstride
== BRW_VERTICAL_STRIDE_0
) {
833 assert(brw_is_single_value_swizzle(reg
.dw1
.bits
.swizzle
));
834 return reg
.subnr
/ 4 + BRW_GET_SWZ(reg
.dw1
.bits
.swizzle
, 0);
836 return reg
.subnr
/ 4;
841 brw_alu3(struct brw_compile
*p
, unsigned opcode
, struct brw_reg dest
,
842 struct brw_reg src0
, struct brw_reg src1
, struct brw_reg src2
)
844 struct brw_context
*brw
= p
->brw
;
845 brw_inst
*inst
= next_insn(p
, opcode
);
847 gen7_convert_mrf_to_grf(p
, &dest
);
849 assert(brw_inst_access_mode(brw
, inst
) == BRW_ALIGN_16
);
851 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
852 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
853 assert(dest
.nr
< 128);
854 assert(dest
.address_mode
== BRW_ADDRESS_DIRECT
);
855 assert(dest
.type
== BRW_REGISTER_TYPE_F
||
856 dest
.type
== BRW_REGISTER_TYPE_D
||
857 dest
.type
== BRW_REGISTER_TYPE_UD
);
859 brw_inst_set_3src_dst_reg_file(brw
, inst
,
860 dest
.file
== BRW_MESSAGE_REGISTER_FILE
);
862 brw_inst_set_3src_dst_reg_nr(brw
, inst
, dest
.nr
);
863 brw_inst_set_3src_dst_subreg_nr(brw
, inst
, dest
.subnr
/ 16);
864 brw_inst_set_3src_dst_writemask(brw
, inst
, dest
.dw1
.bits
.writemask
);
865 guess_execution_size(p
, inst
, dest
);
867 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
);
868 assert(src0
.address_mode
== BRW_ADDRESS_DIRECT
);
869 assert(src0
.nr
< 128);
870 brw_inst_set_3src_src0_swizzle(brw
, inst
, src0
.dw1
.bits
.swizzle
);
871 brw_inst_set_3src_src0_subreg_nr(brw
, inst
, get_3src_subreg_nr(src0
));
872 brw_inst_set_3src_src0_reg_nr(brw
, inst
, src0
.nr
);
873 brw_inst_set_3src_src0_abs(brw
, inst
, src0
.abs
);
874 brw_inst_set_3src_src0_negate(brw
, inst
, src0
.negate
);
875 brw_inst_set_3src_src0_rep_ctrl(brw
, inst
,
876 src0
.vstride
== BRW_VERTICAL_STRIDE_0
);
878 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
);
879 assert(src1
.address_mode
== BRW_ADDRESS_DIRECT
);
880 assert(src1
.nr
< 128);
881 brw_inst_set_3src_src1_swizzle(brw
, inst
, src1
.dw1
.bits
.swizzle
);
882 brw_inst_set_3src_src1_subreg_nr(brw
, inst
, get_3src_subreg_nr(src1
));
883 brw_inst_set_3src_src1_reg_nr(brw
, inst
, src1
.nr
);
884 brw_inst_set_3src_src1_abs(brw
, inst
, src1
.abs
);
885 brw_inst_set_3src_src1_negate(brw
, inst
, src1
.negate
);
886 brw_inst_set_3src_src1_rep_ctrl(brw
, inst
,
887 src1
.vstride
== BRW_VERTICAL_STRIDE_0
);
889 assert(src2
.file
== BRW_GENERAL_REGISTER_FILE
);
890 assert(src2
.address_mode
== BRW_ADDRESS_DIRECT
);
891 assert(src2
.nr
< 128);
892 brw_inst_set_3src_src2_swizzle(brw
, inst
, src2
.dw1
.bits
.swizzle
);
893 brw_inst_set_3src_src2_subreg_nr(brw
, inst
, get_3src_subreg_nr(src2
));
894 brw_inst_set_3src_src2_reg_nr(brw
, inst
, src2
.nr
);
895 brw_inst_set_3src_src2_abs(brw
, inst
, src2
.abs
);
896 brw_inst_set_3src_src2_negate(brw
, inst
, src2
.negate
);
897 brw_inst_set_3src_src2_rep_ctrl(brw
, inst
,
898 src2
.vstride
== BRW_VERTICAL_STRIDE_0
);
901 /* Set both the source and destination types based on dest.type,
902 * ignoring the source register types. The MAD and LRP emitters ensure
903 * that all four types are float. The BFE and BFI2 emitters, however,
904 * may send us mixed D and UD types and want us to ignore that and use
905 * the destination type.
908 case BRW_REGISTER_TYPE_F
:
909 brw_inst_set_3src_src_type(brw
, inst
, BRW_3SRC_TYPE_F
);
910 brw_inst_set_3src_dst_type(brw
, inst
, BRW_3SRC_TYPE_F
);
912 case BRW_REGISTER_TYPE_D
:
913 brw_inst_set_3src_src_type(brw
, inst
, BRW_3SRC_TYPE_D
);
914 brw_inst_set_3src_dst_type(brw
, inst
, BRW_3SRC_TYPE_D
);
916 case BRW_REGISTER_TYPE_UD
:
917 brw_inst_set_3src_src_type(brw
, inst
, BRW_3SRC_TYPE_UD
);
918 brw_inst_set_3src_dst_type(brw
, inst
, BRW_3SRC_TYPE_UD
);
927 /***********************************************************************
928 * Convenience routines.
931 brw_inst *brw_##OP(struct brw_compile *p, \
932 struct brw_reg dest, \
933 struct brw_reg src0) \
935 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
939 brw_inst *brw_##OP(struct brw_compile *p, \
940 struct brw_reg dest, \
941 struct brw_reg src0, \
942 struct brw_reg src1) \
944 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
948 brw_inst *brw_##OP(struct brw_compile *p, \
949 struct brw_reg dest, \
950 struct brw_reg src0, \
951 struct brw_reg src1, \
952 struct brw_reg src2) \
954 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
958 brw_inst *brw_##OP(struct brw_compile *p, \
959 struct brw_reg dest, \
960 struct brw_reg src0, \
961 struct brw_reg src1, \
962 struct brw_reg src2) \
964 assert(dest.type == BRW_REGISTER_TYPE_F); \
965 assert(src0.type == BRW_REGISTER_TYPE_F); \
966 assert(src1.type == BRW_REGISTER_TYPE_F); \
967 assert(src2.type == BRW_REGISTER_TYPE_F); \
968 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
971 /* Rounding operations (other than RNDD) require two instructions - the first
972 * stores a rounded value (possibly the wrong way) in the dest register, but
973 * also sets a per-channel "increment bit" in the flag register. A predicated
974 * add of 1.0 fixes dest to contain the desired result.
976 * Sandybridge and later appear to round correctly without an ADD.
979 void brw_##OP(struct brw_compile *p, \
980 struct brw_reg dest, \
981 struct brw_reg src) \
983 struct brw_context *brw = p->brw; \
984 brw_inst *rnd, *add; \
985 rnd = next_insn(p, BRW_OPCODE_##OP); \
986 brw_set_dest(p, rnd, dest); \
987 brw_set_src0(p, rnd, src); \
989 if (brw->gen < 6) { \
990 /* turn on round-increments */ \
991 brw_inst_set_cond_modifier(brw, rnd, BRW_CONDITIONAL_R); \
992 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
993 brw_inst_set_pred_control(brw, add, BRW_PREDICATE_NORMAL); \
1035 brw_ADD(struct brw_compile
*p
, struct brw_reg dest
,
1036 struct brw_reg src0
, struct brw_reg src1
)
1039 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1040 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1041 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1042 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1043 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1046 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1047 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1048 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1049 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1050 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1053 return brw_alu2(p
, BRW_OPCODE_ADD
, dest
, src0
, src1
);
1057 brw_AVG(struct brw_compile
*p
, struct brw_reg dest
,
1058 struct brw_reg src0
, struct brw_reg src1
)
1060 assert(dest
.type
== src0
.type
);
1061 assert(src0
.type
== src1
.type
);
1062 switch (src0
.type
) {
1063 case BRW_REGISTER_TYPE_B
:
1064 case BRW_REGISTER_TYPE_UB
:
1065 case BRW_REGISTER_TYPE_W
:
1066 case BRW_REGISTER_TYPE_UW
:
1067 case BRW_REGISTER_TYPE_D
:
1068 case BRW_REGISTER_TYPE_UD
:
1071 unreachable("Bad type for brw_AVG");
1074 return brw_alu2(p
, BRW_OPCODE_AVG
, dest
, src0
, src1
);
1078 brw_MUL(struct brw_compile
*p
, struct brw_reg dest
,
1079 struct brw_reg src0
, struct brw_reg src1
)
1082 if (src0
.type
== BRW_REGISTER_TYPE_D
||
1083 src0
.type
== BRW_REGISTER_TYPE_UD
||
1084 src1
.type
== BRW_REGISTER_TYPE_D
||
1085 src1
.type
== BRW_REGISTER_TYPE_UD
) {
1086 assert(dest
.type
!= BRW_REGISTER_TYPE_F
);
1089 if (src0
.type
== BRW_REGISTER_TYPE_F
||
1090 (src0
.file
== BRW_IMMEDIATE_VALUE
&&
1091 src0
.type
== BRW_REGISTER_TYPE_VF
)) {
1092 assert(src1
.type
!= BRW_REGISTER_TYPE_UD
);
1093 assert(src1
.type
!= BRW_REGISTER_TYPE_D
);
1096 if (src1
.type
== BRW_REGISTER_TYPE_F
||
1097 (src1
.file
== BRW_IMMEDIATE_VALUE
&&
1098 src1
.type
== BRW_REGISTER_TYPE_VF
)) {
1099 assert(src0
.type
!= BRW_REGISTER_TYPE_UD
);
1100 assert(src0
.type
!= BRW_REGISTER_TYPE_D
);
1103 assert(src0
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1104 src0
.nr
!= BRW_ARF_ACCUMULATOR
);
1105 assert(src1
.file
!= BRW_ARCHITECTURE_REGISTER_FILE
||
1106 src1
.nr
!= BRW_ARF_ACCUMULATOR
);
1108 return brw_alu2(p
, BRW_OPCODE_MUL
, dest
, src0
, src1
);
1112 brw_F32TO16(struct brw_compile
*p
, struct brw_reg dst
, struct brw_reg src
)
1114 const struct brw_context
*brw
= p
->brw
;
1115 bool align16
= brw_inst_access_mode(brw
, p
->current
) == BRW_ALIGN_16
;
1118 assert(dst
.type
== BRW_REGISTER_TYPE_UD
);
1120 assert(dst
.type
== BRW_REGISTER_TYPE_W
||
1121 dst
.type
== BRW_REGISTER_TYPE_UW
||
1122 dst
.type
== BRW_REGISTER_TYPE_HF
);
1125 if (brw
->gen
>= 8) {
1127 /* Emulate the Gen7 zeroing bug (see comments in vec4_visitor's
1128 * emit_pack_half_2x16 method.)
1130 brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_UD
), brw_imm_ud(0u));
1132 return brw_MOV(p
, retype(dst
, BRW_REGISTER_TYPE_HF
), src
);
1134 assert(brw
->gen
== 7);
1135 return brw_alu1(p
, BRW_OPCODE_F32TO16
, dst
, src
);
1140 brw_F16TO32(struct brw_compile
*p
, struct brw_reg dst
, struct brw_reg src
)
1142 const struct brw_context
*brw
= p
->brw
;
1143 bool align16
= brw_inst_access_mode(brw
, p
->current
) == BRW_ALIGN_16
;
1146 assert(src
.type
== BRW_REGISTER_TYPE_UD
);
1148 assert(src
.type
== BRW_REGISTER_TYPE_W
||
1149 src
.type
== BRW_REGISTER_TYPE_UW
||
1150 src
.type
== BRW_REGISTER_TYPE_HF
);
1153 if (brw
->gen
>= 8) {
1154 return brw_MOV(p
, dst
, retype(src
, BRW_REGISTER_TYPE_HF
));
1156 assert(brw
->gen
== 7);
1157 return brw_alu1(p
, BRW_OPCODE_F16TO32
, dst
, src
);
1162 void brw_NOP(struct brw_compile
*p
)
1164 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_NOP
);
1165 brw_set_dest(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1166 brw_set_src0(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1167 brw_set_src1(p
, insn
, brw_imm_ud(0x0));
1174 /***********************************************************************
1175 * Comparisons, if/else/endif
1179 brw_JMPI(struct brw_compile
*p
, struct brw_reg index
,
1180 unsigned predicate_control
)
1182 const struct brw_context
*brw
= p
->brw
;
1183 struct brw_reg ip
= brw_ip_reg();
1184 brw_inst
*inst
= brw_alu2(p
, BRW_OPCODE_JMPI
, ip
, ip
, index
);
1186 brw_inst_set_exec_size(brw
, inst
, BRW_EXECUTE_2
);
1187 brw_inst_set_qtr_control(brw
, inst
, BRW_COMPRESSION_NONE
);
1188 brw_inst_set_mask_control(brw
, inst
, BRW_MASK_DISABLE
);
1189 brw_inst_set_pred_control(brw
, inst
, predicate_control
);
1195 push_if_stack(struct brw_compile
*p
, brw_inst
*inst
)
1197 p
->if_stack
[p
->if_stack_depth
] = inst
- p
->store
;
1199 p
->if_stack_depth
++;
1200 if (p
->if_stack_array_size
<= p
->if_stack_depth
) {
1201 p
->if_stack_array_size
*= 2;
1202 p
->if_stack
= reralloc(p
->mem_ctx
, p
->if_stack
, int,
1203 p
->if_stack_array_size
);
1208 pop_if_stack(struct brw_compile
*p
)
1210 p
->if_stack_depth
--;
1211 return &p
->store
[p
->if_stack
[p
->if_stack_depth
]];
1215 push_loop_stack(struct brw_compile
*p
, brw_inst
*inst
)
1217 if (p
->loop_stack_array_size
< p
->loop_stack_depth
) {
1218 p
->loop_stack_array_size
*= 2;
1219 p
->loop_stack
= reralloc(p
->mem_ctx
, p
->loop_stack
, int,
1220 p
->loop_stack_array_size
);
1221 p
->if_depth_in_loop
= reralloc(p
->mem_ctx
, p
->if_depth_in_loop
, int,
1222 p
->loop_stack_array_size
);
1225 p
->loop_stack
[p
->loop_stack_depth
] = inst
- p
->store
;
1226 p
->loop_stack_depth
++;
1227 p
->if_depth_in_loop
[p
->loop_stack_depth
] = 0;
1231 get_inner_do_insn(struct brw_compile
*p
)
1233 return &p
->store
[p
->loop_stack
[p
->loop_stack_depth
- 1]];
1236 /* EU takes the value from the flag register and pushes it onto some
1237 * sort of a stack (presumably merging with any flag value already on
1238 * the stack). Within an if block, the flags at the top of the stack
1239 * control execution on each channel of the unit, eg. on each of the
1240 * 16 pixel values in our wm programs.
1242 * When the matching 'else' instruction is reached (presumably by
1243 * countdown of the instruction count patched in by our ELSE/ENDIF
1244 * functions), the relevent flags are inverted.
1246 * When the matching 'endif' instruction is reached, the flags are
1247 * popped off. If the stack is now empty, normal execution resumes.
1250 brw_IF(struct brw_compile
*p
, unsigned execute_size
)
1252 struct brw_context
*brw
= p
->brw
;
1255 insn
= next_insn(p
, BRW_OPCODE_IF
);
1257 /* Override the defaults for this instruction:
1260 brw_set_dest(p
, insn
, brw_ip_reg());
1261 brw_set_src0(p
, insn
, brw_ip_reg());
1262 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1263 } else if (brw
->gen
== 6) {
1264 brw_set_dest(p
, insn
, brw_imm_w(0));
1265 brw_inst_set_gen6_jump_count(brw
, insn
, 0);
1266 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1267 brw_set_src1(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1269 brw_set_dest(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1270 brw_set_src0(p
, insn
, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D
)));
1271 brw_set_src1(p
, insn
, brw_imm_ud(0));
1272 brw_inst_set_jip(brw
, insn
, 0);
1273 brw_inst_set_uip(brw
, insn
, 0);
1276 brw_inst_set_exec_size(brw
, insn
, execute_size
);
1277 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1278 brw_inst_set_pred_control(brw
, insn
, BRW_PREDICATE_NORMAL
);
1279 brw_inst_set_mask_control(brw
, insn
, BRW_MASK_ENABLE
);
1280 if (!p
->single_program_flow
&& brw
->gen
< 6)
1281 brw_inst_set_thread_control(brw
, insn
, BRW_THREAD_SWITCH
);
1283 push_if_stack(p
, insn
);
1284 p
->if_depth_in_loop
[p
->loop_stack_depth
]++;
1288 /* This function is only used for gen6-style IF instructions with an
1289 * embedded comparison (conditional modifier). It is not used on gen7.
1292 gen6_IF(struct brw_compile
*p
, enum brw_conditional_mod conditional
,
1293 struct brw_reg src0
, struct brw_reg src1
)
1295 const struct brw_context
*brw
= p
->brw
;
1298 insn
= next_insn(p
, BRW_OPCODE_IF
);
1300 brw_set_dest(p
, insn
, brw_imm_w(0));
1301 brw_inst_set_exec_size(brw
, insn
, p
->compressed
? BRW_EXECUTE_16
1303 brw_inst_set_gen6_jump_count(brw
, insn
, 0);
1304 brw_set_src0(p
, insn
, src0
);
1305 brw_set_src1(p
, insn
, src1
);
1307 assert(brw_inst_qtr_control(brw
, insn
) == BRW_COMPRESSION_NONE
);
1308 assert(brw_inst_pred_control(brw
, insn
) == BRW_PREDICATE_NONE
);
1309 brw_inst_set_cond_modifier(brw
, insn
, conditional
);
1311 push_if_stack(p
, insn
);
1316 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1319 convert_IF_ELSE_to_ADD(struct brw_compile
*p
,
1320 brw_inst
*if_inst
, brw_inst
*else_inst
)
1322 const struct brw_context
*brw
= p
->brw
;
1324 /* The next instruction (where the ENDIF would be, if it existed) */
1325 brw_inst
*next_inst
= &p
->store
[p
->nr_insn
];
1327 assert(p
->single_program_flow
);
1328 assert(if_inst
!= NULL
&& brw_inst_opcode(brw
, if_inst
) == BRW_OPCODE_IF
);
1329 assert(else_inst
== NULL
|| brw_inst_opcode(brw
, else_inst
) == BRW_OPCODE_ELSE
);
1330 assert(brw_inst_exec_size(brw
, if_inst
) == BRW_EXECUTE_1
);
1332 /* Convert IF to an ADD instruction that moves the instruction pointer
1333 * to the first instruction of the ELSE block. If there is no ELSE
1334 * block, point to where ENDIF would be. Reverse the predicate.
1336 * There's no need to execute an ENDIF since we don't need to do any
1337 * stack operations, and if we're currently executing, we just want to
1338 * continue normally.
1340 brw_inst_set_opcode(brw
, if_inst
, BRW_OPCODE_ADD
);
1341 brw_inst_set_pred_inv(brw
, if_inst
, true);
1343 if (else_inst
!= NULL
) {
1344 /* Convert ELSE to an ADD instruction that points where the ENDIF
1347 brw_inst_set_opcode(brw
, else_inst
, BRW_OPCODE_ADD
);
1349 brw_inst_set_imm_ud(brw
, if_inst
, (else_inst
- if_inst
+ 1) * 16);
1350 brw_inst_set_imm_ud(brw
, else_inst
, (next_inst
- else_inst
) * 16);
1352 brw_inst_set_imm_ud(brw
, if_inst
, (next_inst
- if_inst
) * 16);
1357 * Patch IF and ELSE instructions with appropriate jump targets.
1360 patch_IF_ELSE(struct brw_compile
*p
,
1361 brw_inst
*if_inst
, brw_inst
*else_inst
, brw_inst
*endif_inst
)
1363 struct brw_context
*brw
= p
->brw
;
1365 /* We shouldn't be patching IF and ELSE instructions in single program flow
1366 * mode when gen < 6, because in single program flow mode on those
1367 * platforms, we convert flow control instructions to conditional ADDs that
1368 * operate on IP (see brw_ENDIF).
1370 * However, on Gen6, writing to IP doesn't work in single program flow mode
1371 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1372 * not be updated by non-flow control instructions."). And on later
1373 * platforms, there is no significant benefit to converting control flow
1374 * instructions to conditional ADDs. So we do patch IF and ELSE
1375 * instructions in single program flow mode on those platforms.
1378 assert(!p
->single_program_flow
);
1380 assert(if_inst
!= NULL
&& brw_inst_opcode(brw
, if_inst
) == BRW_OPCODE_IF
);
1381 assert(endif_inst
!= NULL
);
1382 assert(else_inst
== NULL
|| brw_inst_opcode(brw
, else_inst
) == BRW_OPCODE_ELSE
);
1384 unsigned br
= brw_jump_scale(brw
);
1386 assert(brw_inst_opcode(brw
, endif_inst
) == BRW_OPCODE_ENDIF
);
1387 brw_inst_set_exec_size(brw
, endif_inst
, brw_inst_exec_size(brw
, if_inst
));
1389 if (else_inst
== NULL
) {
1390 /* Patch IF -> ENDIF */
1392 /* Turn it into an IFF, which means no mask stack operations for
1393 * all-false and jumping past the ENDIF.
1395 brw_inst_set_opcode(brw
, if_inst
, BRW_OPCODE_IFF
);
1396 brw_inst_set_gen4_jump_count(brw
, if_inst
,
1397 br
* (endif_inst
- if_inst
+ 1));
1398 brw_inst_set_gen4_pop_count(brw
, if_inst
, 0);
1399 } else if (brw
->gen
== 6) {
1400 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1401 brw_inst_set_gen6_jump_count(brw
, if_inst
, br
*(endif_inst
- if_inst
));
1403 brw_inst_set_uip(brw
, if_inst
, br
* (endif_inst
- if_inst
));
1404 brw_inst_set_jip(brw
, if_inst
, br
* (endif_inst
- if_inst
));
1407 brw_inst_set_exec_size(brw
, else_inst
, brw_inst_exec_size(brw
, if_inst
));
1409 /* Patch IF -> ELSE */
1411 brw_inst_set_gen4_jump_count(brw
, if_inst
,
1412 br
* (else_inst
- if_inst
));
1413 brw_inst_set_gen4_pop_count(brw
, if_inst
, 0);
1414 } else if (brw
->gen
== 6) {
1415 brw_inst_set_gen6_jump_count(brw
, if_inst
,
1416 br
* (else_inst
- if_inst
+ 1));
1419 /* Patch ELSE -> ENDIF */
1421 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1424 brw_inst_set_gen4_jump_count(brw
, else_inst
,
1425 br
* (endif_inst
- else_inst
+ 1));
1426 brw_inst_set_gen4_pop_count(brw
, else_inst
, 1);
1427 } else if (brw
->gen
== 6) {
1428 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1429 brw_inst_set_gen6_jump_count(brw
, else_inst
,
1430 br
* (endif_inst
- else_inst
));
1432 /* The IF instruction's JIP should point just past the ELSE */
1433 brw_inst_set_jip(brw
, if_inst
, br
* (else_inst
- if_inst
+ 1));
1434 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1435 brw_inst_set_uip(brw
, if_inst
, br
* (endif_inst
- if_inst
));
1436 brw_inst_set_jip(brw
, else_inst
, br
* (endif_inst
- else_inst
));
1437 if (brw
->gen
>= 8) {
1438 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1439 * should point to ENDIF.
1441 brw_inst_set_uip(brw
, else_inst
, br
* (endif_inst
- else_inst
));
1448 brw_ELSE(struct brw_compile
*p
)
1450 struct brw_context
*brw
= p
->brw
;
1453 insn
= next_insn(p
, BRW_OPCODE_ELSE
);
1456 brw_set_dest(p
, insn
, brw_ip_reg());
1457 brw_set_src0(p
, insn
, brw_ip_reg());
1458 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1459 } else if (brw
->gen
== 6) {
1460 brw_set_dest(p
, insn
, brw_imm_w(0));
1461 brw_inst_set_gen6_jump_count(brw
, insn
, 0);
1462 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1463 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1465 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1466 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1467 brw_set_src1(p
, insn
, brw_imm_ud(0));
1468 brw_inst_set_jip(brw
, insn
, 0);
1469 brw_inst_set_uip(brw
, insn
, 0);
1472 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1473 brw_inst_set_mask_control(brw
, insn
, BRW_MASK_ENABLE
);
1474 if (!p
->single_program_flow
&& brw
->gen
< 6)
1475 brw_inst_set_thread_control(brw
, insn
, BRW_THREAD_SWITCH
);
1477 push_if_stack(p
, insn
);
1481 brw_ENDIF(struct brw_compile
*p
)
1483 struct brw_context
*brw
= p
->brw
;
1484 brw_inst
*insn
= NULL
;
1485 brw_inst
*else_inst
= NULL
;
1486 brw_inst
*if_inst
= NULL
;
1488 bool emit_endif
= true;
1490 /* In single program flow mode, we can express IF and ELSE instructions
1491 * equivalently as ADD instructions that operate on IP. On platforms prior
1492 * to Gen6, flow control instructions cause an implied thread switch, so
1493 * this is a significant savings.
1495 * However, on Gen6, writing to IP doesn't work in single program flow mode
1496 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1497 * not be updated by non-flow control instructions."). And on later
1498 * platforms, there is no significant benefit to converting control flow
1499 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1502 if (brw
->gen
< 6 && p
->single_program_flow
)
1506 * A single next_insn() may change the base adress of instruction store
1507 * memory(p->store), so call it first before referencing the instruction
1508 * store pointer from an index
1511 insn
= next_insn(p
, BRW_OPCODE_ENDIF
);
1513 /* Pop the IF and (optional) ELSE instructions from the stack */
1514 p
->if_depth_in_loop
[p
->loop_stack_depth
]--;
1515 tmp
= pop_if_stack(p
);
1516 if (brw_inst_opcode(brw
, tmp
) == BRW_OPCODE_ELSE
) {
1518 tmp
= pop_if_stack(p
);
1523 /* ENDIF is useless; don't bother emitting it. */
1524 convert_IF_ELSE_to_ADD(p
, if_inst
, else_inst
);
1529 brw_set_dest(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1530 brw_set_src0(p
, insn
, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD
));
1531 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1532 } else if (brw
->gen
== 6) {
1533 brw_set_dest(p
, insn
, brw_imm_w(0));
1534 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1535 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1537 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1538 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1539 brw_set_src1(p
, insn
, brw_imm_ud(0));
1542 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1543 brw_inst_set_mask_control(brw
, insn
, BRW_MASK_ENABLE
);
1545 brw_inst_set_thread_control(brw
, insn
, BRW_THREAD_SWITCH
);
1547 /* Also pop item off the stack in the endif instruction: */
1549 brw_inst_set_gen4_jump_count(brw
, insn
, 0);
1550 brw_inst_set_gen4_pop_count(brw
, insn
, 1);
1551 } else if (brw
->gen
== 6) {
1552 brw_inst_set_gen6_jump_count(brw
, insn
, 2);
1554 brw_inst_set_jip(brw
, insn
, 2);
1556 patch_IF_ELSE(p
, if_inst
, else_inst
, insn
);
1560 brw_BREAK(struct brw_compile
*p
)
1562 struct brw_context
*brw
= p
->brw
;
1565 insn
= next_insn(p
, BRW_OPCODE_BREAK
);
1566 if (brw
->gen
>= 6) {
1567 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1568 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1569 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1571 brw_set_dest(p
, insn
, brw_ip_reg());
1572 brw_set_src0(p
, insn
, brw_ip_reg());
1573 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1574 brw_inst_set_gen4_pop_count(brw
, insn
,
1575 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1577 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1578 brw_inst_set_exec_size(brw
, insn
, p
->compressed
? BRW_EXECUTE_16
1585 brw_CONT(struct brw_compile
*p
)
1587 const struct brw_context
*brw
= p
->brw
;
1590 insn
= next_insn(p
, BRW_OPCODE_CONTINUE
);
1591 brw_set_dest(p
, insn
, brw_ip_reg());
1592 brw_set_src0(p
, insn
, brw_ip_reg());
1593 brw_set_src1(p
, insn
, brw_imm_d(0x0));
1596 brw_inst_set_gen4_pop_count(brw
, insn
,
1597 p
->if_depth_in_loop
[p
->loop_stack_depth
]);
1599 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1600 brw_inst_set_exec_size(brw
, insn
, p
->compressed
? BRW_EXECUTE_16
1606 gen6_HALT(struct brw_compile
*p
)
1608 const struct brw_context
*brw
= p
->brw
;
1611 insn
= next_insn(p
, BRW_OPCODE_HALT
);
1612 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1613 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1614 brw_set_src1(p
, insn
, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1616 if (p
->compressed
) {
1617 brw_inst_set_exec_size(brw
, insn
, BRW_EXECUTE_16
);
1619 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1620 brw_inst_set_exec_size(brw
, insn
, BRW_EXECUTE_8
);
1627 * The DO/WHILE is just an unterminated loop -- break or continue are
1628 * used for control within the loop. We have a few ways they can be
1631 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1632 * jip and no DO instruction.
1634 * For non-uniform control flow pre-gen6, there's a DO instruction to
1635 * push the mask, and a WHILE to jump back, and BREAK to get out and
1638 * For gen6, there's no more mask stack, so no need for DO. WHILE
1639 * just points back to the first instruction of the loop.
1642 brw_DO(struct brw_compile
*p
, unsigned execute_size
)
1644 struct brw_context
*brw
= p
->brw
;
1646 if (brw
->gen
>= 6 || p
->single_program_flow
) {
1647 push_loop_stack(p
, &p
->store
[p
->nr_insn
]);
1648 return &p
->store
[p
->nr_insn
];
1650 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_DO
);
1652 push_loop_stack(p
, insn
);
1654 /* Override the defaults for this instruction:
1656 brw_set_dest(p
, insn
, brw_null_reg());
1657 brw_set_src0(p
, insn
, brw_null_reg());
1658 brw_set_src1(p
, insn
, brw_null_reg());
1660 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1661 brw_inst_set_exec_size(brw
, insn
, execute_size
);
1662 brw_inst_set_pred_control(brw
, insn
, BRW_PREDICATE_NONE
);
1669 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1672 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1673 * nesting, since it can always just point to the end of the block/current loop.
1676 brw_patch_break_cont(struct brw_compile
*p
, brw_inst
*while_inst
)
1678 struct brw_context
*brw
= p
->brw
;
1679 brw_inst
*do_inst
= get_inner_do_insn(p
);
1681 unsigned br
= brw_jump_scale(brw
);
1683 assert(brw
->gen
< 6);
1685 for (inst
= while_inst
- 1; inst
!= do_inst
; inst
--) {
1686 /* If the jump count is != 0, that means that this instruction has already
1687 * been patched because it's part of a loop inside of the one we're
1690 if (brw_inst_opcode(brw
, inst
) == BRW_OPCODE_BREAK
&&
1691 brw_inst_gen4_jump_count(brw
, inst
) == 0) {
1692 brw_inst_set_gen4_jump_count(brw
, inst
, br
*((while_inst
- inst
) + 1));
1693 } else if (brw_inst_opcode(brw
, inst
) == BRW_OPCODE_CONTINUE
&&
1694 brw_inst_gen4_jump_count(brw
, inst
) == 0) {
1695 brw_inst_set_gen4_jump_count(brw
, inst
, br
* (while_inst
- inst
));
1701 brw_WHILE(struct brw_compile
*p
)
1703 struct brw_context
*brw
= p
->brw
;
1704 brw_inst
*insn
, *do_insn
;
1705 unsigned br
= brw_jump_scale(brw
);
1707 if (brw
->gen
>= 7) {
1708 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1709 do_insn
= get_inner_do_insn(p
);
1711 brw_set_dest(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1712 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1713 brw_set_src1(p
, insn
, brw_imm_ud(0));
1714 brw_inst_set_jip(brw
, insn
, br
* (do_insn
- insn
));
1716 brw_inst_set_exec_size(brw
, insn
, p
->compressed
? BRW_EXECUTE_16
1718 } else if (brw
->gen
== 6) {
1719 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1720 do_insn
= get_inner_do_insn(p
);
1722 brw_set_dest(p
, insn
, brw_imm_w(0));
1723 brw_inst_set_gen6_jump_count(brw
, insn
, br
* (do_insn
- insn
));
1724 brw_set_src0(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1725 brw_set_src1(p
, insn
, retype(brw_null_reg(), BRW_REGISTER_TYPE_D
));
1727 brw_inst_set_exec_size(brw
, insn
, p
->compressed
? BRW_EXECUTE_16
1730 if (p
->single_program_flow
) {
1731 insn
= next_insn(p
, BRW_OPCODE_ADD
);
1732 do_insn
= get_inner_do_insn(p
);
1734 brw_set_dest(p
, insn
, brw_ip_reg());
1735 brw_set_src0(p
, insn
, brw_ip_reg());
1736 brw_set_src1(p
, insn
, brw_imm_d((do_insn
- insn
) * 16));
1737 brw_inst_set_exec_size(brw
, insn
, BRW_EXECUTE_1
);
1739 insn
= next_insn(p
, BRW_OPCODE_WHILE
);
1740 do_insn
= get_inner_do_insn(p
);
1742 assert(brw_inst_opcode(brw
, do_insn
) == BRW_OPCODE_DO
);
1744 brw_set_dest(p
, insn
, brw_ip_reg());
1745 brw_set_src0(p
, insn
, brw_ip_reg());
1746 brw_set_src1(p
, insn
, brw_imm_d(0));
1748 brw_inst_set_exec_size(brw
, insn
, brw_inst_exec_size(brw
, do_insn
));
1749 brw_inst_set_gen4_jump_count(brw
, insn
, br
* (do_insn
- insn
+ 1));
1750 brw_inst_set_gen4_pop_count(brw
, insn
, 0);
1752 brw_patch_break_cont(p
, insn
);
1755 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1757 p
->loop_stack_depth
--;
1764 void brw_land_fwd_jump(struct brw_compile
*p
, int jmp_insn_idx
)
1766 struct brw_context
*brw
= p
->brw
;
1767 brw_inst
*jmp_insn
= &p
->store
[jmp_insn_idx
];
1773 assert(brw_inst_opcode(brw
, jmp_insn
) == BRW_OPCODE_JMPI
);
1774 assert(brw_inst_src1_reg_file(brw
, jmp_insn
) == BRW_IMMEDIATE_VALUE
);
1776 brw_inst_set_gen4_jump_count(brw
, jmp_insn
,
1777 jmpi
* (p
->nr_insn
- jmp_insn_idx
- 1));
1780 /* To integrate with the above, it makes sense that the comparison
1781 * instruction should populate the flag register. It might be simpler
1782 * just to use the flag reg for most WM tasks?
1784 void brw_CMP(struct brw_compile
*p
,
1785 struct brw_reg dest
,
1786 unsigned conditional
,
1787 struct brw_reg src0
,
1788 struct brw_reg src1
)
1790 struct brw_context
*brw
= p
->brw
;
1791 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_CMP
);
1793 if (brw
->gen
>= 8) {
1794 /* The CMP instruction appears to behave erratically for floating point
1795 * sources unless the destination type is also float. Overriding it to
1796 * match src0 makes it work in all cases.
1798 dest
.type
= src0
.type
;
1801 brw_inst_set_cond_modifier(brw
, insn
, conditional
);
1802 brw_set_dest(p
, insn
, dest
);
1803 brw_set_src0(p
, insn
, src0
);
1804 brw_set_src1(p
, insn
, src1
);
1806 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1808 * "Any CMP instruction with a null destination must use a {switch}."
1810 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1811 * mentioned on their work-arounds pages.
1813 if (brw
->gen
== 7) {
1814 if (dest
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1815 dest
.nr
== BRW_ARF_NULL
) {
1816 brw_inst_set_thread_control(brw
, insn
, BRW_THREAD_SWITCH
);
1821 /***********************************************************************
1822 * Helpers for the various SEND message types:
1825 /** Extended math function, float[8].
1827 void gen4_math(struct brw_compile
*p
,
1828 struct brw_reg dest
,
1830 unsigned msg_reg_nr
,
1833 unsigned precision
)
1835 struct brw_context
*brw
= p
->brw
;
1836 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1838 assert(brw
->gen
< 6);
1840 /* Example code doesn't set predicate_control for send
1843 brw_inst_set_pred_control(brw
, insn
, 0);
1844 brw_inst_set_base_mrf(brw
, insn
, msg_reg_nr
);
1846 brw_set_dest(p
, insn
, dest
);
1847 brw_set_src0(p
, insn
, src
);
1848 brw_set_math_message(p
,
1851 src
.type
== BRW_REGISTER_TYPE_D
,
1856 void gen6_math(struct brw_compile
*p
,
1857 struct brw_reg dest
,
1859 struct brw_reg src0
,
1860 struct brw_reg src1
)
1862 struct brw_context
*brw
= p
->brw
;
1863 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_MATH
);
1865 assert(brw
->gen
>= 6);
1867 assert(dest
.file
== BRW_GENERAL_REGISTER_FILE
||
1868 (brw
->gen
>= 7 && dest
.file
== BRW_MESSAGE_REGISTER_FILE
));
1869 assert(src0
.file
== BRW_GENERAL_REGISTER_FILE
||
1870 (brw
->gen
>= 8 && src0
.file
== BRW_IMMEDIATE_VALUE
));
1872 assert(dest
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1873 if (brw
->gen
== 6) {
1874 assert(src0
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1875 assert(src1
.hstride
== BRW_HORIZONTAL_STRIDE_1
);
1878 if (function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT
||
1879 function
== BRW_MATH_FUNCTION_INT_DIV_REMAINDER
||
1880 function
== BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER
) {
1881 assert(src0
.type
!= BRW_REGISTER_TYPE_F
);
1882 assert(src1
.type
!= BRW_REGISTER_TYPE_F
);
1883 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1884 (brw
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1886 assert(src0
.type
== BRW_REGISTER_TYPE_F
);
1887 assert(src1
.type
== BRW_REGISTER_TYPE_F
);
1888 if (function
== BRW_MATH_FUNCTION_POW
) {
1889 assert(src1
.file
== BRW_GENERAL_REGISTER_FILE
||
1890 (brw
->gen
>= 8 && src1
.file
== BRW_IMMEDIATE_VALUE
));
1892 assert(src1
.file
== BRW_ARCHITECTURE_REGISTER_FILE
&&
1893 src1
.nr
== BRW_ARF_NULL
);
1897 /* Source modifiers are ignored for extended math instructions on Gen6. */
1898 if (brw
->gen
== 6) {
1899 assert(!src0
.negate
);
1901 assert(!src1
.negate
);
1905 brw_inst_set_math_function(brw
, insn
, function
);
1907 brw_set_dest(p
, insn
, dest
);
1908 brw_set_src0(p
, insn
, src0
);
1909 brw_set_src1(p
, insn
, src1
);
1914 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1915 * using a constant offset per channel.
1917 * The offset must be aligned to oword size (16 bytes). Used for
1918 * register spilling.
1920 void brw_oword_block_write_scratch(struct brw_compile
*p
,
1925 struct brw_context
*brw
= p
->brw
;
1926 uint32_t msg_control
, msg_type
;
1932 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
1934 if (num_regs
== 1) {
1935 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
1938 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
1942 /* Set up the message header. This is g0, with g0.2 filled with
1943 * the offset. We don't want to leave our offset around in g0 or
1944 * it'll screw up texture samples, so set it up inside the message
1948 brw_push_insn_state(p
);
1949 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
1950 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
1952 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
1954 /* set message header global offset field (reg 0, element 2) */
1956 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
1958 2), BRW_REGISTER_TYPE_UD
),
1959 brw_imm_ud(offset
));
1961 brw_pop_insn_state(p
);
1965 struct brw_reg dest
;
1966 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
1967 int send_commit_msg
;
1968 struct brw_reg src_header
= retype(brw_vec8_grf(0, 0),
1969 BRW_REGISTER_TYPE_UW
);
1971 if (brw_inst_qtr_control(brw
, insn
) != BRW_COMPRESSION_NONE
) {
1972 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
1973 src_header
= vec16(src_header
);
1975 assert(brw_inst_pred_control(brw
, insn
) == BRW_PREDICATE_NONE
);
1977 brw_inst_set_base_mrf(brw
, insn
, mrf
.nr
);
1979 /* Until gen6, writes followed by reads from the same location
1980 * are not guaranteed to be ordered unless write_commit is set.
1981 * If set, then a no-op write is issued to the destination
1982 * register to set a dependency, and a read from the destination
1983 * can be used to ensure the ordering.
1985 * For gen6, only writes between different threads need ordering
1986 * protection. Our use of DP writes is all about register
1987 * spilling within a thread.
1989 if (brw
->gen
>= 6) {
1990 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
1991 send_commit_msg
= 0;
1994 send_commit_msg
= 1;
1997 brw_set_dest(p
, insn
, dest
);
1998 if (brw
->gen
>= 6) {
1999 brw_set_src0(p
, insn
, mrf
);
2001 brw_set_src0(p
, insn
, brw_null_reg());
2005 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2007 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE
;
2009 brw_set_dp_write_message(p
,
2011 255, /* binding table index (255=stateless) */
2015 true, /* header_present */
2016 0, /* not a render target */
2017 send_commit_msg
, /* response_length */
2025 * Read a block of owords (half a GRF each) from the scratch buffer
2026 * using a constant index per channel.
2028 * Offset must be aligned to oword size (16 bytes). Used for register
2032 brw_oword_block_read_scratch(struct brw_compile
*p
,
2033 struct brw_reg dest
,
2038 struct brw_context
*brw
= p
->brw
;
2039 uint32_t msg_control
;
2045 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2046 dest
= retype(dest
, BRW_REGISTER_TYPE_UW
);
2048 if (num_regs
== 1) {
2049 msg_control
= BRW_DATAPORT_OWORD_BLOCK_2_OWORDS
;
2052 msg_control
= BRW_DATAPORT_OWORD_BLOCK_4_OWORDS
;
2057 brw_push_insn_state(p
);
2058 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2059 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2061 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2063 /* set message header global offset field (reg 0, element 2) */
2065 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2067 2), BRW_REGISTER_TYPE_UD
),
2068 brw_imm_ud(offset
));
2070 brw_pop_insn_state(p
);
2074 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2076 assert(brw_inst_pred_control(brw
, insn
) == 0);
2077 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
2079 brw_set_dest(p
, insn
, dest
); /* UW? */
2080 if (brw
->gen
>= 6) {
2081 brw_set_src0(p
, insn
, mrf
);
2083 brw_set_src0(p
, insn
, brw_null_reg());
2084 brw_inst_set_base_mrf(brw
, insn
, mrf
.nr
);
2087 brw_set_dp_read_message(p
,
2089 255, /* binding table index (255=stateless) */
2091 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
, /* msg_type */
2092 BRW_DATAPORT_READ_TARGET_RENDER_CACHE
,
2094 true, /* header_present */
2100 gen7_block_read_scratch(struct brw_compile
*p
,
2101 struct brw_reg dest
,
2105 const struct brw_context
*brw
= p
->brw
;
2106 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2107 assert(brw_inst_pred_control(brw
, insn
) == BRW_PREDICATE_NONE
);
2109 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
2110 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UW
));
2112 /* The HW requires that the header is present; this is to get the g0.5
2115 brw_set_src0(p
, insn
, brw_vec8_grf(0, 0));
2117 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2118 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2119 * is 32 bytes, which happens to be the size of a register.
2122 assert(offset
< (1 << 12));
2124 gen7_set_dp_scratch_message(p
, insn
,
2125 false, /* scratch read */
2127 false, /* invalidate after read */
2130 1, /* mlen: just g0 */
2131 num_regs
, /* rlen */
2132 true); /* header present */
2136 * Read a float[4] vector from the data port Data Cache (const buffer).
2137 * Location (in buffer) should be a multiple of 16.
2138 * Used for fetching shader constants.
2140 void brw_oword_block_read(struct brw_compile
*p
,
2141 struct brw_reg dest
,
2144 uint32_t bind_table_index
)
2146 struct brw_context
*brw
= p
->brw
;
2148 /* On newer hardware, offset is in units of owords. */
2152 mrf
= retype(mrf
, BRW_REGISTER_TYPE_UD
);
2154 brw_push_insn_state(p
);
2155 brw_set_default_predicate_control(p
, BRW_PREDICATE_NONE
);
2156 brw_set_default_compression_control(p
, BRW_COMPRESSION_NONE
);
2157 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2159 brw_MOV(p
, mrf
, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD
));
2161 /* set message header global offset field (reg 0, element 2) */
2163 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE
,
2165 2), BRW_REGISTER_TYPE_UD
),
2166 brw_imm_ud(offset
));
2168 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2170 /* cast dest to a uword[8] vector */
2171 dest
= retype(vec8(dest
), BRW_REGISTER_TYPE_UW
);
2173 brw_set_dest(p
, insn
, dest
);
2174 if (brw
->gen
>= 6) {
2175 brw_set_src0(p
, insn
, mrf
);
2177 brw_set_src0(p
, insn
, brw_null_reg());
2178 brw_inst_set_base_mrf(brw
, insn
, mrf
.nr
);
2181 brw_set_dp_read_message(p
,
2184 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW
,
2185 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ
,
2186 BRW_DATAPORT_READ_TARGET_DATA_CACHE
,
2188 true, /* header_present */
2189 1); /* response_length (1 reg, 2 owords!) */
2191 brw_pop_insn_state(p
);
2195 void brw_fb_WRITE(struct brw_compile
*p
,
2197 unsigned msg_reg_nr
,
2198 struct brw_reg src0
,
2199 unsigned msg_control
,
2200 unsigned binding_table_index
,
2201 unsigned msg_length
,
2202 unsigned response_length
,
2204 bool header_present
)
2206 struct brw_context
*brw
= p
->brw
;
2209 struct brw_reg dest
;
2211 if (dispatch_width
== 16)
2212 dest
= retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2214 dest
= retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW
);
2216 if (brw
->gen
>= 6) {
2217 insn
= next_insn(p
, BRW_OPCODE_SENDC
);
2219 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2221 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
2223 if (brw
->gen
>= 6) {
2224 /* headerless version, just submit color payload */
2225 src0
= brw_message_reg(msg_reg_nr
);
2227 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2229 brw_inst_set_base_mrf(brw
, insn
, msg_reg_nr
);
2231 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE
;
2234 brw_set_dest(p
, insn
, dest
);
2235 brw_set_src0(p
, insn
, src0
);
2236 brw_set_dp_write_message(p
,
2238 binding_table_index
,
2243 eot
, /* last render target write */
2246 0 /* send_commit_msg */);
2251 * Texture sample instruction.
2252 * Note: the msg_type plus msg_length values determine exactly what kind
2253 * of sampling operation is performed. See volume 4, page 161 of docs.
2255 void brw_SAMPLE(struct brw_compile
*p
,
2256 struct brw_reg dest
,
2257 unsigned msg_reg_nr
,
2258 struct brw_reg src0
,
2259 unsigned binding_table_index
,
2262 unsigned response_length
,
2263 unsigned msg_length
,
2264 unsigned header_present
,
2266 unsigned return_format
)
2268 struct brw_context
*brw
= p
->brw
;
2271 if (msg_reg_nr
!= -1)
2272 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2274 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2275 brw_inst_set_pred_control(brw
, insn
, BRW_PREDICATE_NONE
); /* XXX */
2277 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2279 * "Instruction compression is not allowed for this instruction (that
2280 * is, send). The hardware behavior is undefined if this instruction is
2281 * set as compressed. However, compress control can be set to "SecHalf"
2282 * to affect the EMask generation."
2284 * No similar wording is found in later PRMs, but there are examples
2285 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2286 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2287 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2289 if (brw_inst_qtr_control(brw
, insn
) != BRW_COMPRESSION_2NDHALF
)
2290 brw_inst_set_qtr_control(brw
, insn
, BRW_COMPRESSION_NONE
);
2293 brw_inst_set_base_mrf(brw
, insn
, msg_reg_nr
);
2295 brw_set_dest(p
, insn
, dest
);
2296 brw_set_src0(p
, insn
, src0
);
2297 brw_set_sampler_message(p
, insn
,
2298 binding_table_index
,
2308 /* All these variables are pretty confusing - we might be better off
2309 * using bitmasks and macros for this, in the old style. Or perhaps
2310 * just having the caller instantiate the fields in dword3 itself.
2312 void brw_urb_WRITE(struct brw_compile
*p
,
2313 struct brw_reg dest
,
2314 unsigned msg_reg_nr
,
2315 struct brw_reg src0
,
2316 enum brw_urb_write_flags flags
,
2317 unsigned msg_length
,
2318 unsigned response_length
,
2322 struct brw_context
*brw
= p
->brw
;
2325 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2327 if (brw
->gen
>= 7 && !(flags
& BRW_URB_WRITE_USE_CHANNEL_MASKS
)) {
2328 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2329 brw_push_insn_state(p
);
2330 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2331 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2332 brw_MOV(p
, get_element_ud(brw_message_reg(msg_reg_nr
), 5),
2333 brw_imm_ud(0xff00));
2334 brw_pop_insn_state(p
);
2337 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2339 assert(msg_length
< BRW_MAX_MRF
);
2341 brw_set_dest(p
, insn
, dest
);
2342 brw_set_src0(p
, insn
, src0
);
2343 brw_set_src1(p
, insn
, brw_imm_d(0));
2346 brw_inst_set_base_mrf(brw
, insn
, msg_reg_nr
);
2348 brw_set_urb_message(p
,
2358 brw_find_next_block_end(struct brw_compile
*p
, int start_offset
)
2361 void *store
= p
->store
;
2362 const struct brw_context
*brw
= p
->brw
;
2364 for (offset
= next_offset(brw
, store
, start_offset
);
2365 offset
< p
->next_insn_offset
;
2366 offset
= next_offset(brw
, store
, offset
)) {
2367 brw_inst
*insn
= store
+ offset
;
2369 switch (brw_inst_opcode(brw
, insn
)) {
2370 case BRW_OPCODE_ENDIF
:
2371 case BRW_OPCODE_ELSE
:
2372 case BRW_OPCODE_WHILE
:
2373 case BRW_OPCODE_HALT
:
2381 /* There is no DO instruction on gen6, so to find the end of the loop
2382 * we have to see if the loop is jumping back before our start
2386 brw_find_loop_end(struct brw_compile
*p
, int start_offset
)
2388 struct brw_context
*brw
= p
->brw
;
2390 int scale
= 16 / brw_jump_scale(brw
);
2391 void *store
= p
->store
;
2393 assert(brw
->gen
>= 6);
2395 /* Always start after the instruction (such as a WHILE) we're trying to fix
2398 for (offset
= next_offset(brw
, store
, start_offset
);
2399 offset
< p
->next_insn_offset
;
2400 offset
= next_offset(brw
, store
, offset
)) {
2401 brw_inst
*insn
= store
+ offset
;
2403 if (brw_inst_opcode(brw
, insn
) == BRW_OPCODE_WHILE
) {
2404 int jip
= brw
->gen
== 6 ? brw_inst_gen6_jump_count(brw
, insn
)
2405 : brw_inst_jip(brw
, insn
);
2406 if (offset
+ jip
* scale
<= start_offset
)
2410 assert(!"not reached");
2411 return start_offset
;
2414 /* After program generation, go back and update the UIP and JIP of
2415 * BREAK, CONT, and HALT instructions to their correct locations.
2418 brw_set_uip_jip(struct brw_compile
*p
)
2420 struct brw_context
*brw
= p
->brw
;
2422 int br
= brw_jump_scale(brw
);
2423 int scale
= 16 / br
;
2424 void *store
= p
->store
;
2429 for (offset
= 0; offset
< p
->next_insn_offset
;
2430 offset
= next_offset(brw
, store
, offset
)) {
2431 brw_inst
*insn
= store
+ offset
;
2433 if (brw_inst_cmpt_control(brw
, insn
)) {
2434 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2435 assert(brw_inst_opcode(brw
, insn
) != BRW_OPCODE_BREAK
&&
2436 brw_inst_opcode(brw
, insn
) != BRW_OPCODE_CONTINUE
&&
2437 brw_inst_opcode(brw
, insn
) != BRW_OPCODE_HALT
);
2441 int block_end_offset
= brw_find_next_block_end(p
, offset
);
2442 switch (brw_inst_opcode(brw
, insn
)) {
2443 case BRW_OPCODE_BREAK
:
2444 assert(block_end_offset
!= 0);
2445 brw_inst_set_jip(brw
, insn
, (block_end_offset
- offset
) / scale
);
2446 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2447 brw_inst_set_uip(brw
, insn
,
2448 (brw_find_loop_end(p
, offset
) - offset
+
2449 (brw
->gen
== 6 ? 16 : 0)) / scale
);
2451 case BRW_OPCODE_CONTINUE
:
2452 assert(block_end_offset
!= 0);
2453 brw_inst_set_jip(brw
, insn
, (block_end_offset
- offset
) / scale
);
2454 brw_inst_set_uip(brw
, insn
,
2455 (brw_find_loop_end(p
, offset
) - offset
) / scale
);
2457 assert(brw_inst_uip(brw
, insn
) != 0);
2458 assert(brw_inst_jip(brw
, insn
) != 0);
2461 case BRW_OPCODE_ENDIF
:
2462 if (block_end_offset
== 0)
2463 brw_inst_set_jip(brw
, insn
, 1 * br
);
2465 brw_inst_set_jip(brw
, insn
, (block_end_offset
- offset
) / scale
);
2468 case BRW_OPCODE_HALT
:
2469 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2471 * "In case of the halt instruction not inside any conditional
2472 * code block, the value of <JIP> and <UIP> should be the
2473 * same. In case of the halt instruction inside conditional code
2474 * block, the <UIP> should be the end of the program, and the
2475 * <JIP> should be end of the most inner conditional code block."
2477 * The uip will have already been set by whoever set up the
2480 if (block_end_offset
== 0) {
2481 brw_inst_set_jip(brw
, insn
, brw_inst_uip(brw
, insn
));
2483 brw_inst_set_jip(brw
, insn
, (block_end_offset
- offset
) / scale
);
2485 assert(brw_inst_uip(brw
, insn
) != 0);
2486 assert(brw_inst_jip(brw
, insn
) != 0);
2492 void brw_ff_sync(struct brw_compile
*p
,
2493 struct brw_reg dest
,
2494 unsigned msg_reg_nr
,
2495 struct brw_reg src0
,
2497 unsigned response_length
,
2500 struct brw_context
*brw
= p
->brw
;
2503 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2505 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2506 brw_set_dest(p
, insn
, dest
);
2507 brw_set_src0(p
, insn
, src0
);
2508 brw_set_src1(p
, insn
, brw_imm_d(0));
2511 brw_inst_set_base_mrf(brw
, insn
, msg_reg_nr
);
2513 brw_set_ff_sync_message(p
,
2521 * Emit the SEND instruction necessary to generate stream output data on Gen6
2522 * (for transform feedback).
2524 * If send_commit_msg is true, this is the last piece of stream output data
2525 * from this thread, so send the data as a committed write. According to the
2526 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2528 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2529 * writes are complete by sending the final write as a committed write."
2532 brw_svb_write(struct brw_compile
*p
,
2533 struct brw_reg dest
,
2534 unsigned msg_reg_nr
,
2535 struct brw_reg src0
,
2536 unsigned binding_table_index
,
2537 bool send_commit_msg
)
2541 gen6_resolve_implied_move(p
, &src0
, msg_reg_nr
);
2543 insn
= next_insn(p
, BRW_OPCODE_SEND
);
2544 brw_set_dest(p
, insn
, dest
);
2545 brw_set_src0(p
, insn
, src0
);
2546 brw_set_src1(p
, insn
, brw_imm_d(0));
2547 brw_set_dp_write_message(p
, insn
,
2548 binding_table_index
,
2549 0, /* msg_control: ignored */
2550 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE
,
2552 true, /* header_present */
2553 0, /* last_render_target: ignored */
2554 send_commit_msg
, /* response_length */
2555 0, /* end_of_thread */
2556 send_commit_msg
); /* send_commit_msg */
2560 brw_set_dp_untyped_atomic_message(struct brw_compile
*p
,
2563 unsigned bind_table_index
,
2564 unsigned msg_length
,
2565 unsigned response_length
,
2566 bool header_present
)
2568 const struct brw_context
*brw
= p
->brw
;
2570 unsigned msg_control
=
2571 atomic_op
| /* Atomic Operation Type: BRW_AOP_* */
2572 (response_length
? 1 << 5 : 0); /* Return data expected */
2574 if (brw
->gen
>= 8 || brw
->is_haswell
) {
2575 brw_set_message_descriptor(p
, insn
, HSW_SFID_DATAPORT_DATA_CACHE_1
,
2576 msg_length
, response_length
,
2577 header_present
, false);
2580 if (brw_inst_access_mode(brw
, insn
) == BRW_ALIGN_1
) {
2581 if (brw_inst_exec_size(brw
, insn
) != BRW_EXECUTE_16
)
2582 msg_control
|= 1 << 4; /* SIMD8 mode */
2584 brw_inst_set_dp_msg_type(brw
, insn
,
2585 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
);
2587 brw_inst_set_dp_msg_type(brw
, insn
,
2588 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
);
2591 brw_set_message_descriptor(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
2592 msg_length
, response_length
,
2593 header_present
, false);
2595 brw_inst_set_dp_msg_type(brw
, insn
, GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
);
2597 if (brw_inst_exec_size(brw
, insn
) != BRW_EXECUTE_16
)
2598 msg_control
|= 1 << 4; /* SIMD8 mode */
2601 brw_inst_set_binding_table_index(brw
, insn
, bind_table_index
);
2602 brw_inst_set_dp_msg_control(brw
, insn
, msg_control
);
2606 brw_untyped_atomic(struct brw_compile
*p
,
2607 struct brw_reg dest
,
2610 unsigned bind_table_index
,
2611 unsigned msg_length
,
2612 unsigned response_length
) {
2613 const struct brw_context
*brw
= p
->brw
;
2614 brw_inst
*insn
= brw_next_insn(p
, BRW_OPCODE_SEND
);
2616 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UD
));
2617 brw_set_src0(p
, insn
, retype(mrf
, BRW_REGISTER_TYPE_UD
));
2618 brw_set_src1(p
, insn
, brw_imm_d(0));
2619 brw_set_dp_untyped_atomic_message(
2620 p
, insn
, atomic_op
, bind_table_index
, msg_length
, response_length
,
2621 brw_inst_access_mode(brw
, insn
) == BRW_ALIGN_1
);
2625 brw_set_dp_untyped_surface_read_message(struct brw_compile
*p
,
2627 unsigned bind_table_index
,
2628 unsigned msg_length
,
2629 unsigned response_length
,
2630 bool header_present
)
2632 const struct brw_context
*brw
= p
->brw
;
2633 const unsigned dispatch_width
=
2634 (brw_inst_exec_size(brw
, insn
) == BRW_EXECUTE_16
? 16 : 8);
2635 const unsigned num_channels
= response_length
/ (dispatch_width
/ 8);
2637 if (brw
->gen
>= 8 || brw
->is_haswell
) {
2638 brw_set_message_descriptor(p
, insn
, HSW_SFID_DATAPORT_DATA_CACHE_1
,
2639 msg_length
, response_length
,
2640 header_present
, false);
2642 brw_inst_set_dp_msg_type(brw
, insn
,
2643 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
);
2645 brw_set_message_descriptor(p
, insn
, GEN7_SFID_DATAPORT_DATA_CACHE
,
2646 msg_length
, response_length
,
2647 header_present
, false);
2649 brw_inst_set_dp_msg_type(brw
, insn
,
2650 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
);
2653 /* Set mask of 32-bit channels to drop. */
2654 unsigned msg_control
= (0xf & (0xf << num_channels
));
2656 if (brw_inst_access_mode(brw
, insn
) == BRW_ALIGN_1
) {
2657 if (dispatch_width
== 16)
2658 msg_control
|= 1 << 4; /* SIMD16 mode */
2660 msg_control
|= 2 << 4; /* SIMD8 mode */
2663 brw_inst_set_binding_table_index(brw
, insn
, bind_table_index
);
2664 brw_inst_set_dp_msg_control(brw
, insn
, msg_control
);
2668 brw_untyped_surface_read(struct brw_compile
*p
,
2669 struct brw_reg dest
,
2671 unsigned bind_table_index
,
2672 unsigned msg_length
,
2673 unsigned response_length
)
2675 const struct brw_context
*brw
= p
->brw
;
2676 brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2678 brw_set_dest(p
, insn
, retype(dest
, BRW_REGISTER_TYPE_UD
));
2679 brw_set_src0(p
, insn
, retype(mrf
, BRW_REGISTER_TYPE_UD
));
2680 brw_set_dp_untyped_surface_read_message(
2681 p
, insn
, bind_table_index
, msg_length
, response_length
,
2682 brw_inst_access_mode(brw
, insn
) == BRW_ALIGN_1
);
2686 brw_pixel_interpolator_query(struct brw_compile
*p
,
2687 struct brw_reg dest
,
2692 unsigned msg_length
,
2693 unsigned response_length
)
2695 const struct brw_context
*brw
= p
->brw
;
2696 struct brw_inst
*insn
= next_insn(p
, BRW_OPCODE_SEND
);
2698 brw_set_dest(p
, insn
, dest
);
2699 brw_set_src0(p
, insn
, mrf
);
2700 brw_set_message_descriptor(p
, insn
, GEN7_SFID_PIXEL_INTERPOLATOR
,
2701 msg_length
, response_length
,
2702 false /* header is never present for PI */,
2705 brw_inst_set_pi_simd_mode(
2706 brw
, insn
, brw_inst_exec_size(brw
, insn
) == BRW_EXECUTE_16
);
2707 brw_inst_set_pi_slot_group(brw
, insn
, 0); /* zero unless 32/64px dispatch */
2708 brw_inst_set_pi_nopersp(brw
, insn
, noperspective
);
2709 brw_inst_set_pi_message_type(brw
, insn
, mode
);
2710 brw_inst_set_pi_message_data(brw
, insn
, data
);
2714 * This instruction is generated as a single-channel align1 instruction by
2715 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
2717 * We can't use the typed atomic op in the FS because that has the execution
2718 * mask ANDed with the pixel mask, but we just want to write the one dword for
2721 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
2722 * one u32. So we use the same untyped atomic write message as the pixel
2725 * The untyped atomic operation requires a BUFFER surface type with RAW
2726 * format, and is only accessible through the legacy DATA_CACHE dataport
2729 void brw_shader_time_add(struct brw_compile
*p
,
2730 struct brw_reg payload
,
2731 uint32_t surf_index
)
2733 assert(p
->brw
->gen
>= 7);
2735 brw_push_insn_state(p
);
2736 brw_set_default_access_mode(p
, BRW_ALIGN_1
);
2737 brw_set_default_mask_control(p
, BRW_MASK_DISABLE
);
2738 brw_inst
*send
= brw_next_insn(p
, BRW_OPCODE_SEND
);
2739 brw_pop_insn_state(p
);
2741 /* We use brw_vec1_reg and unmasked because we want to increment the given
2744 brw_set_dest(p
, send
, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE
,
2746 brw_set_src0(p
, send
, brw_vec1_reg(payload
.file
,
2748 brw_set_dp_untyped_atomic_message(p
, send
, BRW_AOP_ADD
, surf_index
,
2749 2 /* message length */,
2750 0 /* response length */,
2751 false /* header present */);