projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
i965/vec4: Simplify opt_reduce_swizzle() using the swizzle utils.
[mesa.git] / src / mesa / drivers / dri / i965 / brw_eu_emit.c
1 /*
2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
5
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
13
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
17
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
26 **********************************************************************/
27 /*
28 * Authors:
29 * Keith Whitwell <keithw@vmware.com>
30 */
31
32
33 #include "brw_context.h"
34 #include "brw_defines.h"
35 #include "brw_eu.h"
36
37 #include "util/ralloc.h"
38
39 /***********************************************************************
40 * Internal helper for constructing instructions
41 */
42
43 static void guess_execution_size(struct brw_compile *p,
44 brw_inst *insn,
45 struct brw_reg reg)
46 {
47 const struct brw_context *brw = p->brw;
48
49 if (reg.width == BRW_WIDTH_8 && p->compressed) {
50 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_16);
51 } else {
52 /* Register width definitions are compatible with BRW_EXECUTE_* enums. */
53 brw_inst_set_exec_size(brw, insn, reg.width);
54 }
55 }
56
57
58 /**
59 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
60 * registers, implicitly moving the operand to a message register.
61 *
62 * On Sandybridge, this is no longer the case. This function performs the
63 * explicit move; it should be called before emitting a SEND instruction.
64 */
65 void
66 gen6_resolve_implied_move(struct brw_compile *p,
67 struct brw_reg *src,
68 unsigned msg_reg_nr)
69 {
70 struct brw_context *brw = p->brw;
71 if (brw->gen < 6)
72 return;
73
74 if (src->file == BRW_MESSAGE_REGISTER_FILE)
75 return;
76
77 if (src->file != BRW_ARCHITECTURE_REGISTER_FILE || src->nr != BRW_ARF_NULL) {
78 brw_push_insn_state(p);
79 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
80 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
81 brw_MOV(p, retype(brw_message_reg(msg_reg_nr), BRW_REGISTER_TYPE_UD),
82 retype(*src, BRW_REGISTER_TYPE_UD));
83 brw_pop_insn_state(p);
84 }
85 *src = brw_message_reg(msg_reg_nr);
86 }
87
88 static void
89 gen7_convert_mrf_to_grf(struct brw_compile *p, struct brw_reg *reg)
90 {
91 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
92 * "The send with EOT should use register space R112-R127 for <src>. This is
93 * to enable loading of a new thread into the same slot while the message
94 * with EOT for current thread is pending dispatch."
95 *
96 * Since we're pretending to have 16 MRFs anyway, we may as well use the
97 * registers required for messages with EOT.
98 */
99 struct brw_context *brw = p->brw;
100 if (brw->gen >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) {
101 reg->file = BRW_GENERAL_REGISTER_FILE;
102 reg->nr += GEN7_MRF_HACK_START;
103 }
104 }
105
106 /**
107 * Convert a brw_reg_type enumeration value into the hardware representation.
108 *
109 * The hardware encoding may depend on whether the value is an immediate.
110 */
111 unsigned
112 brw_reg_type_to_hw_type(const struct brw_context *brw,
113 enum brw_reg_type type, unsigned file)
114 {
115 if (file == BRW_IMMEDIATE_VALUE) {
116 const static int imm_hw_types[] = {
117 [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
118 [BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D,
119 [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
120 [BRW_REGISTER_TYPE_W] = BRW_HW_REG_TYPE_W,
121 [BRW_REGISTER_TYPE_F] = BRW_HW_REG_TYPE_F,
122 [BRW_REGISTER_TYPE_UB] = -1,
123 [BRW_REGISTER_TYPE_B] = -1,
124 [BRW_REGISTER_TYPE_UV] = BRW_HW_REG_IMM_TYPE_UV,
125 [BRW_REGISTER_TYPE_VF] = BRW_HW_REG_IMM_TYPE_VF,
126 [BRW_REGISTER_TYPE_V] = BRW_HW_REG_IMM_TYPE_V,
127 [BRW_REGISTER_TYPE_DF] = GEN8_HW_REG_IMM_TYPE_DF,
128 [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_IMM_TYPE_HF,
129 [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ,
130 [BRW_REGISTER_TYPE_Q] = GEN8_HW_REG_TYPE_Q,
131 };
132 assert(type < ARRAY_SIZE(imm_hw_types));
133 assert(imm_hw_types[type] != -1);
134 assert(brw->gen >= 8 || type < BRW_REGISTER_TYPE_DF);
135 return imm_hw_types[type];
136 } else {
137 /* Non-immediate registers */
138 const static int hw_types[] = {
139 [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
140 [BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D,
141 [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
142 [BRW_REGISTER_TYPE_W] = BRW_HW_REG_TYPE_W,
143 [BRW_REGISTER_TYPE_UB] = BRW_HW_REG_NON_IMM_TYPE_UB,
144 [BRW_REGISTER_TYPE_B] = BRW_HW_REG_NON_IMM_TYPE_B,
145 [BRW_REGISTER_TYPE_F] = BRW_HW_REG_TYPE_F,
146 [BRW_REGISTER_TYPE_UV] = -1,
147 [BRW_REGISTER_TYPE_VF] = -1,
148 [BRW_REGISTER_TYPE_V] = -1,
149 [BRW_REGISTER_TYPE_DF] = GEN7_HW_REG_NON_IMM_TYPE_DF,
150 [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_NON_IMM_TYPE_HF,
151 [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ,
152 [BRW_REGISTER_TYPE_Q] = GEN8_HW_REG_TYPE_Q,
153 };
154 assert(type < ARRAY_SIZE(hw_types));
155 assert(hw_types[type] != -1);
156 assert(brw->gen >= 7 || type < BRW_REGISTER_TYPE_DF);
157 assert(brw->gen >= 8 || type < BRW_REGISTER_TYPE_HF);
158 return hw_types[type];
159 }
160 }
161
162 void
163 brw_set_dest(struct brw_compile *p, brw_inst *inst, struct brw_reg dest)
164 {
165 const struct brw_context *brw = p->brw;
166
167 if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE &&
168 dest.file != BRW_MESSAGE_REGISTER_FILE)
169 assert(dest.nr < 128);
170
171 gen7_convert_mrf_to_grf(p, &dest);
172
173 brw_inst_set_dst_reg_file(brw, inst, dest.file);
174 brw_inst_set_dst_reg_type(brw, inst, brw_reg_type_to_hw_type(brw, dest.type,
175 dest.file));
176 brw_inst_set_dst_address_mode(brw, inst, dest.address_mode);
177
178 if (dest.address_mode == BRW_ADDRESS_DIRECT) {
179 brw_inst_set_dst_da_reg_nr(brw, inst, dest.nr);
180
181 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
182 brw_inst_set_dst_da1_subreg_nr(brw, inst, dest.subnr);
183 if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
184 dest.hstride = BRW_HORIZONTAL_STRIDE_1;
185 brw_inst_set_dst_hstride(brw, inst, dest.hstride);
186 } else {
187 brw_inst_set_dst_da16_subreg_nr(brw, inst, dest.subnr / 16);
188 brw_inst_set_da16_writemask(brw, inst, dest.dw1.bits.writemask);
189 if (dest.file == BRW_GENERAL_REGISTER_FILE ||
190 dest.file == BRW_MESSAGE_REGISTER_FILE) {
191 assert(dest.dw1.bits.writemask != 0);
192 }
193 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
194 * Although Dst.HorzStride is a don't care for Align16, HW needs
195 * this to be programmed as "01".
196 */
197 brw_inst_set_dst_hstride(brw, inst, 1);
198 }
199 } else {
200 brw_inst_set_dst_ia_subreg_nr(brw, inst, dest.subnr);
201
202 /* These are different sizes in align1 vs align16:
203 */
204 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
205 brw_inst_set_dst_ia1_addr_imm(brw, inst,
206 dest.dw1.bits.indirect_offset);
207 if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
208 dest.hstride = BRW_HORIZONTAL_STRIDE_1;
209 brw_inst_set_dst_hstride(brw, inst, dest.hstride);
210 } else {
211 brw_inst_set_dst_ia16_addr_imm(brw, inst,
212 dest.dw1.bits.indirect_offset);
213 /* even ignored in da16, still need to set as '01' */
214 brw_inst_set_dst_hstride(brw, inst, 1);
215 }
216 }
217
218 /* NEW: Set the execution size based on dest.width and
219 * inst->compression_control:
220 */
221 guess_execution_size(p, inst, dest);
222 }
223
224 extern int reg_type_size[];
225
226 static void
227 validate_reg(const struct brw_context *brw, brw_inst *inst, struct brw_reg reg)
228 {
229 const int hstride_for_reg[] = {0, 1, 2, 4};
230 const int vstride_for_reg[] = {0, 1, 2, 4, 8, 16, 32};
231 const int width_for_reg[] = {1, 2, 4, 8, 16};
232 const int execsize_for_reg[] = {1, 2, 4, 8, 16};
233 int width, hstride, vstride, execsize;
234
235 if (reg.file == BRW_IMMEDIATE_VALUE) {
236 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
237 * mean the destination has to be 128-bit aligned and the
238 * destination horiz stride has to be a word.
239 */
240 if (reg.type == BRW_REGISTER_TYPE_V) {
241 assert(hstride_for_reg[brw_inst_dst_hstride(brw, inst)] *
242 reg_type_size[brw_inst_dst_reg_type(brw, inst)] == 2);
243 }
244
245 return;
246 }
247
248 if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE &&
249 reg.file == BRW_ARF_NULL)
250 return;
251
252 assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg));
253 hstride = hstride_for_reg[reg.hstride];
254
255 if (reg.vstride == 0xf) {
256 vstride = -1;
257 } else {
258 assert(reg.vstride >= 0 && reg.vstride < ARRAY_SIZE(vstride_for_reg));
259 vstride = vstride_for_reg[reg.vstride];
260 }
261
262 assert(reg.width >= 0 && reg.width < ARRAY_SIZE(width_for_reg));
263 width = width_for_reg[reg.width];
264
265 assert(brw_inst_exec_size(brw, inst) >= 0 &&
266 brw_inst_exec_size(brw, inst) < ARRAY_SIZE(execsize_for_reg));
267 execsize = execsize_for_reg[brw_inst_exec_size(brw, inst)];
268
269 /* Restrictions from 3.3.10: Register Region Restrictions. */
270 /* 3. */
271 assert(execsize >= width);
272
273 /* 4. */
274 if (execsize == width && hstride != 0) {
275 assert(vstride == -1 || vstride == width * hstride);
276 }
277
278 /* 5. */
279 if (execsize == width && hstride == 0) {
280 /* no restriction on vstride. */
281 }
282
283 /* 6. */
284 if (width == 1) {
285 assert(hstride == 0);
286 }
287
288 /* 7. */
289 if (execsize == 1 && width == 1) {
290 assert(hstride == 0);
291 assert(vstride == 0);
292 }
293
294 /* 8. */
295 if (vstride == 0 && hstride == 0) {
296 assert(width == 1);
297 }
298
299 /* 10. Check destination issues. */
300 }
301
302 static bool
303 is_compactable_immediate(unsigned imm)
304 {
305 /* We get the low 12 bits as-is. */
306 imm &= ~0xfff;
307
308 /* We get one bit replicated through the top 20 bits. */
309 return imm == 0 || imm == 0xfffff000;
310 }
311
312 void
313 brw_set_src0(struct brw_compile *p, brw_inst *inst, struct brw_reg reg)
314 {
315 struct brw_context *brw = p->brw;
316
317 if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
318 assert(reg.nr < 128);
319
320 gen7_convert_mrf_to_grf(p, &reg);
321
322 if (brw->gen >= 6 && (brw_inst_opcode(brw, inst) == BRW_OPCODE_SEND ||
323 brw_inst_opcode(brw, inst) == BRW_OPCODE_SENDC)) {
324 /* Any source modifiers or regions will be ignored, since this just
325 * identifies the MRF/GRF to start reading the message contents from.
326 * Check for some likely failures.
327 */
328 assert(!reg.negate);
329 assert(!reg.abs);
330 assert(reg.address_mode == BRW_ADDRESS_DIRECT);
331 }
332
333 validate_reg(brw, inst, reg);
334
335 brw_inst_set_src0_reg_file(brw, inst, reg.file);
336 brw_inst_set_src0_reg_type(brw, inst,
337 brw_reg_type_to_hw_type(brw, reg.type, reg.file));
338 brw_inst_set_src0_abs(brw, inst, reg.abs);
339 brw_inst_set_src0_negate(brw, inst, reg.negate);
340 brw_inst_set_src0_address_mode(brw, inst, reg.address_mode);
341
342 if (reg.file == BRW_IMMEDIATE_VALUE) {
343 brw_inst_set_imm_ud(brw, inst, reg.dw1.ud);
344
345 /* The Bspec's section titled "Non-present Operands" claims that if src0
346 * is an immediate that src1's type must be the same as that of src0.
347 *
348 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
349 * that do not follow this rule. E.g., from the IVB/HSW table:
350 *
351 * DataTypeIndex 18-Bit Mapping Mapped Meaning
352 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
353 *
354 * And from the SNB table:
355 *
356 * DataTypeIndex 18-Bit Mapping Mapped Meaning
357 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
358 *
359 * Neither of these cause warnings from the simulator when used,
360 * compacted or otherwise. In fact, all compaction mappings that have an
361 * immediate in src0 use a:ud for src1.
362 *
363 * The GM45 instruction compaction tables do not contain mapped meanings
364 * so it's not clear whether it has the restriction. We'll assume it was
365 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
366 */
367 brw_inst_set_src1_reg_file(brw, inst, BRW_ARCHITECTURE_REGISTER_FILE);
368 if (brw->gen < 6) {
369 brw_inst_set_src1_reg_type(brw, inst,
370 brw_inst_src0_reg_type(brw, inst));
371 } else {
372 brw_inst_set_src1_reg_type(brw, inst, BRW_HW_REG_TYPE_UD);
373 }
374
375 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
376 * for immediate values. Presumably the hardware engineers realized
377 * that the only useful floating-point value that could be represented
378 * in this format is 0.0, which can also be represented as a VF-typed
379 * immediate, so they gave us the previously mentioned mapping on IVB+.
380 *
381 * Strangely, we do have a mapping for imm:f in src1, so we don't need
382 * to do this there.
383 *
384 * If we see a 0.0:F, change the type to VF so that it can be compacted.
385 */
386 if (brw_inst_imm_ud(brw, inst) == 0x0 &&
387 brw_inst_src0_reg_type(brw, inst) == BRW_HW_REG_TYPE_F) {
388 brw_inst_set_src0_reg_type(brw, inst, BRW_HW_REG_IMM_TYPE_VF);
389 }
390
391 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
392 * set the types to :UD so the instruction can be compacted.
393 */
394 if (is_compactable_immediate(brw_inst_imm_ud(brw, inst)) &&
395 brw_inst_cond_modifier(brw, inst) == BRW_CONDITIONAL_NONE &&
396 brw_inst_src0_reg_type(brw, inst) == BRW_HW_REG_TYPE_D &&
397 brw_inst_dst_reg_type(brw, inst) == BRW_HW_REG_TYPE_D) {
398 brw_inst_set_src0_reg_type(brw, inst, BRW_HW_REG_TYPE_UD);
399 brw_inst_set_dst_reg_type(brw, inst, BRW_HW_REG_TYPE_UD);
400 }
401 } else {
402 if (reg.address_mode == BRW_ADDRESS_DIRECT) {
403 brw_inst_set_src0_da_reg_nr(brw, inst, reg.nr);
404 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
405 brw_inst_set_src0_da1_subreg_nr(brw, inst, reg.subnr);
406 } else {
407 brw_inst_set_src0_da16_subreg_nr(brw, inst, reg.subnr / 16);
408 }
409 } else {
410 brw_inst_set_src0_ia_subreg_nr(brw, inst, reg.subnr);
411
412 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
413 brw_inst_set_src0_ia1_addr_imm(brw, inst, reg.dw1.bits.indirect_offset);
414 } else {
415 brw_inst_set_src0_ia_subreg_nr(brw, inst, reg.dw1.bits.indirect_offset);
416 }
417 }
418
419 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
420 if (reg.width == BRW_WIDTH_1 &&
421 brw_inst_exec_size(brw, inst) == BRW_EXECUTE_1) {
422 brw_inst_set_src0_hstride(brw, inst, BRW_HORIZONTAL_STRIDE_0);
423 brw_inst_set_src0_width(brw, inst, BRW_WIDTH_1);
424 brw_inst_set_src0_vstride(brw, inst, BRW_VERTICAL_STRIDE_0);
425 } else {
426 brw_inst_set_src0_hstride(brw, inst, reg.hstride);
427 brw_inst_set_src0_width(brw, inst, reg.width);
428 brw_inst_set_src0_vstride(brw, inst, reg.vstride);
429 }
430 } else {
431 brw_inst_set_src0_da16_swiz_x(brw, inst,
432 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X));
433 brw_inst_set_src0_da16_swiz_y(brw, inst,
434 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y));
435 brw_inst_set_src0_da16_swiz_z(brw, inst,
436 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z));
437 brw_inst_set_src0_da16_swiz_w(brw, inst,
438 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W));
439
440 /* This is an oddity of the fact we're using the same
441 * descriptions for registers in align_16 as align_1:
442 */
443 if (reg.vstride == BRW_VERTICAL_STRIDE_8)
444 brw_inst_set_src0_vstride(brw, inst, BRW_VERTICAL_STRIDE_4);
445 else
446 brw_inst_set_src0_vstride(brw, inst, reg.vstride);
447 }
448 }
449 }
450
451
452 void
453 brw_set_src1(struct brw_compile *p, brw_inst *inst, struct brw_reg reg)
454 {
455 const struct brw_context *brw = p->brw;
456
457 if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
458 assert(reg.nr < 128);
459
460 gen7_convert_mrf_to_grf(p, &reg);
461 assert(reg.file != BRW_MESSAGE_REGISTER_FILE);
462
463 validate_reg(brw, inst, reg);
464
465 brw_inst_set_src1_reg_file(brw, inst, reg.file);
466 brw_inst_set_src1_reg_type(brw, inst,
467 brw_reg_type_to_hw_type(brw, reg.type, reg.file));
468 brw_inst_set_src1_abs(brw, inst, reg.abs);
469 brw_inst_set_src1_negate(brw, inst, reg.negate);
470
471 /* Only src1 can be immediate in two-argument instructions.
472 */
473 assert(brw_inst_src0_reg_file(brw, inst) != BRW_IMMEDIATE_VALUE);
474
475 if (reg.file == BRW_IMMEDIATE_VALUE) {
476 brw_inst_set_imm_ud(brw, inst, reg.dw1.ud);
477 } else {
478 /* This is a hardware restriction, which may or may not be lifted
479 * in the future:
480 */
481 assert (reg.address_mode == BRW_ADDRESS_DIRECT);
482 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
483
484 brw_inst_set_src1_da_reg_nr(brw, inst, reg.nr);
485 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
486 brw_inst_set_src1_da1_subreg_nr(brw, inst, reg.subnr);
487 } else {
488 brw_inst_set_src1_da16_subreg_nr(brw, inst, reg.subnr / 16);
489 }
490
491 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
492 if (reg.width == BRW_WIDTH_1 &&
493 brw_inst_exec_size(brw, inst) == BRW_EXECUTE_1) {
494 brw_inst_set_src1_hstride(brw, inst, BRW_HORIZONTAL_STRIDE_0);
495 brw_inst_set_src1_width(brw, inst, BRW_WIDTH_1);
496 brw_inst_set_src1_vstride(brw, inst, BRW_VERTICAL_STRIDE_0);
497 } else {
498 brw_inst_set_src1_hstride(brw, inst, reg.hstride);
499 brw_inst_set_src1_width(brw, inst, reg.width);
500 brw_inst_set_src1_vstride(brw, inst, reg.vstride);
501 }
502 } else {
503 brw_inst_set_src1_da16_swiz_x(brw, inst,
504 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X));
505 brw_inst_set_src1_da16_swiz_y(brw, inst,
506 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y));
507 brw_inst_set_src1_da16_swiz_z(brw, inst,
508 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z));
509 brw_inst_set_src1_da16_swiz_w(brw, inst,
510 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W));
511
512 /* This is an oddity of the fact we're using the same
513 * descriptions for registers in align_16 as align_1:
514 */
515 if (reg.vstride == BRW_VERTICAL_STRIDE_8)
516 brw_inst_set_src1_vstride(brw, inst, BRW_VERTICAL_STRIDE_4);
517 else
518 brw_inst_set_src1_vstride(brw, inst, reg.vstride);
519 }
520 }
521 }
522
523 /**
524 * Set the Message Descriptor and Extended Message Descriptor fields
525 * for SEND messages.
526 *
527 * \note This zeroes out the Function Control bits, so it must be called
528 * \b before filling out any message-specific data. Callers can
529 * choose not to fill in irrelevant bits; they will be zero.
530 */
531 static void
532 brw_set_message_descriptor(struct brw_compile *p,
533 brw_inst *inst,
534 enum brw_message_target sfid,
535 unsigned msg_length,
536 unsigned response_length,
537 bool header_present,
538 bool end_of_thread)
539 {
540 struct brw_context *brw = p->brw;
541
542 brw_set_src1(p, inst, brw_imm_d(0));
543
544 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
545 * itself; instead, it will be a MOV/OR into the address register.
546 *
547 * In this case, we avoid setting the extended message descriptor bits,
548 * since they go on the later SEND/SENDC instead and if set here would
549 * instead clobber the conditionalmod bits.
550 */
551 unsigned opcode = brw_inst_opcode(brw, inst);
552 if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC) {
553 brw_inst_set_sfid(brw, inst, sfid);
554 }
555
556 brw_inst_set_mlen(brw, inst, msg_length);
557 brw_inst_set_rlen(brw, inst, response_length);
558 brw_inst_set_eot(brw, inst, end_of_thread);
559
560 if (brw->gen >= 5) {
561 brw_inst_set_header_present(brw, inst, header_present);
562 }
563 }
564
565 static void brw_set_math_message( struct brw_compile *p,
566 brw_inst *inst,
567 unsigned function,
568 unsigned integer_type,
569 bool low_precision,
570 unsigned dataType )
571 {
572 struct brw_context *brw = p->brw;
573 unsigned msg_length;
574 unsigned response_length;
575
576 /* Infer message length from the function */
577 switch (function) {
578 case BRW_MATH_FUNCTION_POW:
579 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT:
580 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER:
581 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
582 msg_length = 2;
583 break;
584 default:
585 msg_length = 1;
586 break;
587 }
588
589 /* Infer response length from the function */
590 switch (function) {
591 case BRW_MATH_FUNCTION_SINCOS:
592 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
593 response_length = 2;
594 break;
595 default:
596 response_length = 1;
597 break;
598 }
599
600
601 brw_set_message_descriptor(p, inst, BRW_SFID_MATH,
602 msg_length, response_length, false, false);
603 brw_inst_set_math_msg_function(brw, inst, function);
604 brw_inst_set_math_msg_signed_int(brw, inst, integer_type);
605 brw_inst_set_math_msg_precision(brw, inst, low_precision);
606 brw_inst_set_math_msg_saturate(brw, inst, brw_inst_saturate(brw, inst));
607 brw_inst_set_math_msg_data_type(brw, inst, dataType);
608 brw_inst_set_saturate(brw, inst, 0);
609 }
610
611
612 static void brw_set_ff_sync_message(struct brw_compile *p,
613 brw_inst *insn,
614 bool allocate,
615 unsigned response_length,
616 bool end_of_thread)
617 {
618 const struct brw_context *brw = p->brw;
619
620 brw_set_message_descriptor(p, insn, BRW_SFID_URB,
621 1, response_length, true, end_of_thread);
622 brw_inst_set_urb_opcode(brw, insn, 1); /* FF_SYNC */
623 brw_inst_set_urb_allocate(brw, insn, allocate);
624 /* The following fields are not used by FF_SYNC: */
625 brw_inst_set_urb_global_offset(brw, insn, 0);
626 brw_inst_set_urb_swizzle_control(brw, insn, 0);
627 brw_inst_set_urb_used(brw, insn, 0);
628 brw_inst_set_urb_complete(brw, insn, 0);
629 }
630
631 static void brw_set_urb_message( struct brw_compile *p,
632 brw_inst *insn,
633 enum brw_urb_write_flags flags,
634 unsigned msg_length,
635 unsigned response_length,
636 unsigned offset,
637 unsigned swizzle_control )
638 {
639 struct brw_context *brw = p->brw;
640
641 assert(brw->gen < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE);
642 assert(brw->gen < 7 || !(flags & BRW_URB_WRITE_ALLOCATE));
643 assert(brw->gen >= 7 || !(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
644
645 brw_set_message_descriptor(p, insn, BRW_SFID_URB,
646 msg_length, response_length, true,
647 flags & BRW_URB_WRITE_EOT);
648
649 if (flags & BRW_URB_WRITE_OWORD) {
650 assert(msg_length == 2); /* header + one OWORD of data */
651 brw_inst_set_urb_opcode(brw, insn, BRW_URB_OPCODE_WRITE_OWORD);
652 } else {
653 brw_inst_set_urb_opcode(brw, insn, BRW_URB_OPCODE_WRITE_HWORD);
654 }
655
656 brw_inst_set_urb_global_offset(brw, insn, offset);
657 brw_inst_set_urb_swizzle_control(brw, insn, swizzle_control);
658
659 if (brw->gen < 8) {
660 brw_inst_set_urb_complete(brw, insn, !!(flags & BRW_URB_WRITE_COMPLETE));
661 }
662
663 if (brw->gen < 7) {
664 brw_inst_set_urb_allocate(brw, insn, !!(flags & BRW_URB_WRITE_ALLOCATE));
665 brw_inst_set_urb_used(brw, insn, !(flags & BRW_URB_WRITE_UNUSED));
666 } else {
667 brw_inst_set_urb_per_slot_offset(brw, insn,
668 !!(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
669 }
670 }
671
672 void
673 brw_set_dp_write_message(struct brw_compile *p,
674 brw_inst *insn,
675 unsigned binding_table_index,
676 unsigned msg_control,
677 unsigned msg_type,
678 unsigned msg_length,
679 bool header_present,
680 unsigned last_render_target,
681 unsigned response_length,
682 unsigned end_of_thread,
683 unsigned send_commit_msg)
684 {
685 struct brw_context *brw = p->brw;
686 unsigned sfid;
687
688 if (brw->gen >= 7) {
689 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
690 if (msg_type == GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE)
691 sfid = GEN6_SFID_DATAPORT_RENDER_CACHE;
692 else
693 sfid = GEN7_SFID_DATAPORT_DATA_CACHE;
694 } else if (brw->gen == 6) {
695 /* Use the render cache for all write messages. */
696 sfid = GEN6_SFID_DATAPORT_RENDER_CACHE;
697 } else {
698 sfid = BRW_SFID_DATAPORT_WRITE;
699 }
700
701 brw_set_message_descriptor(p, insn, sfid, msg_length, response_length,
702 header_present, end_of_thread);
703
704 brw_inst_set_binding_table_index(brw, insn, binding_table_index);
705 brw_inst_set_dp_write_msg_type(brw, insn, msg_type);
706 brw_inst_set_dp_write_msg_control(brw, insn, msg_control);
707 brw_inst_set_rt_last(brw, insn, last_render_target);
708 if (brw->gen < 7) {
709 brw_inst_set_dp_write_commit(brw, insn, send_commit_msg);
710 }
711 }
712
713 void
714 brw_set_dp_read_message(struct brw_compile *p,
715 brw_inst *insn,
716 unsigned binding_table_index,
717 unsigned msg_control,
718 unsigned msg_type,
719 unsigned target_cache,
720 unsigned msg_length,
721 bool header_present,
722 unsigned response_length)
723 {
724 struct brw_context *brw = p->brw;
725 unsigned sfid;
726
727 if (brw->gen >= 7) {
728 sfid = GEN7_SFID_DATAPORT_DATA_CACHE;
729 } else if (brw->gen == 6) {
730 if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE)
731 sfid = GEN6_SFID_DATAPORT_RENDER_CACHE;
732 else
733 sfid = GEN6_SFID_DATAPORT_SAMPLER_CACHE;
734 } else {
735 sfid = BRW_SFID_DATAPORT_READ;
736 }
737
738 brw_set_message_descriptor(p, insn, sfid, msg_length, response_length,
739 header_present, false);
740
741 brw_inst_set_binding_table_index(brw, insn, binding_table_index);
742 brw_inst_set_dp_read_msg_type(brw, insn, msg_type);
743 brw_inst_set_dp_read_msg_control(brw, insn, msg_control);
744 if (brw->gen < 6)
745 brw_inst_set_dp_read_target_cache(brw, insn, target_cache);
746 }
747
748 void
749 brw_set_sampler_message(struct brw_compile *p,
750 brw_inst *inst,
751 unsigned binding_table_index,
752 unsigned sampler,
753 unsigned msg_type,
754 unsigned response_length,
755 unsigned msg_length,
756 unsigned header_present,
757 unsigned simd_mode,
758 unsigned return_format)
759 {
760 struct brw_context *brw = p->brw;
761
762 brw_set_message_descriptor(p, inst, BRW_SFID_SAMPLER, msg_length,
763 response_length, header_present, false);
764
765 brw_inst_set_binding_table_index(brw, inst, binding_table_index);
766 brw_inst_set_sampler(brw, inst, sampler);
767 brw_inst_set_sampler_msg_type(brw, inst, msg_type);
768 if (brw->gen >= 5) {
769 brw_inst_set_sampler_simd_mode(brw, inst, simd_mode);
770 } else if (brw->gen == 4 && !brw->is_g4x) {
771 brw_inst_set_sampler_return_format(brw, inst, return_format);
772 }
773 }
774
775 static void
776 gen7_set_dp_scratch_message(struct brw_compile *p,
777 brw_inst *inst,
778 bool write,
779 bool dword,
780 bool invalidate_after_read,
781 unsigned num_regs,
782 unsigned addr_offset,
783 unsigned mlen,
784 unsigned rlen,
785 bool header_present)
786 {
787 const struct brw_context *brw = p->brw;
788 assert(num_regs == 1 || num_regs == 2 || num_regs == 4 ||
789 (brw->gen >= 8 && num_regs == 8));
790 brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE,
791 mlen, rlen, header_present, false);
792 brw_inst_set_dp_category(brw, inst, 1); /* Scratch Block Read/Write msgs */
793 brw_inst_set_scratch_read_write(brw, inst, write);
794 brw_inst_set_scratch_type(brw, inst, dword);
795 brw_inst_set_scratch_invalidate_after_read(brw, inst, invalidate_after_read);
796 brw_inst_set_scratch_block_size(brw, inst, ffs(num_regs) - 1);
797 brw_inst_set_scratch_addr_offset(brw, inst, addr_offset);
798 }
799
800 #define next_insn brw_next_insn
801 brw_inst *
802 brw_next_insn(struct brw_compile *p, unsigned opcode)
803 {
804 const struct brw_context *brw = p->brw;
805 brw_inst *insn;
806
807 if (p->nr_insn + 1 > p->store_size) {
808 p->store_size <<= 1;
809 p->store = reralloc(p->mem_ctx, p->store, brw_inst, p->store_size);
810 }
811
812 p->next_insn_offset += 16;
813 insn = &p->store[p->nr_insn++];
814 memcpy(insn, p->current, sizeof(*insn));
815
816 brw_inst_set_opcode(brw, insn, opcode);
817 return insn;
818 }
819
820 static brw_inst *
821 brw_alu1(struct brw_compile *p, unsigned opcode,
822 struct brw_reg dest, struct brw_reg src)
823 {
824 brw_inst *insn = next_insn(p, opcode);
825 brw_set_dest(p, insn, dest);
826 brw_set_src0(p, insn, src);
827 return insn;
828 }
829
830 static brw_inst *
831 brw_alu2(struct brw_compile *p, unsigned opcode,
832 struct brw_reg dest, struct brw_reg src0, struct brw_reg src1)
833 {
834 brw_inst *insn = next_insn(p, opcode);
835 brw_set_dest(p, insn, dest);
836 brw_set_src0(p, insn, src0);
837 brw_set_src1(p, insn, src1);
838 return insn;
839 }
840
841 static int
842 get_3src_subreg_nr(struct brw_reg reg)
843 {
844 if (reg.vstride == BRW_VERTICAL_STRIDE_0) {
845 assert(brw_is_single_value_swizzle(reg.dw1.bits.swizzle));
846 return reg.subnr / 4 + BRW_GET_SWZ(reg.dw1.bits.swizzle, 0);
847 } else {
848 return reg.subnr / 4;
849 }
850 }
851
852 static brw_inst *
853 brw_alu3(struct brw_compile *p, unsigned opcode, struct brw_reg dest,
854 struct brw_reg src0, struct brw_reg src1, struct brw_reg src2)
855 {
856 struct brw_context *brw = p->brw;
857 brw_inst *inst = next_insn(p, opcode);
858
859 gen7_convert_mrf_to_grf(p, &dest);
860
861 assert(brw_inst_access_mode(brw, inst) == BRW_ALIGN_16);
862
863 assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
864 dest.file == BRW_MESSAGE_REGISTER_FILE);
865 assert(dest.nr < 128);
866 assert(dest.address_mode == BRW_ADDRESS_DIRECT);
867 assert(dest.type == BRW_REGISTER_TYPE_F ||
868 dest.type == BRW_REGISTER_TYPE_D ||
869 dest.type == BRW_REGISTER_TYPE_UD);
870 if (brw->gen == 6) {
871 brw_inst_set_3src_dst_reg_file(brw, inst,
872 dest.file == BRW_MESSAGE_REGISTER_FILE);
873 }
874 brw_inst_set_3src_dst_reg_nr(brw, inst, dest.nr);
875 brw_inst_set_3src_dst_subreg_nr(brw, inst, dest.subnr / 16);
876 brw_inst_set_3src_dst_writemask(brw, inst, dest.dw1.bits.writemask);
877 guess_execution_size(p, inst, dest);
878
879 assert(src0.file == BRW_GENERAL_REGISTER_FILE);
880 assert(src0.address_mode == BRW_ADDRESS_DIRECT);
881 assert(src0.nr < 128);
882 brw_inst_set_3src_src0_swizzle(brw, inst, src0.dw1.bits.swizzle);
883 brw_inst_set_3src_src0_subreg_nr(brw, inst, get_3src_subreg_nr(src0));
884 brw_inst_set_3src_src0_reg_nr(brw, inst, src0.nr);
885 brw_inst_set_3src_src0_abs(brw, inst, src0.abs);
886 brw_inst_set_3src_src0_negate(brw, inst, src0.negate);
887 brw_inst_set_3src_src0_rep_ctrl(brw, inst,
888 src0.vstride == BRW_VERTICAL_STRIDE_0);
889
890 assert(src1.file == BRW_GENERAL_REGISTER_FILE);
891 assert(src1.address_mode == BRW_ADDRESS_DIRECT);
892 assert(src1.nr < 128);
893 brw_inst_set_3src_src1_swizzle(brw, inst, src1.dw1.bits.swizzle);
894 brw_inst_set_3src_src1_subreg_nr(brw, inst, get_3src_subreg_nr(src1));
895 brw_inst_set_3src_src1_reg_nr(brw, inst, src1.nr);
896 brw_inst_set_3src_src1_abs(brw, inst, src1.abs);
897 brw_inst_set_3src_src1_negate(brw, inst, src1.negate);
898 brw_inst_set_3src_src1_rep_ctrl(brw, inst,
899 src1.vstride == BRW_VERTICAL_STRIDE_0);
900
901 assert(src2.file == BRW_GENERAL_REGISTER_FILE);
902 assert(src2.address_mode == BRW_ADDRESS_DIRECT);
903 assert(src2.nr < 128);
904 brw_inst_set_3src_src2_swizzle(brw, inst, src2.dw1.bits.swizzle);
905 brw_inst_set_3src_src2_subreg_nr(brw, inst, get_3src_subreg_nr(src2));
906 brw_inst_set_3src_src2_reg_nr(brw, inst, src2.nr);
907 brw_inst_set_3src_src2_abs(brw, inst, src2.abs);
908 brw_inst_set_3src_src2_negate(brw, inst, src2.negate);
909 brw_inst_set_3src_src2_rep_ctrl(brw, inst,
910 src2.vstride == BRW_VERTICAL_STRIDE_0);
911
912 if (brw->gen >= 7) {
913 /* Set both the source and destination types based on dest.type,
914 * ignoring the source register types. The MAD and LRP emitters ensure
915 * that all four types are float. The BFE and BFI2 emitters, however,
916 * may send us mixed D and UD types and want us to ignore that and use
917 * the destination type.
918 */
919 switch (dest.type) {
920 case BRW_REGISTER_TYPE_F:
921 brw_inst_set_3src_src_type(brw, inst, BRW_3SRC_TYPE_F);
922 brw_inst_set_3src_dst_type(brw, inst, BRW_3SRC_TYPE_F);
923 break;
924 case BRW_REGISTER_TYPE_D:
925 brw_inst_set_3src_src_type(brw, inst, BRW_3SRC_TYPE_D);
926 brw_inst_set_3src_dst_type(brw, inst, BRW_3SRC_TYPE_D);
927 break;
928 case BRW_REGISTER_TYPE_UD:
929 brw_inst_set_3src_src_type(brw, inst, BRW_3SRC_TYPE_UD);
930 brw_inst_set_3src_dst_type(brw, inst, BRW_3SRC_TYPE_UD);
931 break;
932 }
933 }
934
935 return inst;
936 }
937
938
939 /***********************************************************************
940 * Convenience routines.
941 */
942 #define ALU1(OP) \
943 brw_inst *brw_##OP(struct brw_compile *p, \
944 struct brw_reg dest, \
945 struct brw_reg src0) \
946 { \
947 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
948 }
949
950 #define ALU2(OP) \
951 brw_inst *brw_##OP(struct brw_compile *p, \
952 struct brw_reg dest, \
953 struct brw_reg src0, \
954 struct brw_reg src1) \
955 { \
956 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
957 }
958
959 #define ALU3(OP) \
960 brw_inst *brw_##OP(struct brw_compile *p, \
961 struct brw_reg dest, \
962 struct brw_reg src0, \
963 struct brw_reg src1, \
964 struct brw_reg src2) \
965 { \
966 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
967 }
968
969 #define ALU3F(OP) \
970 brw_inst *brw_##OP(struct brw_compile *p, \
971 struct brw_reg dest, \
972 struct brw_reg src0, \
973 struct brw_reg src1, \
974 struct brw_reg src2) \
975 { \
976 assert(dest.type == BRW_REGISTER_TYPE_F); \
977 assert(src0.type == BRW_REGISTER_TYPE_F); \
978 assert(src1.type == BRW_REGISTER_TYPE_F); \
979 assert(src2.type == BRW_REGISTER_TYPE_F); \
980 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
981 }
982
983 /* Rounding operations (other than RNDD) require two instructions - the first
984 * stores a rounded value (possibly the wrong way) in the dest register, but
985 * also sets a per-channel "increment bit" in the flag register. A predicated
986 * add of 1.0 fixes dest to contain the desired result.
987 *
988 * Sandybridge and later appear to round correctly without an ADD.
989 */
990 #define ROUND(OP) \
991 void brw_##OP(struct brw_compile *p, \
992 struct brw_reg dest, \
993 struct brw_reg src) \
994 { \
995 struct brw_context *brw = p->brw; \
996 brw_inst *rnd, *add; \
997 rnd = next_insn(p, BRW_OPCODE_##OP); \
998 brw_set_dest(p, rnd, dest); \
999 brw_set_src0(p, rnd, src); \
1000 \
1001 if (brw->gen < 6) { \
1002 /* turn on round-increments */ \
1003 brw_inst_set_cond_modifier(brw, rnd, BRW_CONDITIONAL_R); \
1004 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1005 brw_inst_set_pred_control(brw, add, BRW_PREDICATE_NORMAL); \
1006 } \
1007 }
1008
1009
1010 ALU1(MOV)
1011 ALU2(SEL)
1012 ALU1(NOT)
1013 ALU2(AND)
1014 ALU2(OR)
1015 ALU2(XOR)
1016 ALU2(SHR)
1017 ALU2(SHL)
1018 ALU2(ASR)
1019 ALU1(FRC)
1020 ALU1(RNDD)
1021 ALU2(MAC)
1022 ALU2(MACH)
1023 ALU1(LZD)
1024 ALU2(DP4)
1025 ALU2(DPH)
1026 ALU2(DP3)
1027 ALU2(DP2)
1028 ALU2(PLN)
1029 ALU3F(MAD)
1030 ALU3F(LRP)
1031 ALU1(BFREV)
1032 ALU3(BFE)
1033 ALU2(BFI1)
1034 ALU3(BFI2)
1035 ALU1(FBH)
1036 ALU1(FBL)
1037 ALU1(CBIT)
1038 ALU2(ADDC)
1039 ALU2(SUBB)
1040
1041 ROUND(RNDZ)
1042 ROUND(RNDE)
1043
1044
1045 brw_inst *
1046 brw_ADD(struct brw_compile *p, struct brw_reg dest,
1047 struct brw_reg src0, struct brw_reg src1)
1048 {
1049 /* 6.2.2: add */
1050 if (src0.type == BRW_REGISTER_TYPE_F ||
1051 (src0.file == BRW_IMMEDIATE_VALUE &&
1052 src0.type == BRW_REGISTER_TYPE_VF)) {
1053 assert(src1.type != BRW_REGISTER_TYPE_UD);
1054 assert(src1.type != BRW_REGISTER_TYPE_D);
1055 }
1056
1057 if (src1.type == BRW_REGISTER_TYPE_F ||
1058 (src1.file == BRW_IMMEDIATE_VALUE &&
1059 src1.type == BRW_REGISTER_TYPE_VF)) {
1060 assert(src0.type != BRW_REGISTER_TYPE_UD);
1061 assert(src0.type != BRW_REGISTER_TYPE_D);
1062 }
1063
1064 return brw_alu2(p, BRW_OPCODE_ADD, dest, src0, src1);
1065 }
1066
1067 brw_inst *
1068 brw_AVG(struct brw_compile *p, struct brw_reg dest,
1069 struct brw_reg src0, struct brw_reg src1)
1070 {
1071 assert(dest.type == src0.type);
1072 assert(src0.type == src1.type);
1073 switch (src0.type) {
1074 case BRW_REGISTER_TYPE_B:
1075 case BRW_REGISTER_TYPE_UB:
1076 case BRW_REGISTER_TYPE_W:
1077 case BRW_REGISTER_TYPE_UW:
1078 case BRW_REGISTER_TYPE_D:
1079 case BRW_REGISTER_TYPE_UD:
1080 break;
1081 default:
1082 unreachable("Bad type for brw_AVG");
1083 }
1084
1085 return brw_alu2(p, BRW_OPCODE_AVG, dest, src0, src1);
1086 }
1087
1088 brw_inst *
1089 brw_MUL(struct brw_compile *p, struct brw_reg dest,
1090 struct brw_reg src0, struct brw_reg src1)
1091 {
1092 /* 6.32.38: mul */
1093 if (src0.type == BRW_REGISTER_TYPE_D ||
1094 src0.type == BRW_REGISTER_TYPE_UD ||
1095 src1.type == BRW_REGISTER_TYPE_D ||
1096 src1.type == BRW_REGISTER_TYPE_UD) {
1097 assert(dest.type != BRW_REGISTER_TYPE_F);
1098 }
1099
1100 if (src0.type == BRW_REGISTER_TYPE_F ||
1101 (src0.file == BRW_IMMEDIATE_VALUE &&
1102 src0.type == BRW_REGISTER_TYPE_VF)) {
1103 assert(src1.type != BRW_REGISTER_TYPE_UD);
1104 assert(src1.type != BRW_REGISTER_TYPE_D);
1105 }
1106
1107 if (src1.type == BRW_REGISTER_TYPE_F ||
1108 (src1.file == BRW_IMMEDIATE_VALUE &&
1109 src1.type == BRW_REGISTER_TYPE_VF)) {
1110 assert(src0.type != BRW_REGISTER_TYPE_UD);
1111 assert(src0.type != BRW_REGISTER_TYPE_D);
1112 }
1113
1114 assert(src0.file != BRW_ARCHITECTURE_REGISTER_FILE ||
1115 src0.nr != BRW_ARF_ACCUMULATOR);
1116 assert(src1.file != BRW_ARCHITECTURE_REGISTER_FILE ||
1117 src1.nr != BRW_ARF_ACCUMULATOR);
1118
1119 return brw_alu2(p, BRW_OPCODE_MUL, dest, src0, src1);
1120 }
1121
1122 brw_inst *
1123 brw_LINE(struct brw_compile *p, struct brw_reg dest,
1124 struct brw_reg src0, struct brw_reg src1)
1125 {
1126 src0.vstride = BRW_VERTICAL_STRIDE_0;
1127 src0.width = BRW_WIDTH_1;
1128 src0.hstride = BRW_HORIZONTAL_STRIDE_0;
1129 return brw_alu2(p, BRW_OPCODE_LINE, dest, src0, src1);
1130 }
1131
1132 brw_inst *
1133 brw_F32TO16(struct brw_compile *p, struct brw_reg dst, struct brw_reg src)
1134 {
1135 const struct brw_context *brw = p->brw;
1136 const bool align16 = brw_inst_access_mode(brw, p->current) == BRW_ALIGN_16;
1137 /* The F32TO16 instruction doesn't support 32-bit destination types in
1138 * Align1 mode, and neither does the Gen8 implementation in terms of a
1139 * converting MOV. Gen7 does zero out the high 16 bits in Align16 mode as
1140 * an undocumented feature.
1141 */
1142 const bool needs_zero_fill = (dst.type == BRW_REGISTER_TYPE_UD &&
1143 (!align16 || brw->gen >= 8));
1144 brw_inst *inst;
1145
1146 if (align16) {
1147 assert(dst.type == BRW_REGISTER_TYPE_UD);
1148 } else {
1149 assert(dst.type == BRW_REGISTER_TYPE_UD ||
1150 dst.type == BRW_REGISTER_TYPE_W ||
1151 dst.type == BRW_REGISTER_TYPE_UW ||
1152 dst.type == BRW_REGISTER_TYPE_HF);
1153 }
1154
1155 brw_push_insn_state(p);
1156
1157 if (needs_zero_fill) {
1158 brw_set_default_access_mode(p, BRW_ALIGN_1);
1159 dst = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1160 }
1161
1162 if (brw->gen >= 8) {
1163 inst = brw_MOV(p, retype(dst, BRW_REGISTER_TYPE_HF), src);
1164 } else {
1165 assert(brw->gen == 7);
1166 inst = brw_alu1(p, BRW_OPCODE_F32TO16, dst, src);
1167 }
1168
1169 if (needs_zero_fill) {
1170 brw_inst_set_no_dd_clear(brw, inst, true);
1171 inst = brw_MOV(p, suboffset(dst, 1), brw_imm_ud(0u));
1172 brw_inst_set_no_dd_check(brw, inst, true);
1173 }
1174
1175 brw_pop_insn_state(p);
1176 return inst;
1177 }
1178
1179 brw_inst *
1180 brw_F16TO32(struct brw_compile *p, struct brw_reg dst, struct brw_reg src)
1181 {
1182 const struct brw_context *brw = p->brw;
1183 bool align16 = brw_inst_access_mode(brw, p->current) == BRW_ALIGN_16;
1184
1185 if (align16) {
1186 assert(src.type == BRW_REGISTER_TYPE_UD);
1187 } else {
1188 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1189 *
1190 * Because this instruction does not have a 16-bit floating-point
1191 * type, the source data type must be Word (W). The destination type
1192 * must be F (Float).
1193 */
1194 if (src.type == BRW_REGISTER_TYPE_UD)
1195 src = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1196
1197 assert(src.type == BRW_REGISTER_TYPE_W ||
1198 src.type == BRW_REGISTER_TYPE_UW ||
1199 src.type == BRW_REGISTER_TYPE_HF);
1200 }
1201
1202 if (brw->gen >= 8) {
1203 return brw_MOV(p, dst, retype(src, BRW_REGISTER_TYPE_HF));
1204 } else {
1205 assert(brw->gen == 7);
1206 return brw_alu1(p, BRW_OPCODE_F16TO32, dst, src);
1207 }
1208 }
1209
1210
1211 void brw_NOP(struct brw_compile *p)
1212 {
1213 brw_inst *insn = next_insn(p, BRW_OPCODE_NOP);
1214 brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1215 brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1216 brw_set_src1(p, insn, brw_imm_ud(0x0));
1217 }
1218
1219
1220
1221
1222
1223 /***********************************************************************
1224 * Comparisons, if/else/endif
1225 */
1226
1227 brw_inst *
1228 brw_JMPI(struct brw_compile *p, struct brw_reg index,
1229 unsigned predicate_control)
1230 {
1231 const struct brw_context *brw = p->brw;
1232 struct brw_reg ip = brw_ip_reg();
1233 brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index);
1234
1235 brw_inst_set_exec_size(brw, inst, BRW_EXECUTE_2);
1236 brw_inst_set_qtr_control(brw, inst, BRW_COMPRESSION_NONE);
1237 brw_inst_set_mask_control(brw, inst, BRW_MASK_DISABLE);
1238 brw_inst_set_pred_control(brw, inst, predicate_control);
1239
1240 return inst;
1241 }
1242
1243 static void
1244 push_if_stack(struct brw_compile *p, brw_inst *inst)
1245 {
1246 p->if_stack[p->if_stack_depth] = inst - p->store;
1247
1248 p->if_stack_depth++;
1249 if (p->if_stack_array_size <= p->if_stack_depth) {
1250 p->if_stack_array_size *= 2;
1251 p->if_stack = reralloc(p->mem_ctx, p->if_stack, int,
1252 p->if_stack_array_size);
1253 }
1254 }
1255
1256 static brw_inst *
1257 pop_if_stack(struct brw_compile *p)
1258 {
1259 p->if_stack_depth--;
1260 return &p->store[p->if_stack[p->if_stack_depth]];
1261 }
1262
1263 static void
1264 push_loop_stack(struct brw_compile *p, brw_inst *inst)
1265 {
1266 if (p->loop_stack_array_size < p->loop_stack_depth) {
1267 p->loop_stack_array_size *= 2;
1268 p->loop_stack = reralloc(p->mem_ctx, p->loop_stack, int,
1269 p->loop_stack_array_size);
1270 p->if_depth_in_loop = reralloc(p->mem_ctx, p->if_depth_in_loop, int,
1271 p->loop_stack_array_size);
1272 }
1273
1274 p->loop_stack[p->loop_stack_depth] = inst - p->store;
1275 p->loop_stack_depth++;
1276 p->if_depth_in_loop[p->loop_stack_depth] = 0;
1277 }
1278
1279 static brw_inst *
1280 get_inner_do_insn(struct brw_compile *p)
1281 {
1282 return &p->store[p->loop_stack[p->loop_stack_depth - 1]];
1283 }
1284
1285 /* EU takes the value from the flag register and pushes it onto some
1286 * sort of a stack (presumably merging with any flag value already on
1287 * the stack). Within an if block, the flags at the top of the stack
1288 * control execution on each channel of the unit, eg. on each of the
1289 * 16 pixel values in our wm programs.
1290 *
1291 * When the matching 'else' instruction is reached (presumably by
1292 * countdown of the instruction count patched in by our ELSE/ENDIF
1293 * functions), the relevent flags are inverted.
1294 *
1295 * When the matching 'endif' instruction is reached, the flags are
1296 * popped off. If the stack is now empty, normal execution resumes.
1297 */
1298 brw_inst *
1299 brw_IF(struct brw_compile *p, unsigned execute_size)
1300 {
1301 struct brw_context *brw = p->brw;
1302 brw_inst *insn;
1303
1304 insn = next_insn(p, BRW_OPCODE_IF);
1305
1306 /* Override the defaults for this instruction:
1307 */
1308 if (brw->gen < 6) {
1309 brw_set_dest(p, insn, brw_ip_reg());
1310 brw_set_src0(p, insn, brw_ip_reg());
1311 brw_set_src1(p, insn, brw_imm_d(0x0));
1312 } else if (brw->gen == 6) {
1313 brw_set_dest(p, insn, brw_imm_w(0));
1314 brw_inst_set_gen6_jump_count(brw, insn, 0);
1315 brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1316 brw_set_src1(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1317 } else if (brw->gen == 7) {
1318 brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1319 brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1320 brw_set_src1(p, insn, brw_imm_w(0));
1321 brw_inst_set_jip(brw, insn, 0);
1322 brw_inst_set_uip(brw, insn, 0);
1323 } else {
1324 brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1325 brw_set_src0(p, insn, brw_imm_d(0));
1326 brw_inst_set_jip(brw, insn, 0);
1327 brw_inst_set_uip(brw, insn, 0);
1328 }
1329
1330 brw_inst_set_exec_size(brw, insn, execute_size);
1331 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1332 brw_inst_set_pred_control(brw, insn, BRW_PREDICATE_NORMAL);
1333 brw_inst_set_mask_control(brw, insn, BRW_MASK_ENABLE);
1334 if (!p->single_program_flow && brw->gen < 6)
1335 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1336
1337 push_if_stack(p, insn);
1338 p->if_depth_in_loop[p->loop_stack_depth]++;
1339 return insn;
1340 }
1341
1342 /* This function is only used for gen6-style IF instructions with an
1343 * embedded comparison (conditional modifier). It is not used on gen7.
1344 */
1345 brw_inst *
1346 gen6_IF(struct brw_compile *p, enum brw_conditional_mod conditional,
1347 struct brw_reg src0, struct brw_reg src1)
1348 {
1349 const struct brw_context *brw = p->brw;
1350 brw_inst *insn;
1351
1352 insn = next_insn(p, BRW_OPCODE_IF);
1353
1354 brw_set_dest(p, insn, brw_imm_w(0));
1355 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1356 : BRW_EXECUTE_8);
1357 brw_inst_set_gen6_jump_count(brw, insn, 0);
1358 brw_set_src0(p, insn, src0);
1359 brw_set_src1(p, insn, src1);
1360
1361 assert(brw_inst_qtr_control(brw, insn) == BRW_COMPRESSION_NONE);
1362 assert(brw_inst_pred_control(brw, insn) == BRW_PREDICATE_NONE);
1363 brw_inst_set_cond_modifier(brw, insn, conditional);
1364
1365 push_if_stack(p, insn);
1366 return insn;
1367 }
1368
1369 /**
1370 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1371 */
1372 static void
1373 convert_IF_ELSE_to_ADD(struct brw_compile *p,
1374 brw_inst *if_inst, brw_inst *else_inst)
1375 {
1376 const struct brw_context *brw = p->brw;
1377
1378 /* The next instruction (where the ENDIF would be, if it existed) */
1379 brw_inst *next_inst = &p->store[p->nr_insn];
1380
1381 assert(p->single_program_flow);
1382 assert(if_inst != NULL && brw_inst_opcode(brw, if_inst) == BRW_OPCODE_IF);
1383 assert(else_inst == NULL || brw_inst_opcode(brw, else_inst) == BRW_OPCODE_ELSE);
1384 assert(brw_inst_exec_size(brw, if_inst) == BRW_EXECUTE_1);
1385
1386 /* Convert IF to an ADD instruction that moves the instruction pointer
1387 * to the first instruction of the ELSE block. If there is no ELSE
1388 * block, point to where ENDIF would be. Reverse the predicate.
1389 *
1390 * There's no need to execute an ENDIF since we don't need to do any
1391 * stack operations, and if we're currently executing, we just want to
1392 * continue normally.
1393 */
1394 brw_inst_set_opcode(brw, if_inst, BRW_OPCODE_ADD);
1395 brw_inst_set_pred_inv(brw, if_inst, true);
1396
1397 if (else_inst != NULL) {
1398 /* Convert ELSE to an ADD instruction that points where the ENDIF
1399 * would be.
1400 */
1401 brw_inst_set_opcode(brw, else_inst, BRW_OPCODE_ADD);
1402
1403 brw_inst_set_imm_ud(brw, if_inst, (else_inst - if_inst + 1) * 16);
1404 brw_inst_set_imm_ud(brw, else_inst, (next_inst - else_inst) * 16);
1405 } else {
1406 brw_inst_set_imm_ud(brw, if_inst, (next_inst - if_inst) * 16);
1407 }
1408 }
1409
1410 /**
1411 * Patch IF and ELSE instructions with appropriate jump targets.
1412 */
1413 static void
1414 patch_IF_ELSE(struct brw_compile *p,
1415 brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst)
1416 {
1417 struct brw_context *brw = p->brw;
1418
1419 /* We shouldn't be patching IF and ELSE instructions in single program flow
1420 * mode when gen < 6, because in single program flow mode on those
1421 * platforms, we convert flow control instructions to conditional ADDs that
1422 * operate on IP (see brw_ENDIF).
1423 *
1424 * However, on Gen6, writing to IP doesn't work in single program flow mode
1425 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1426 * not be updated by non-flow control instructions."). And on later
1427 * platforms, there is no significant benefit to converting control flow
1428 * instructions to conditional ADDs. So we do patch IF and ELSE
1429 * instructions in single program flow mode on those platforms.
1430 */
1431 if (brw->gen < 6)
1432 assert(!p->single_program_flow);
1433
1434 assert(if_inst != NULL && brw_inst_opcode(brw, if_inst) == BRW_OPCODE_IF);
1435 assert(endif_inst != NULL);
1436 assert(else_inst == NULL || brw_inst_opcode(brw, else_inst) == BRW_OPCODE_ELSE);
1437
1438 unsigned br = brw_jump_scale(brw);
1439
1440 assert(brw_inst_opcode(brw, endif_inst) == BRW_OPCODE_ENDIF);
1441 brw_inst_set_exec_size(brw, endif_inst, brw_inst_exec_size(brw, if_inst));
1442
1443 if (else_inst == NULL) {
1444 /* Patch IF -> ENDIF */
1445 if (brw->gen < 6) {
1446 /* Turn it into an IFF, which means no mask stack operations for
1447 * all-false and jumping past the ENDIF.
1448 */
1449 brw_inst_set_opcode(brw, if_inst, BRW_OPCODE_IFF);
1450 brw_inst_set_gen4_jump_count(brw, if_inst,
1451 br * (endif_inst - if_inst + 1));
1452 brw_inst_set_gen4_pop_count(brw, if_inst, 0);
1453 } else if (brw->gen == 6) {
1454 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1455 brw_inst_set_gen6_jump_count(brw, if_inst, br*(endif_inst - if_inst));
1456 } else {
1457 brw_inst_set_uip(brw, if_inst, br * (endif_inst - if_inst));
1458 brw_inst_set_jip(brw, if_inst, br * (endif_inst - if_inst));
1459 }
1460 } else {
1461 brw_inst_set_exec_size(brw, else_inst, brw_inst_exec_size(brw, if_inst));
1462
1463 /* Patch IF -> ELSE */
1464 if (brw->gen < 6) {
1465 brw_inst_set_gen4_jump_count(brw, if_inst,
1466 br * (else_inst - if_inst));
1467 brw_inst_set_gen4_pop_count(brw, if_inst, 0);
1468 } else if (brw->gen == 6) {
1469 brw_inst_set_gen6_jump_count(brw, if_inst,
1470 br * (else_inst - if_inst + 1));
1471 }
1472
1473 /* Patch ELSE -> ENDIF */
1474 if (brw->gen < 6) {
1475 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1476 * matching ENDIF.
1477 */
1478 brw_inst_set_gen4_jump_count(brw, else_inst,
1479 br * (endif_inst - else_inst + 1));
1480 brw_inst_set_gen4_pop_count(brw, else_inst, 1);
1481 } else if (brw->gen == 6) {
1482 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1483 brw_inst_set_gen6_jump_count(brw, else_inst,
1484 br * (endif_inst - else_inst));
1485 } else {
1486 /* The IF instruction's JIP should point just past the ELSE */
1487 brw_inst_set_jip(brw, if_inst, br * (else_inst - if_inst + 1));
1488 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1489 brw_inst_set_uip(brw, if_inst, br * (endif_inst - if_inst));
1490 brw_inst_set_jip(brw, else_inst, br * (endif_inst - else_inst));
1491 if (brw->gen >= 8) {
1492 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1493 * should point to ENDIF.
1494 */
1495 brw_inst_set_uip(brw, else_inst, br * (endif_inst - else_inst));
1496 }
1497 }
1498 }
1499 }
1500
1501 void
1502 brw_ELSE(struct brw_compile *p)
1503 {
1504 struct brw_context *brw = p->brw;
1505 brw_inst *insn;
1506
1507 insn = next_insn(p, BRW_OPCODE_ELSE);
1508
1509 if (brw->gen < 6) {
1510 brw_set_dest(p, insn, brw_ip_reg());
1511 brw_set_src0(p, insn, brw_ip_reg());
1512 brw_set_src1(p, insn, brw_imm_d(0x0));
1513 } else if (brw->gen == 6) {
1514 brw_set_dest(p, insn, brw_imm_w(0));
1515 brw_inst_set_gen6_jump_count(brw, insn, 0);
1516 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1517 brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1518 } else if (brw->gen == 7) {
1519 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1520 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1521 brw_set_src1(p, insn, brw_imm_w(0));
1522 brw_inst_set_jip(brw, insn, 0);
1523 brw_inst_set_uip(brw, insn, 0);
1524 } else {
1525 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1526 brw_set_src0(p, insn, brw_imm_d(0));
1527 brw_inst_set_jip(brw, insn, 0);
1528 brw_inst_set_uip(brw, insn, 0);
1529 }
1530
1531 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1532 brw_inst_set_mask_control(brw, insn, BRW_MASK_ENABLE);
1533 if (!p->single_program_flow && brw->gen < 6)
1534 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1535
1536 push_if_stack(p, insn);
1537 }
1538
1539 void
1540 brw_ENDIF(struct brw_compile *p)
1541 {
1542 struct brw_context *brw = p->brw;
1543 brw_inst *insn = NULL;
1544 brw_inst *else_inst = NULL;
1545 brw_inst *if_inst = NULL;
1546 brw_inst *tmp;
1547 bool emit_endif = true;
1548
1549 /* In single program flow mode, we can express IF and ELSE instructions
1550 * equivalently as ADD instructions that operate on IP. On platforms prior
1551 * to Gen6, flow control instructions cause an implied thread switch, so
1552 * this is a significant savings.
1553 *
1554 * However, on Gen6, writing to IP doesn't work in single program flow mode
1555 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1556 * not be updated by non-flow control instructions."). And on later
1557 * platforms, there is no significant benefit to converting control flow
1558 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1559 * Gen5.
1560 */
1561 if (brw->gen < 6 && p->single_program_flow)
1562 emit_endif = false;
1563
1564 /*
1565 * A single next_insn() may change the base adress of instruction store
1566 * memory(p->store), so call it first before referencing the instruction
1567 * store pointer from an index
1568 */
1569 if (emit_endif)
1570 insn = next_insn(p, BRW_OPCODE_ENDIF);
1571
1572 /* Pop the IF and (optional) ELSE instructions from the stack */
1573 p->if_depth_in_loop[p->loop_stack_depth]--;
1574 tmp = pop_if_stack(p);
1575 if (brw_inst_opcode(brw, tmp) == BRW_OPCODE_ELSE) {
1576 else_inst = tmp;
1577 tmp = pop_if_stack(p);
1578 }
1579 if_inst = tmp;
1580
1581 if (!emit_endif) {
1582 /* ENDIF is useless; don't bother emitting it. */
1583 convert_IF_ELSE_to_ADD(p, if_inst, else_inst);
1584 return;
1585 }
1586
1587 if (brw->gen < 6) {
1588 brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1589 brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1590 brw_set_src1(p, insn, brw_imm_d(0x0));
1591 } else if (brw->gen == 6) {
1592 brw_set_dest(p, insn, brw_imm_w(0));
1593 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1594 brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1595 } else if (brw->gen == 7) {
1596 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1597 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1598 brw_set_src1(p, insn, brw_imm_w(0));
1599 } else {
1600 brw_set_src0(p, insn, brw_imm_d(0));
1601 }
1602
1603 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1604 brw_inst_set_mask_control(brw, insn, BRW_MASK_ENABLE);
1605 if (brw->gen < 6)
1606 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1607
1608 /* Also pop item off the stack in the endif instruction: */
1609 if (brw->gen < 6) {
1610 brw_inst_set_gen4_jump_count(brw, insn, 0);
1611 brw_inst_set_gen4_pop_count(brw, insn, 1);
1612 } else if (brw->gen == 6) {
1613 brw_inst_set_gen6_jump_count(brw, insn, 2);
1614 } else {
1615 brw_inst_set_jip(brw, insn, 2);
1616 }
1617 patch_IF_ELSE(p, if_inst, else_inst, insn);
1618 }
1619
1620 brw_inst *
1621 brw_BREAK(struct brw_compile *p)
1622 {
1623 struct brw_context *brw = p->brw;
1624 brw_inst *insn;
1625
1626 insn = next_insn(p, BRW_OPCODE_BREAK);
1627 if (brw->gen >= 8) {
1628 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1629 brw_set_src0(p, insn, brw_imm_d(0x0));
1630 } else if (brw->gen >= 6) {
1631 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1632 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1633 brw_set_src1(p, insn, brw_imm_d(0x0));
1634 } else {
1635 brw_set_dest(p, insn, brw_ip_reg());
1636 brw_set_src0(p, insn, brw_ip_reg());
1637 brw_set_src1(p, insn, brw_imm_d(0x0));
1638 brw_inst_set_gen4_pop_count(brw, insn,
1639 p->if_depth_in_loop[p->loop_stack_depth]);
1640 }
1641 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1642 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1643 : BRW_EXECUTE_8);
1644
1645 return insn;
1646 }
1647
1648 brw_inst *
1649 brw_CONT(struct brw_compile *p)
1650 {
1651 const struct brw_context *brw = p->brw;
1652 brw_inst *insn;
1653
1654 insn = next_insn(p, BRW_OPCODE_CONTINUE);
1655 brw_set_dest(p, insn, brw_ip_reg());
1656 if (brw->gen >= 8) {
1657 brw_set_src0(p, insn, brw_imm_d(0x0));
1658 } else {
1659 brw_set_src0(p, insn, brw_ip_reg());
1660 brw_set_src1(p, insn, brw_imm_d(0x0));
1661 }
1662
1663 if (brw->gen < 6) {
1664 brw_inst_set_gen4_pop_count(brw, insn,
1665 p->if_depth_in_loop[p->loop_stack_depth]);
1666 }
1667 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1668 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1669 : BRW_EXECUTE_8);
1670 return insn;
1671 }
1672
1673 brw_inst *
1674 gen6_HALT(struct brw_compile *p)
1675 {
1676 const struct brw_context *brw = p->brw;
1677 brw_inst *insn;
1678
1679 insn = next_insn(p, BRW_OPCODE_HALT);
1680 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1681 if (brw->gen >= 8) {
1682 brw_set_src0(p, insn, brw_imm_d(0x0));
1683 } else {
1684 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1685 brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1686 }
1687
1688 if (p->compressed) {
1689 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_16);
1690 } else {
1691 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1692 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_8);
1693 }
1694 return insn;
1695 }
1696
1697 /* DO/WHILE loop:
1698 *
1699 * The DO/WHILE is just an unterminated loop -- break or continue are
1700 * used for control within the loop. We have a few ways they can be
1701 * done.
1702 *
1703 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1704 * jip and no DO instruction.
1705 *
1706 * For non-uniform control flow pre-gen6, there's a DO instruction to
1707 * push the mask, and a WHILE to jump back, and BREAK to get out and
1708 * pop the mask.
1709 *
1710 * For gen6, there's no more mask stack, so no need for DO. WHILE
1711 * just points back to the first instruction of the loop.
1712 */
1713 brw_inst *
1714 brw_DO(struct brw_compile *p, unsigned execute_size)
1715 {
1716 struct brw_context *brw = p->brw;
1717
1718 if (brw->gen >= 6 || p->single_program_flow) {
1719 push_loop_stack(p, &p->store[p->nr_insn]);
1720 return &p->store[p->nr_insn];
1721 } else {
1722 brw_inst *insn = next_insn(p, BRW_OPCODE_DO);
1723
1724 push_loop_stack(p, insn);
1725
1726 /* Override the defaults for this instruction:
1727 */
1728 brw_set_dest(p, insn, brw_null_reg());
1729 brw_set_src0(p, insn, brw_null_reg());
1730 brw_set_src1(p, insn, brw_null_reg());
1731
1732 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1733 brw_inst_set_exec_size(brw, insn, execute_size);
1734 brw_inst_set_pred_control(brw, insn, BRW_PREDICATE_NONE);
1735
1736 return insn;
1737 }
1738 }
1739
1740 /**
1741 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1742 * instruction here.
1743 *
1744 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1745 * nesting, since it can always just point to the end of the block/current loop.
1746 */
1747 static void
1748 brw_patch_break_cont(struct brw_compile *p, brw_inst *while_inst)
1749 {
1750 struct brw_context *brw = p->brw;
1751 brw_inst *do_inst = get_inner_do_insn(p);
1752 brw_inst *inst;
1753 unsigned br = brw_jump_scale(brw);
1754
1755 assert(brw->gen < 6);
1756
1757 for (inst = while_inst - 1; inst != do_inst; inst--) {
1758 /* If the jump count is != 0, that means that this instruction has already
1759 * been patched because it's part of a loop inside of the one we're
1760 * patching.
1761 */
1762 if (brw_inst_opcode(brw, inst) == BRW_OPCODE_BREAK &&
1763 brw_inst_gen4_jump_count(brw, inst) == 0) {
1764 brw_inst_set_gen4_jump_count(brw, inst, br*((while_inst - inst) + 1));
1765 } else if (brw_inst_opcode(brw, inst) == BRW_OPCODE_CONTINUE &&
1766 brw_inst_gen4_jump_count(brw, inst) == 0) {
1767 brw_inst_set_gen4_jump_count(brw, inst, br * (while_inst - inst));
1768 }
1769 }
1770 }
1771
1772 brw_inst *
1773 brw_WHILE(struct brw_compile *p)
1774 {
1775 struct brw_context *brw = p->brw;
1776 brw_inst *insn, *do_insn;
1777 unsigned br = brw_jump_scale(brw);
1778
1779 if (brw->gen >= 6) {
1780 insn = next_insn(p, BRW_OPCODE_WHILE);
1781 do_insn = get_inner_do_insn(p);
1782
1783 if (brw->gen >= 8) {
1784 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1785 brw_set_src0(p, insn, brw_imm_d(0));
1786 brw_inst_set_jip(brw, insn, br * (do_insn - insn));
1787 } else if (brw->gen == 7) {
1788 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1789 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1790 brw_set_src1(p, insn, brw_imm_w(0));
1791 brw_inst_set_jip(brw, insn, br * (do_insn - insn));
1792 } else {
1793 brw_set_dest(p, insn, brw_imm_w(0));
1794 brw_inst_set_gen6_jump_count(brw, insn, br * (do_insn - insn));
1795 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1796 brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1797 }
1798
1799 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1800 : BRW_EXECUTE_8);
1801 } else {
1802 if (p->single_program_flow) {
1803 insn = next_insn(p, BRW_OPCODE_ADD);
1804 do_insn = get_inner_do_insn(p);
1805
1806 brw_set_dest(p, insn, brw_ip_reg());
1807 brw_set_src0(p, insn, brw_ip_reg());
1808 brw_set_src1(p, insn, brw_imm_d((do_insn - insn) * 16));
1809 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_1);
1810 } else {
1811 insn = next_insn(p, BRW_OPCODE_WHILE);
1812 do_insn = get_inner_do_insn(p);
1813
1814 assert(brw_inst_opcode(brw, do_insn) == BRW_OPCODE_DO);
1815
1816 brw_set_dest(p, insn, brw_ip_reg());
1817 brw_set_src0(p, insn, brw_ip_reg());
1818 brw_set_src1(p, insn, brw_imm_d(0));
1819
1820 brw_inst_set_exec_size(brw, insn, brw_inst_exec_size(brw, do_insn));
1821 brw_inst_set_gen4_jump_count(brw, insn, br * (do_insn - insn + 1));
1822 brw_inst_set_gen4_pop_count(brw, insn, 0);
1823
1824 brw_patch_break_cont(p, insn);
1825 }
1826 }
1827 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1828
1829 p->loop_stack_depth--;
1830
1831 return insn;
1832 }
1833
1834 /* FORWARD JUMPS:
1835 */
1836 void brw_land_fwd_jump(struct brw_compile *p, int jmp_insn_idx)
1837 {
1838 struct brw_context *brw = p->brw;
1839 brw_inst *jmp_insn = &p->store[jmp_insn_idx];
1840 unsigned jmpi = 1;
1841
1842 if (brw->gen >= 5)
1843 jmpi = 2;
1844
1845 assert(brw_inst_opcode(brw, jmp_insn) == BRW_OPCODE_JMPI);
1846 assert(brw_inst_src1_reg_file(brw, jmp_insn) == BRW_IMMEDIATE_VALUE);
1847
1848 brw_inst_set_gen4_jump_count(brw, jmp_insn,
1849 jmpi * (p->nr_insn - jmp_insn_idx - 1));
1850 }
1851
1852 /* To integrate with the above, it makes sense that the comparison
1853 * instruction should populate the flag register. It might be simpler
1854 * just to use the flag reg for most WM tasks?
1855 */
1856 void brw_CMP(struct brw_compile *p,
1857 struct brw_reg dest,
1858 unsigned conditional,
1859 struct brw_reg src0,
1860 struct brw_reg src1)
1861 {
1862 struct brw_context *brw = p->brw;
1863 brw_inst *insn = next_insn(p, BRW_OPCODE_CMP);
1864
1865 brw_inst_set_cond_modifier(brw, insn, conditional);
1866 brw_set_dest(p, insn, dest);
1867 brw_set_src0(p, insn, src0);
1868 brw_set_src1(p, insn, src1);
1869
1870 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1871 * page says:
1872 * "Any CMP instruction with a null destination must use a {switch}."
1873 *
1874 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1875 * mentioned on their work-arounds pages.
1876 */
1877 if (brw->gen == 7) {
1878 if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE &&
1879 dest.nr == BRW_ARF_NULL) {
1880 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1881 }
1882 }
1883 }
1884
1885 /***********************************************************************
1886 * Helpers for the various SEND message types:
1887 */
1888
1889 /** Extended math function, float[8].
1890 */
1891 void gen4_math(struct brw_compile *p,
1892 struct brw_reg dest,
1893 unsigned function,
1894 unsigned msg_reg_nr,
1895 struct brw_reg src,
1896 unsigned precision )
1897 {
1898 struct brw_context *brw = p->brw;
1899 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
1900 unsigned data_type;
1901 if (has_scalar_region(src)) {
1902 data_type = BRW_MATH_DATA_SCALAR;
1903 } else {
1904 data_type = BRW_MATH_DATA_VECTOR;
1905 }
1906
1907 assert(brw->gen < 6);
1908
1909 /* Example code doesn't set predicate_control for send
1910 * instructions.
1911 */
1912 brw_inst_set_pred_control(brw, insn, 0);
1913 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
1914
1915 brw_set_dest(p, insn, dest);
1916 brw_set_src0(p, insn, src);
1917 brw_set_math_message(p,
1918 insn,
1919 function,
1920 src.type == BRW_REGISTER_TYPE_D,
1921 precision,
1922 data_type);
1923 }
1924
1925 void gen6_math(struct brw_compile *p,
1926 struct brw_reg dest,
1927 unsigned function,
1928 struct brw_reg src0,
1929 struct brw_reg src1)
1930 {
1931 struct brw_context *brw = p->brw;
1932 brw_inst *insn = next_insn(p, BRW_OPCODE_MATH);
1933
1934 assert(brw->gen >= 6);
1935
1936 assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
1937 (brw->gen >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE));
1938 assert(src0.file == BRW_GENERAL_REGISTER_FILE ||
1939 (brw->gen >= 8 && src0.file == BRW_IMMEDIATE_VALUE));
1940
1941 assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1);
1942 if (brw->gen == 6) {
1943 assert(src0.hstride == BRW_HORIZONTAL_STRIDE_1);
1944 assert(src1.hstride == BRW_HORIZONTAL_STRIDE_1);
1945 }
1946
1947 if (function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT ||
1948 function == BRW_MATH_FUNCTION_INT_DIV_REMAINDER ||
1949 function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER) {
1950 assert(src0.type != BRW_REGISTER_TYPE_F);
1951 assert(src1.type != BRW_REGISTER_TYPE_F);
1952 assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
1953 (brw->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
1954 } else {
1955 assert(src0.type == BRW_REGISTER_TYPE_F);
1956 assert(src1.type == BRW_REGISTER_TYPE_F);
1957 if (function == BRW_MATH_FUNCTION_POW) {
1958 assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
1959 (brw->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
1960 } else {
1961 assert(src1.file == BRW_ARCHITECTURE_REGISTER_FILE &&
1962 src1.nr == BRW_ARF_NULL);
1963 }
1964 }
1965
1966 /* Source modifiers are ignored for extended math instructions on Gen6. */
1967 if (brw->gen == 6) {
1968 assert(!src0.negate);
1969 assert(!src0.abs);
1970 assert(!src1.negate);
1971 assert(!src1.abs);
1972 }
1973
1974 brw_inst_set_math_function(brw, insn, function);
1975
1976 brw_set_dest(p, insn, dest);
1977 brw_set_src0(p, insn, src0);
1978 brw_set_src1(p, insn, src1);
1979 }
1980
1981
1982 /**
1983 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1984 * using a constant offset per channel.
1985 *
1986 * The offset must be aligned to oword size (16 bytes). Used for
1987 * register spilling.
1988 */
1989 void brw_oword_block_write_scratch(struct brw_compile *p,
1990 struct brw_reg mrf,
1991 int num_regs,
1992 unsigned offset)
1993 {
1994 struct brw_context *brw = p->brw;
1995 uint32_t msg_control, msg_type;
1996 int mlen;
1997
1998 if (brw->gen >= 6)
1999 offset /= 16;
2000
2001 mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2002
2003 if (num_regs == 1) {
2004 msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS;
2005 mlen = 2;
2006 } else {
2007 msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS;
2008 mlen = 3;
2009 }
2010
2011 /* Set up the message header. This is g0, with g0.2 filled with
2012 * the offset. We don't want to leave our offset around in g0 or
2013 * it'll screw up texture samples, so set it up inside the message
2014 * reg.
2015 */
2016 {
2017 brw_push_insn_state(p);
2018 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2019 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2020
2021 brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2022
2023 /* set message header global offset field (reg 0, element 2) */
2024 brw_MOV(p,
2025 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
2026 mrf.nr,
2027 2), BRW_REGISTER_TYPE_UD),
2028 brw_imm_ud(offset));
2029
2030 brw_pop_insn_state(p);
2031 }
2032
2033 {
2034 struct brw_reg dest;
2035 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2036 int send_commit_msg;
2037 struct brw_reg src_header = retype(brw_vec8_grf(0, 0),
2038 BRW_REGISTER_TYPE_UW);
2039
2040 if (brw_inst_qtr_control(brw, insn) != BRW_COMPRESSION_NONE) {
2041 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2042 src_header = vec16(src_header);
2043 }
2044 assert(brw_inst_pred_control(brw, insn) == BRW_PREDICATE_NONE);
2045 if (brw->gen < 6)
2046 brw_inst_set_base_mrf(brw, insn, mrf.nr);
2047
2048 /* Until gen6, writes followed by reads from the same location
2049 * are not guaranteed to be ordered unless write_commit is set.
2050 * If set, then a no-op write is issued to the destination
2051 * register to set a dependency, and a read from the destination
2052 * can be used to ensure the ordering.
2053 *
2054 * For gen6, only writes between different threads need ordering
2055 * protection. Our use of DP writes is all about register
2056 * spilling within a thread.
2057 */
2058 if (brw->gen >= 6) {
2059 dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2060 send_commit_msg = 0;
2061 } else {
2062 dest = src_header;
2063 send_commit_msg = 1;
2064 }
2065
2066 brw_set_dest(p, insn, dest);
2067 if (brw->gen >= 6) {
2068 brw_set_src0(p, insn, mrf);
2069 } else {
2070 brw_set_src0(p, insn, brw_null_reg());
2071 }
2072
2073 if (brw->gen >= 6)
2074 msg_type = GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
2075 else
2076 msg_type = BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
2077
2078 brw_set_dp_write_message(p,
2079 insn,
2080 255, /* binding table index (255=stateless) */
2081 msg_control,
2082 msg_type,
2083 mlen,
2084 true, /* header_present */
2085 0, /* not a render target */
2086 send_commit_msg, /* response_length */
2087 0, /* eot */
2088 send_commit_msg);
2089 }
2090 }
2091
2092
2093 /**
2094 * Read a block of owords (half a GRF each) from the scratch buffer
2095 * using a constant index per channel.
2096 *
2097 * Offset must be aligned to oword size (16 bytes). Used for register
2098 * spilling.
2099 */
2100 void
2101 brw_oword_block_read_scratch(struct brw_compile *p,
2102 struct brw_reg dest,
2103 struct brw_reg mrf,
2104 int num_regs,
2105 unsigned offset)
2106 {
2107 struct brw_context *brw = p->brw;
2108 uint32_t msg_control;
2109 int rlen;
2110
2111 if (brw->gen >= 6)
2112 offset /= 16;
2113
2114 if (p->brw->gen >= 7) {
2115 /* On gen 7 and above, we no longer have message registers and we can
2116 * send from any register we want. By using the destination register
2117 * for the message, we guarantee that the implied message write won't
2118 * accidentally overwrite anything. This has been a problem because
2119 * the MRF registers and source for the final FB write are both fixed
2120 * and may overlap.
2121 */
2122 mrf = retype(dest, BRW_REGISTER_TYPE_UD);
2123 } else {
2124 mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2125 }
2126 dest = retype(dest, BRW_REGISTER_TYPE_UW);
2127
2128 if (num_regs == 1) {
2129 msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS;
2130 rlen = 1;
2131 } else {
2132 msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS;
2133 rlen = 2;
2134 }
2135
2136 {
2137 brw_push_insn_state(p);
2138 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2139 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2140
2141 brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2142
2143 /* set message header global offset field (reg 0, element 2) */
2144 brw_MOV(p, get_element_ud(mrf, 2), brw_imm_ud(offset));
2145
2146 brw_pop_insn_state(p);
2147 }
2148
2149 {
2150 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2151
2152 assert(brw_inst_pred_control(brw, insn) == 0);
2153 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2154
2155 brw_set_dest(p, insn, dest); /* UW? */
2156 if (brw->gen >= 6) {
2157 brw_set_src0(p, insn, mrf);
2158 } else {
2159 brw_set_src0(p, insn, brw_null_reg());
2160 brw_inst_set_base_mrf(brw, insn, mrf.nr);
2161 }
2162
2163 brw_set_dp_read_message(p,
2164 insn,
2165 255, /* binding table index (255=stateless) */
2166 msg_control,
2167 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ, /* msg_type */
2168 BRW_DATAPORT_READ_TARGET_RENDER_CACHE,
2169 1, /* msg_length */
2170 true, /* header_present */
2171 rlen);
2172 }
2173 }
2174
2175 void
2176 gen7_block_read_scratch(struct brw_compile *p,
2177 struct brw_reg dest,
2178 int num_regs,
2179 unsigned offset)
2180 {
2181 const struct brw_context *brw = p->brw;
2182 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2183 assert(brw_inst_pred_control(brw, insn) == BRW_PREDICATE_NONE);
2184
2185 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2186 brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW));
2187
2188 /* The HW requires that the header is present; this is to get the g0.5
2189 * scratch offset.
2190 */
2191 brw_set_src0(p, insn, brw_vec8_grf(0, 0));
2192
2193 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2194 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2195 * is 32 bytes, which happens to be the size of a register.
2196 */
2197 offset /= REG_SIZE;
2198 assert(offset < (1 << 12));
2199
2200 gen7_set_dp_scratch_message(p, insn,
2201 false, /* scratch read */
2202 false, /* OWords */
2203 false, /* invalidate after read */
2204 num_regs,
2205 offset,
2206 1, /* mlen: just g0 */
2207 num_regs, /* rlen */
2208 true); /* header present */
2209 }
2210
2211 /**
2212 * Read a float[4] vector from the data port Data Cache (const buffer).
2213 * Location (in buffer) should be a multiple of 16.
2214 * Used for fetching shader constants.
2215 */
2216 void brw_oword_block_read(struct brw_compile *p,
2217 struct brw_reg dest,
2218 struct brw_reg mrf,
2219 uint32_t offset,
2220 uint32_t bind_table_index)
2221 {
2222 struct brw_context *brw = p->brw;
2223
2224 /* On newer hardware, offset is in units of owords. */
2225 if (brw->gen >= 6)
2226 offset /= 16;
2227
2228 mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2229
2230 brw_push_insn_state(p);
2231 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
2232 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2233 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2234
2235 brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2236
2237 /* set message header global offset field (reg 0, element 2) */
2238 brw_MOV(p,
2239 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
2240 mrf.nr,
2241 2), BRW_REGISTER_TYPE_UD),
2242 brw_imm_ud(offset));
2243
2244 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2245
2246 /* cast dest to a uword[8] vector */
2247 dest = retype(vec8(dest), BRW_REGISTER_TYPE_UW);
2248
2249 brw_set_dest(p, insn, dest);
2250 if (brw->gen >= 6) {
2251 brw_set_src0(p, insn, mrf);
2252 } else {
2253 brw_set_src0(p, insn, brw_null_reg());
2254 brw_inst_set_base_mrf(brw, insn, mrf.nr);
2255 }
2256
2257 brw_set_dp_read_message(p,
2258 insn,
2259 bind_table_index,
2260 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW,
2261 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ,
2262 BRW_DATAPORT_READ_TARGET_DATA_CACHE,
2263 1, /* msg_length */
2264 true, /* header_present */
2265 1); /* response_length (1 reg, 2 owords!) */
2266
2267 brw_pop_insn_state(p);
2268 }
2269
2270
2271 void brw_fb_WRITE(struct brw_compile *p,
2272 int dispatch_width,
2273 struct brw_reg payload,
2274 struct brw_reg implied_header,
2275 unsigned msg_control,
2276 unsigned binding_table_index,
2277 unsigned msg_length,
2278 unsigned response_length,
2279 bool eot,
2280 bool last_render_target,
2281 bool header_present)
2282 {
2283 struct brw_context *brw = p->brw;
2284 brw_inst *insn;
2285 unsigned msg_type;
2286 struct brw_reg dest, src0;
2287
2288 if (dispatch_width == 16)
2289 dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2290 else
2291 dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2292
2293 if (brw->gen >= 6) {
2294 insn = next_insn(p, BRW_OPCODE_SENDC);
2295 } else {
2296 insn = next_insn(p, BRW_OPCODE_SEND);
2297 }
2298 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2299
2300 if (brw->gen >= 6) {
2301 /* headerless version, just submit color payload */
2302 src0 = payload;
2303
2304 msg_type = GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE;
2305 } else {
2306 assert(payload.file == BRW_MESSAGE_REGISTER_FILE);
2307 brw_inst_set_base_mrf(brw, insn, payload.nr);
2308 src0 = implied_header;
2309
2310 msg_type = BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE;
2311 }
2312
2313 brw_set_dest(p, insn, dest);
2314 brw_set_src0(p, insn, src0);
2315 brw_set_dp_write_message(p,
2316 insn,
2317 binding_table_index,
2318 msg_control,
2319 msg_type,
2320 msg_length,
2321 header_present,
2322 last_render_target,
2323 response_length,
2324 eot,
2325 0 /* send_commit_msg */);
2326 }
2327
2328
2329 /**
2330 * Texture sample instruction.
2331 * Note: the msg_type plus msg_length values determine exactly what kind
2332 * of sampling operation is performed. See volume 4, page 161 of docs.
2333 */
2334 void brw_SAMPLE(struct brw_compile *p,
2335 struct brw_reg dest,
2336 unsigned msg_reg_nr,
2337 struct brw_reg src0,
2338 unsigned binding_table_index,
2339 unsigned sampler,
2340 unsigned msg_type,
2341 unsigned response_length,
2342 unsigned msg_length,
2343 unsigned header_present,
2344 unsigned simd_mode,
2345 unsigned return_format)
2346 {
2347 struct brw_context *brw = p->brw;
2348 brw_inst *insn;
2349
2350 if (msg_reg_nr != -1)
2351 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2352
2353 insn = next_insn(p, BRW_OPCODE_SEND);
2354 brw_inst_set_pred_control(brw, insn, BRW_PREDICATE_NONE); /* XXX */
2355
2356 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2357 *
2358 * "Instruction compression is not allowed for this instruction (that
2359 * is, send). The hardware behavior is undefined if this instruction is
2360 * set as compressed. However, compress control can be set to "SecHalf"
2361 * to affect the EMask generation."
2362 *
2363 * No similar wording is found in later PRMs, but there are examples
2364 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2365 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2366 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2367 */
2368 if (brw_inst_qtr_control(brw, insn) != BRW_COMPRESSION_2NDHALF)
2369 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2370
2371 if (brw->gen < 6)
2372 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
2373
2374 brw_set_dest(p, insn, dest);
2375 brw_set_src0(p, insn, src0);
2376 brw_set_sampler_message(p, insn,
2377 binding_table_index,
2378 sampler,
2379 msg_type,
2380 response_length,
2381 msg_length,
2382 header_present,
2383 simd_mode,
2384 return_format);
2385 }
2386
2387 /* Adjust the message header's sampler state pointer to
2388 * select the correct group of 16 samplers.
2389 */
2390 void brw_adjust_sampler_state_pointer(struct brw_compile *p,
2391 struct brw_reg header,
2392 struct brw_reg sampler_index)
2393 {
2394 /* The "Sampler Index" field can only store values between 0 and 15.
2395 * However, we can add an offset to the "Sampler State Pointer"
2396 * field, effectively selecting a different set of 16 samplers.
2397 *
2398 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2399 * offset, and each sampler state is only 16-bytes, so we can't
2400 * exclusively use the offset - we have to use both.
2401 */
2402
2403 struct brw_context *brw = p->brw;
2404
2405 if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
2406 const int sampler_state_size = 16; /* 16 bytes */
2407 uint32_t sampler = sampler_index.dw1.ud;
2408
2409 if (sampler >= 16) {
2410 assert(brw->is_haswell || brw->gen >= 8);
2411 brw_ADD(p,
2412 get_element_ud(header, 3),
2413 get_element_ud(brw_vec8_grf(0, 0), 3),
2414 brw_imm_ud(16 * (sampler / 16) * sampler_state_size));
2415 }
2416 } else {
2417 /* Non-const sampler array indexing case */
2418 if (brw->gen < 8 && !brw->is_haswell) {
2419 return;
2420 }
2421
2422 struct brw_reg temp = get_element_ud(header, 3);
2423
2424 brw_AND(p, temp, get_element_ud(sampler_index, 0), brw_imm_ud(0x0f0));
2425 brw_SHL(p, temp, temp, brw_imm_ud(4));
2426 brw_ADD(p,
2427 get_element_ud(header, 3),
2428 get_element_ud(brw_vec8_grf(0, 0), 3),
2429 temp);
2430 }
2431 }
2432
2433 /* All these variables are pretty confusing - we might be better off
2434 * using bitmasks and macros for this, in the old style. Or perhaps
2435 * just having the caller instantiate the fields in dword3 itself.
2436 */
2437 void brw_urb_WRITE(struct brw_compile *p,
2438 struct brw_reg dest,
2439 unsigned msg_reg_nr,
2440 struct brw_reg src0,
2441 enum brw_urb_write_flags flags,
2442 unsigned msg_length,
2443 unsigned response_length,
2444 unsigned offset,
2445 unsigned swizzle)
2446 {
2447 struct brw_context *brw = p->brw;
2448 brw_inst *insn;
2449
2450 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2451
2452 if (brw->gen >= 7 && !(flags & BRW_URB_WRITE_USE_CHANNEL_MASKS)) {
2453 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2454 brw_push_insn_state(p);
2455 brw_set_default_access_mode(p, BRW_ALIGN_1);
2456 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2457 brw_OR(p, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE, msg_reg_nr, 5),
2458 BRW_REGISTER_TYPE_UD),
2459 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD),
2460 brw_imm_ud(0xff00));
2461 brw_pop_insn_state(p);
2462 }
2463
2464 insn = next_insn(p, BRW_OPCODE_SEND);
2465
2466 assert(msg_length < BRW_MAX_MRF);
2467
2468 brw_set_dest(p, insn, dest);
2469 brw_set_src0(p, insn, src0);
2470 brw_set_src1(p, insn, brw_imm_d(0));
2471
2472 if (brw->gen < 6)
2473 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
2474
2475 brw_set_urb_message(p,
2476 insn,
2477 flags,
2478 msg_length,
2479 response_length,
2480 offset,
2481 swizzle);
2482 }
2483
2484 struct brw_inst *
2485 brw_send_indirect_message(struct brw_compile *p,
2486 unsigned sfid,
2487 struct brw_reg dst,
2488 struct brw_reg payload,
2489 struct brw_reg desc)
2490 {
2491 const struct brw_context *brw = p->brw;
2492 struct brw_inst *send, *setup;
2493
2494 assert(desc.type == BRW_REGISTER_TYPE_UD);
2495
2496 if (desc.file == BRW_IMMEDIATE_VALUE) {
2497 setup = send = next_insn(p, BRW_OPCODE_SEND);
2498 brw_set_src1(p, send, desc);
2499
2500 } else {
2501 struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
2502
2503 brw_push_insn_state(p);
2504 brw_set_default_access_mode(p, BRW_ALIGN_1);
2505 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2506 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
2507
2508 /* Load the indirect descriptor to an address register using OR so the
2509 * caller can specify additional descriptor bits with the usual
2510 * brw_set_*_message() helper functions.
2511 */
2512 setup = brw_OR(p, addr, desc, brw_imm_ud(0));
2513
2514 brw_pop_insn_state(p);
2515
2516 send = next_insn(p, BRW_OPCODE_SEND);
2517 brw_set_src1(p, send, addr);
2518 }
2519
2520 brw_set_dest(p, send, dst);
2521 brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
2522 brw_inst_set_sfid(brw, send, sfid);
2523
2524 return setup;
2525 }
2526
2527 static int
2528 brw_find_next_block_end(struct brw_compile *p, int start_offset)
2529 {
2530 int offset;
2531 void *store = p->store;
2532 const struct brw_context *brw = p->brw;
2533
2534 for (offset = next_offset(brw, store, start_offset);
2535 offset < p->next_insn_offset;
2536 offset = next_offset(brw, store, offset)) {
2537 brw_inst *insn = store + offset;
2538
2539 switch (brw_inst_opcode(brw, insn)) {
2540 case BRW_OPCODE_ENDIF:
2541 case BRW_OPCODE_ELSE:
2542 case BRW_OPCODE_WHILE:
2543 case BRW_OPCODE_HALT:
2544 return offset;
2545 }
2546 }
2547
2548 return 0;
2549 }
2550
2551 /* There is no DO instruction on gen6, so to find the end of the loop
2552 * we have to see if the loop is jumping back before our start
2553 * instruction.
2554 */
2555 static int
2556 brw_find_loop_end(struct brw_compile *p, int start_offset)
2557 {
2558 struct brw_context *brw = p->brw;
2559 int offset;
2560 int scale = 16 / brw_jump_scale(brw);
2561 void *store = p->store;
2562
2563 assert(brw->gen >= 6);
2564
2565 /* Always start after the instruction (such as a WHILE) we're trying to fix
2566 * up.
2567 */
2568 for (offset = next_offset(brw, store, start_offset);
2569 offset < p->next_insn_offset;
2570 offset = next_offset(brw, store, offset)) {
2571 brw_inst *insn = store + offset;
2572
2573 if (brw_inst_opcode(brw, insn) == BRW_OPCODE_WHILE) {
2574 int jip = brw->gen == 6 ? brw_inst_gen6_jump_count(brw, insn)
2575 : brw_inst_jip(brw, insn);
2576 if (offset + jip * scale <= start_offset)
2577 return offset;
2578 }
2579 }
2580 assert(!"not reached");
2581 return start_offset;
2582 }
2583
2584 /* After program generation, go back and update the UIP and JIP of
2585 * BREAK, CONT, and HALT instructions to their correct locations.
2586 */
2587 void
2588 brw_set_uip_jip(struct brw_compile *p)
2589 {
2590 struct brw_context *brw = p->brw;
2591 int offset;
2592 int br = brw_jump_scale(brw);
2593 int scale = 16 / br;
2594 void *store = p->store;
2595
2596 if (brw->gen < 6)
2597 return;
2598
2599 for (offset = 0; offset < p->next_insn_offset;
2600 offset = next_offset(brw, store, offset)) {
2601 brw_inst *insn = store + offset;
2602
2603 if (brw_inst_cmpt_control(brw, insn)) {
2604 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2605 assert(brw_inst_opcode(brw, insn) != BRW_OPCODE_BREAK &&
2606 brw_inst_opcode(brw, insn) != BRW_OPCODE_CONTINUE &&
2607 brw_inst_opcode(brw, insn) != BRW_OPCODE_HALT);
2608 continue;
2609 }
2610
2611 int block_end_offset = brw_find_next_block_end(p, offset);
2612 switch (brw_inst_opcode(brw, insn)) {
2613 case BRW_OPCODE_BREAK:
2614 assert(block_end_offset != 0);
2615 brw_inst_set_jip(brw, insn, (block_end_offset - offset) / scale);
2616 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2617 brw_inst_set_uip(brw, insn,
2618 (brw_find_loop_end(p, offset) - offset +
2619 (brw->gen == 6 ? 16 : 0)) / scale);
2620 break;
2621 case BRW_OPCODE_CONTINUE:
2622 assert(block_end_offset != 0);
2623 brw_inst_set_jip(brw, insn, (block_end_offset - offset) / scale);
2624 brw_inst_set_uip(brw, insn,
2625 (brw_find_loop_end(p, offset) - offset) / scale);
2626
2627 assert(brw_inst_uip(brw, insn) != 0);
2628 assert(brw_inst_jip(brw, insn) != 0);
2629 break;
2630
2631 case BRW_OPCODE_ENDIF: {
2632 int32_t jump = (block_end_offset == 0) ?
2633 1 * br : (block_end_offset - offset) / scale;
2634 if (brw->gen >= 7)
2635 brw_inst_set_jip(brw, insn, jump);
2636 else
2637 brw_inst_set_gen6_jump_count(brw, insn, jump);
2638 break;
2639 }
2640
2641 case BRW_OPCODE_HALT:
2642 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2643 *
2644 * "In case of the halt instruction not inside any conditional
2645 * code block, the value of <JIP> and <UIP> should be the
2646 * same. In case of the halt instruction inside conditional code
2647 * block, the <UIP> should be the end of the program, and the
2648 * <JIP> should be end of the most inner conditional code block."
2649 *
2650 * The uip will have already been set by whoever set up the
2651 * instruction.
2652 */
2653 if (block_end_offset == 0) {
2654 brw_inst_set_jip(brw, insn, brw_inst_uip(brw, insn));
2655 } else {
2656 brw_inst_set_jip(brw, insn, (block_end_offset - offset) / scale);
2657 }
2658 assert(brw_inst_uip(brw, insn) != 0);
2659 assert(brw_inst_jip(brw, insn) != 0);
2660 break;
2661 }
2662 }
2663 }
2664
2665 void brw_ff_sync(struct brw_compile *p,
2666 struct brw_reg dest,
2667 unsigned msg_reg_nr,
2668 struct brw_reg src0,
2669 bool allocate,
2670 unsigned response_length,
2671 bool eot)
2672 {
2673 struct brw_context *brw = p->brw;
2674 brw_inst *insn;
2675
2676 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2677
2678 insn = next_insn(p, BRW_OPCODE_SEND);
2679 brw_set_dest(p, insn, dest);
2680 brw_set_src0(p, insn, src0);
2681 brw_set_src1(p, insn, brw_imm_d(0));
2682
2683 if (brw->gen < 6)
2684 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
2685
2686 brw_set_ff_sync_message(p,
2687 insn,
2688 allocate,
2689 response_length,
2690 eot);
2691 }
2692
2693 /**
2694 * Emit the SEND instruction necessary to generate stream output data on Gen6
2695 * (for transform feedback).
2696 *
2697 * If send_commit_msg is true, this is the last piece of stream output data
2698 * from this thread, so send the data as a committed write. According to the
2699 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2700 *
2701 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2702 * writes are complete by sending the final write as a committed write."
2703 */
2704 void
2705 brw_svb_write(struct brw_compile *p,
2706 struct brw_reg dest,
2707 unsigned msg_reg_nr,
2708 struct brw_reg src0,
2709 unsigned binding_table_index,
2710 bool send_commit_msg)
2711 {
2712 brw_inst *insn;
2713
2714 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2715
2716 insn = next_insn(p, BRW_OPCODE_SEND);
2717 brw_set_dest(p, insn, dest);
2718 brw_set_src0(p, insn, src0);
2719 brw_set_src1(p, insn, brw_imm_d(0));
2720 brw_set_dp_write_message(p, insn,
2721 binding_table_index,
2722 0, /* msg_control: ignored */
2723 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE,
2724 1, /* msg_length */
2725 true, /* header_present */
2726 0, /* last_render_target: ignored */
2727 send_commit_msg, /* response_length */
2728 0, /* end_of_thread */
2729 send_commit_msg); /* send_commit_msg */
2730 }
2731
2732 static unsigned
2733 brw_surface_payload_size(struct brw_compile *p,
2734 unsigned num_channels,
2735 bool has_simd4x2,
2736 bool has_simd16)
2737 {
2738 if (has_simd4x2 && brw_inst_access_mode(p->brw, p->current) == BRW_ALIGN_16)
2739 return 1;
2740 else if (has_simd16 && p->compressed)
2741 return 2 * num_channels;
2742 else
2743 return num_channels;
2744 }
2745
2746 static void
2747 brw_set_dp_untyped_atomic_message(struct brw_compile *p,
2748 brw_inst *insn,
2749 unsigned atomic_op,
2750 unsigned bind_table_index,
2751 unsigned msg_length,
2752 unsigned response_length,
2753 bool header_present)
2754 {
2755 const struct brw_context *brw = p->brw;
2756
2757 unsigned msg_control =
2758 atomic_op | /* Atomic Operation Type: BRW_AOP_* */
2759 (response_length ? 1 << 5 : 0); /* Return data expected */
2760
2761 if (brw->gen >= 8 || brw->is_haswell) {
2762 brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1,
2763 msg_length, response_length,
2764 header_present, false);
2765
2766
2767 if (brw_inst_access_mode(brw, insn) == BRW_ALIGN_1) {
2768 if (brw_inst_exec_size(brw, insn) != BRW_EXECUTE_16)
2769 msg_control |= 1 << 4; /* SIMD8 mode */
2770
2771 brw_inst_set_dp_msg_type(brw, insn,
2772 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP);
2773 } else {
2774 brw_inst_set_dp_msg_type(brw, insn,
2775 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2);
2776 }
2777 } else {
2778 brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
2779 msg_length, response_length,
2780 header_present, false);
2781
2782 brw_inst_set_dp_msg_type(brw, insn, GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP);
2783
2784 if (brw_inst_exec_size(brw, insn) != BRW_EXECUTE_16)
2785 msg_control |= 1 << 4; /* SIMD8 mode */
2786 }
2787
2788 brw_inst_set_binding_table_index(brw, insn, bind_table_index);
2789 brw_inst_set_dp_msg_control(brw, insn, msg_control);
2790 }
2791
2792 void
2793 brw_untyped_atomic(struct brw_compile *p,
2794 struct brw_reg dest,
2795 struct brw_reg payload,
2796 unsigned atomic_op,
2797 unsigned bind_table_index,
2798 unsigned msg_length,
2799 bool response_expected)
2800 {
2801 const struct brw_context *brw = p->brw;
2802 const bool align1 = brw_inst_access_mode(brw, p->current) == BRW_ALIGN_1;
2803 /* Mask out unused components -- This is especially important in Align16
2804 * mode on generations that don't have native support for SIMD4x2 atomics,
2805 * because unused but enabled components will cause the dataport to perform
2806 * additional atomic operations on the addresses that happen to be in the
2807 * uninitialized Y, Z and W coordinates of the payload.
2808 */
2809 const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X;
2810 brw_inst *insn = brw_next_insn(p, BRW_OPCODE_SEND);
2811
2812 brw_set_dest(p, insn, retype(brw_writemask(dest, mask),
2813 BRW_REGISTER_TYPE_UD));
2814 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UD));
2815 brw_set_src1(p, insn, brw_imm_d(0));
2816 brw_set_dp_untyped_atomic_message(
2817 p, insn, atomic_op, bind_table_index, msg_length,
2818 brw_surface_payload_size(p, response_expected,
2819 brw->gen >= 8 || brw->is_haswell, true),
2820 align1);
2821 }
2822
2823 static void
2824 brw_set_dp_untyped_surface_read_message(struct brw_compile *p,
2825 brw_inst *insn,
2826 unsigned bind_table_index,
2827 unsigned msg_length,
2828 unsigned response_length,
2829 unsigned num_channels,
2830 bool header_present)
2831 {
2832 const struct brw_context *brw = p->brw;
2833 const unsigned dispatch_width =
2834 (brw_inst_exec_size(brw, insn) == BRW_EXECUTE_16 ? 16 : 8);
2835
2836 if (brw->gen >= 8 || brw->is_haswell) {
2837 brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1,
2838 msg_length, response_length,
2839 header_present, false);
2840
2841 brw_inst_set_dp_msg_type(brw, insn,
2842 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ);
2843 } else {
2844 brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
2845 msg_length, response_length,
2846 header_present, false);
2847
2848 brw_inst_set_dp_msg_type(brw, insn,
2849 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ);
2850 }
2851
2852 /* Set mask of 32-bit channels to drop. */
2853 unsigned msg_control = (0xf & (0xf << num_channels));
2854
2855 if (brw_inst_access_mode(brw, insn) == BRW_ALIGN_1) {
2856 if (dispatch_width == 16)
2857 msg_control |= 1 << 4; /* SIMD16 mode */
2858 else
2859 msg_control |= 2 << 4; /* SIMD8 mode */
2860 }
2861
2862 brw_inst_set_binding_table_index(brw, insn, bind_table_index);
2863 brw_inst_set_dp_msg_control(brw, insn, msg_control);
2864 }
2865
2866 void
2867 brw_untyped_surface_read(struct brw_compile *p,
2868 struct brw_reg dest,
2869 struct brw_reg mrf,
2870 unsigned bind_table_index,
2871 unsigned msg_length,
2872 unsigned num_channels)
2873 {
2874 const struct brw_context *brw = p->brw;
2875 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2876
2877 brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD));
2878 brw_set_src0(p, insn, retype(mrf, BRW_REGISTER_TYPE_UD));
2879 brw_set_dp_untyped_surface_read_message(
2880 p, insn, bind_table_index, msg_length,
2881 brw_surface_payload_size(p, num_channels, true, true),
2882 num_channels, brw_inst_access_mode(brw, insn) == BRW_ALIGN_1);
2883 }
2884
2885 void
2886 brw_pixel_interpolator_query(struct brw_compile *p,
2887 struct brw_reg dest,
2888 struct brw_reg mrf,
2889 bool noperspective,
2890 unsigned mode,
2891 unsigned data,
2892 unsigned msg_length,
2893 unsigned response_length)
2894 {
2895 const struct brw_context *brw = p->brw;
2896 struct brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2897
2898 brw_set_dest(p, insn, dest);
2899 brw_set_src0(p, insn, mrf);
2900 brw_set_message_descriptor(p, insn, GEN7_SFID_PIXEL_INTERPOLATOR,
2901 msg_length, response_length,
2902 false /* header is never present for PI */,
2903 false);
2904
2905 brw_inst_set_pi_simd_mode(
2906 brw, insn, brw_inst_exec_size(brw, insn) == BRW_EXECUTE_16);
2907 brw_inst_set_pi_slot_group(brw, insn, 0); /* zero unless 32/64px dispatch */
2908 brw_inst_set_pi_nopersp(brw, insn, noperspective);
2909 brw_inst_set_pi_message_type(brw, insn, mode);
2910 brw_inst_set_pi_message_data(brw, insn, data);
2911 }
2912
2913 /**
2914 * This instruction is generated as a single-channel align1 instruction by
2915 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
2916 *
2917 * We can't use the typed atomic op in the FS because that has the execution
2918 * mask ANDed with the pixel mask, but we just want to write the one dword for
2919 * all the pixels.
2920 *
2921 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
2922 * one u32. So we use the same untyped atomic write message as the pixel
2923 * shader.
2924 *
2925 * The untyped atomic operation requires a BUFFER surface type with RAW
2926 * format, and is only accessible through the legacy DATA_CACHE dataport
2927 * messages.
2928 */
2929 void brw_shader_time_add(struct brw_compile *p,
2930 struct brw_reg payload,
2931 uint32_t surf_index)
2932 {
2933 assert(p->brw->gen >= 7);
2934
2935 brw_push_insn_state(p);
2936 brw_set_default_access_mode(p, BRW_ALIGN_1);
2937 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2938 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
2939 brw_pop_insn_state(p);
2940
2941 /* We use brw_vec1_reg and unmasked because we want to increment the given
2942 * offset only once.
2943 */
2944 brw_set_dest(p, send, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE,
2945 BRW_ARF_NULL, 0));
2946 brw_set_src0(p, send, brw_vec1_reg(payload.file,
2947 payload.nr, 0));
2948 brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, surf_index,
2949 2 /* message length */,
2950 0 /* response length */,
2951 false /* header present */);
2952 }