projects / mesa.git / blob
? search:


782706a82af5271263efcf689e05e72063a05a2a
[mesa.git] / src / mesa / drivers / dri / i965 / brw_eu_emit.c
1 /*
2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
5
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
13
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
17
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
26 **********************************************************************/
27 /*
28 * Authors:
29 * Keith Whitwell <keithw@vmware.com>
30 */
31
32
33 #include "brw_context.h"
34 #include "brw_defines.h"
35 #include "brw_eu.h"
36
37 #include "util/ralloc.h"
38
39 /***********************************************************************
40 * Internal helper for constructing instructions
41 */
42
43 static void guess_execution_size(struct brw_compile *p,
44 brw_inst *insn,
45 struct brw_reg reg)
46 {
47 const struct brw_context *brw = p->brw;
48
49 if (reg.width == BRW_WIDTH_8 && p->compressed) {
50 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_16);
51 } else {
52 /* Register width definitions are compatible with BRW_EXECUTE_* enums. */
53 brw_inst_set_exec_size(brw, insn, reg.width);
54 }
55 }
56
57
58 /**
59 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
60 * registers, implicitly moving the operand to a message register.
61 *
62 * On Sandybridge, this is no longer the case. This function performs the
63 * explicit move; it should be called before emitting a SEND instruction.
64 */
65 void
66 gen6_resolve_implied_move(struct brw_compile *p,
67 struct brw_reg *src,
68 unsigned msg_reg_nr)
69 {
70 struct brw_context *brw = p->brw;
71 if (brw->gen < 6)
72 return;
73
74 if (src->file == BRW_MESSAGE_REGISTER_FILE)
75 return;
76
77 if (src->file != BRW_ARCHITECTURE_REGISTER_FILE || src->nr != BRW_ARF_NULL) {
78 brw_push_insn_state(p);
79 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
80 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
81 brw_MOV(p, retype(brw_message_reg(msg_reg_nr), BRW_REGISTER_TYPE_UD),
82 retype(*src, BRW_REGISTER_TYPE_UD));
83 brw_pop_insn_state(p);
84 }
85 *src = brw_message_reg(msg_reg_nr);
86 }
87
88 static void
89 gen7_convert_mrf_to_grf(struct brw_compile *p, struct brw_reg *reg)
90 {
91 /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
92 * "The send with EOT should use register space R112-R127 for <src>. This is
93 * to enable loading of a new thread into the same slot while the message
94 * with EOT for current thread is pending dispatch."
95 *
96 * Since we're pretending to have 16 MRFs anyway, we may as well use the
97 * registers required for messages with EOT.
98 */
99 struct brw_context *brw = p->brw;
100 if (brw->gen >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) {
101 reg->file = BRW_GENERAL_REGISTER_FILE;
102 reg->nr += GEN7_MRF_HACK_START;
103 }
104 }
105
106 /**
107 * Convert a brw_reg_type enumeration value into the hardware representation.
108 *
109 * The hardware encoding may depend on whether the value is an immediate.
110 */
111 unsigned
112 brw_reg_type_to_hw_type(const struct brw_context *brw,
113 enum brw_reg_type type, unsigned file)
114 {
115 if (file == BRW_IMMEDIATE_VALUE) {
116 const static int imm_hw_types[] = {
117 [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
118 [BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D,
119 [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
120 [BRW_REGISTER_TYPE_W] = BRW_HW_REG_TYPE_W,
121 [BRW_REGISTER_TYPE_F] = BRW_HW_REG_TYPE_F,
122 [BRW_REGISTER_TYPE_UB] = -1,
123 [BRW_REGISTER_TYPE_B] = -1,
124 [BRW_REGISTER_TYPE_UV] = BRW_HW_REG_IMM_TYPE_UV,
125 [BRW_REGISTER_TYPE_VF] = BRW_HW_REG_IMM_TYPE_VF,
126 [BRW_REGISTER_TYPE_V] = BRW_HW_REG_IMM_TYPE_V,
127 [BRW_REGISTER_TYPE_DF] = GEN8_HW_REG_IMM_TYPE_DF,
128 [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_IMM_TYPE_HF,
129 [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ,
130 [BRW_REGISTER_TYPE_Q] = GEN8_HW_REG_TYPE_Q,
131 };
132 assert(type < ARRAY_SIZE(imm_hw_types));
133 assert(imm_hw_types[type] != -1);
134 assert(brw->gen >= 8 || type < BRW_REGISTER_TYPE_DF);
135 return imm_hw_types[type];
136 } else {
137 /* Non-immediate registers */
138 const static int hw_types[] = {
139 [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
140 [BRW_REGISTER_TYPE_D] = BRW_HW_REG_TYPE_D,
141 [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
142 [BRW_REGISTER_TYPE_W] = BRW_HW_REG_TYPE_W,
143 [BRW_REGISTER_TYPE_UB] = BRW_HW_REG_NON_IMM_TYPE_UB,
144 [BRW_REGISTER_TYPE_B] = BRW_HW_REG_NON_IMM_TYPE_B,
145 [BRW_REGISTER_TYPE_F] = BRW_HW_REG_TYPE_F,
146 [BRW_REGISTER_TYPE_UV] = -1,
147 [BRW_REGISTER_TYPE_VF] = -1,
148 [BRW_REGISTER_TYPE_V] = -1,
149 [BRW_REGISTER_TYPE_DF] = GEN7_HW_REG_NON_IMM_TYPE_DF,
150 [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_NON_IMM_TYPE_HF,
151 [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ,
152 [BRW_REGISTER_TYPE_Q] = GEN8_HW_REG_TYPE_Q,
153 };
154 assert(type < ARRAY_SIZE(hw_types));
155 assert(hw_types[type] != -1);
156 assert(brw->gen >= 7 || type < BRW_REGISTER_TYPE_DF);
157 assert(brw->gen >= 8 || type < BRW_REGISTER_TYPE_HF);
158 return hw_types[type];
159 }
160 }
161
162 void
163 brw_set_dest(struct brw_compile *p, brw_inst *inst, struct brw_reg dest)
164 {
165 const struct brw_context *brw = p->brw;
166
167 if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE &&
168 dest.file != BRW_MESSAGE_REGISTER_FILE)
169 assert(dest.nr < 128);
170
171 gen7_convert_mrf_to_grf(p, &dest);
172
173 brw_inst_set_dst_reg_file(brw, inst, dest.file);
174 brw_inst_set_dst_reg_type(brw, inst, brw_reg_type_to_hw_type(brw, dest.type,
175 dest.file));
176 brw_inst_set_dst_address_mode(brw, inst, dest.address_mode);
177
178 if (dest.address_mode == BRW_ADDRESS_DIRECT) {
179 brw_inst_set_dst_da_reg_nr(brw, inst, dest.nr);
180
181 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
182 brw_inst_set_dst_da1_subreg_nr(brw, inst, dest.subnr);
183 if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
184 dest.hstride = BRW_HORIZONTAL_STRIDE_1;
185 brw_inst_set_dst_hstride(brw, inst, dest.hstride);
186 } else {
187 brw_inst_set_dst_da16_subreg_nr(brw, inst, dest.subnr / 16);
188 brw_inst_set_da16_writemask(brw, inst, dest.dw1.bits.writemask);
189 if (dest.file == BRW_GENERAL_REGISTER_FILE ||
190 dest.file == BRW_MESSAGE_REGISTER_FILE) {
191 assert(dest.dw1.bits.writemask != 0);
192 }
193 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
194 * Although Dst.HorzStride is a don't care for Align16, HW needs
195 * this to be programmed as "01".
196 */
197 brw_inst_set_dst_hstride(brw, inst, 1);
198 }
199 } else {
200 brw_inst_set_dst_ia_subreg_nr(brw, inst, dest.subnr);
201
202 /* These are different sizes in align1 vs align16:
203 */
204 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
205 brw_inst_set_dst_ia1_addr_imm(brw, inst,
206 dest.dw1.bits.indirect_offset);
207 if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
208 dest.hstride = BRW_HORIZONTAL_STRIDE_1;
209 brw_inst_set_dst_hstride(brw, inst, dest.hstride);
210 } else {
211 brw_inst_set_dst_ia16_addr_imm(brw, inst,
212 dest.dw1.bits.indirect_offset);
213 /* even ignored in da16, still need to set as '01' */
214 brw_inst_set_dst_hstride(brw, inst, 1);
215 }
216 }
217
218 /* NEW: Set the execution size based on dest.width and
219 * inst->compression_control:
220 */
221 guess_execution_size(p, inst, dest);
222 }
223
224 extern int reg_type_size[];
225
226 static void
227 validate_reg(const struct brw_context *brw, brw_inst *inst, struct brw_reg reg)
228 {
229 int hstride_for_reg[] = {0, 1, 2, 4};
230 int vstride_for_reg[] = {0, 1, 2, 4, 8, 16, 32};
231 int width_for_reg[] = {1, 2, 4, 8, 16};
232 int execsize_for_reg[] = {1, 2, 4, 8, 16};
233 int width, hstride, vstride, execsize;
234
235 if (reg.file == BRW_IMMEDIATE_VALUE) {
236 /* 3.3.6: Region Parameters. Restriction: Immediate vectors
237 * mean the destination has to be 128-bit aligned and the
238 * destination horiz stride has to be a word.
239 */
240 if (reg.type == BRW_REGISTER_TYPE_V) {
241 assert(hstride_for_reg[brw_inst_dst_hstride(brw, inst)] *
242 reg_type_size[brw_inst_dst_reg_type(brw, inst)] == 2);
243 }
244
245 return;
246 }
247
248 if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE &&
249 reg.file == BRW_ARF_NULL)
250 return;
251
252 assert(reg.hstride >= 0 && reg.hstride < Elements(hstride_for_reg));
253 hstride = hstride_for_reg[reg.hstride];
254
255 if (reg.vstride == 0xf) {
256 vstride = -1;
257 } else {
258 assert(reg.vstride >= 0 && reg.vstride < Elements(vstride_for_reg));
259 vstride = vstride_for_reg[reg.vstride];
260 }
261
262 assert(reg.width >= 0 && reg.width < Elements(width_for_reg));
263 width = width_for_reg[reg.width];
264
265 assert(brw_inst_exec_size(brw, inst) >= 0 &&
266 brw_inst_exec_size(brw, inst) < Elements(execsize_for_reg));
267 execsize = execsize_for_reg[brw_inst_exec_size(brw, inst)];
268
269 /* Restrictions from 3.3.10: Register Region Restrictions. */
270 /* 3. */
271 assert(execsize >= width);
272
273 /* 4. */
274 if (execsize == width && hstride != 0) {
275 assert(vstride == -1 || vstride == width * hstride);
276 }
277
278 /* 5. */
279 if (execsize == width && hstride == 0) {
280 /* no restriction on vstride. */
281 }
282
283 /* 6. */
284 if (width == 1) {
285 assert(hstride == 0);
286 }
287
288 /* 7. */
289 if (execsize == 1 && width == 1) {
290 assert(hstride == 0);
291 assert(vstride == 0);
292 }
293
294 /* 8. */
295 if (vstride == 0 && hstride == 0) {
296 assert(width == 1);
297 }
298
299 /* 10. Check destination issues. */
300 }
301
302 static bool
303 is_compactable_immediate(unsigned imm)
304 {
305 /* We get the low 12 bits as-is. */
306 imm &= ~0xfff;
307
308 /* We get one bit replicated through the top 20 bits. */
309 return imm == 0 || imm == 0xfffff000;
310 }
311
312 void
313 brw_set_src0(struct brw_compile *p, brw_inst *inst, struct brw_reg reg)
314 {
315 struct brw_context *brw = p->brw;
316
317 if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
318 assert(reg.nr < 128);
319
320 gen7_convert_mrf_to_grf(p, &reg);
321
322 if (brw->gen >= 6 && (brw_inst_opcode(brw, inst) == BRW_OPCODE_SEND ||
323 brw_inst_opcode(brw, inst) == BRW_OPCODE_SENDC)) {
324 /* Any source modifiers or regions will be ignored, since this just
325 * identifies the MRF/GRF to start reading the message contents from.
326 * Check for some likely failures.
327 */
328 assert(!reg.negate);
329 assert(!reg.abs);
330 assert(reg.address_mode == BRW_ADDRESS_DIRECT);
331 }
332
333 validate_reg(brw, inst, reg);
334
335 brw_inst_set_src0_reg_file(brw, inst, reg.file);
336 brw_inst_set_src0_reg_type(brw, inst,
337 brw_reg_type_to_hw_type(brw, reg.type, reg.file));
338 brw_inst_set_src0_abs(brw, inst, reg.abs);
339 brw_inst_set_src0_negate(brw, inst, reg.negate);
340 brw_inst_set_src0_address_mode(brw, inst, reg.address_mode);
341
342 if (reg.file == BRW_IMMEDIATE_VALUE) {
343 brw_inst_set_imm_ud(brw, inst, reg.dw1.ud);
344
345 /* The Bspec's section titled "Non-present Operands" claims that if src0
346 * is an immediate that src1's type must be the same as that of src0.
347 *
348 * The SNB+ DataTypeIndex instruction compaction tables contain mappings
349 * that do not follow this rule. E.g., from the IVB/HSW table:
350 *
351 * DataTypeIndex 18-Bit Mapping Mapped Meaning
352 * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir |
353 *
354 * And from the SNB table:
355 *
356 * DataTypeIndex 18-Bit Mapping Mapped Meaning
357 * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir |
358 *
359 * Neither of these cause warnings from the simulator when used,
360 * compacted or otherwise. In fact, all compaction mappings that have an
361 * immediate in src0 use a:ud for src1.
362 *
363 * The GM45 instruction compaction tables do not contain mapped meanings
364 * so it's not clear whether it has the restriction. We'll assume it was
365 * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
366 */
367 brw_inst_set_src1_reg_file(brw, inst, BRW_ARCHITECTURE_REGISTER_FILE);
368 if (brw->gen < 6) {
369 brw_inst_set_src1_reg_type(brw, inst,
370 brw_inst_src0_reg_type(brw, inst));
371 } else {
372 brw_inst_set_src1_reg_type(brw, inst, BRW_HW_REG_TYPE_UD);
373 }
374
375 /* Compacted instructions only have 12-bits (plus 1 for the other 20)
376 * for immediate values. Presumably the hardware engineers realized
377 * that the only useful floating-point value that could be represented
378 * in this format is 0.0, which can also be represented as a VF-typed
379 * immediate, so they gave us the previously mentioned mapping on IVB+.
380 *
381 * Strangely, we do have a mapping for imm:f in src1, so we don't need
382 * to do this there.
383 *
384 * If we see a 0.0:F, change the type to VF so that it can be compacted.
385 */
386 if (brw_inst_imm_ud(brw, inst) == 0x0 &&
387 brw_inst_src0_reg_type(brw, inst) == BRW_HW_REG_TYPE_F) {
388 brw_inst_set_src0_reg_type(brw, inst, BRW_HW_REG_IMM_TYPE_VF);
389 }
390
391 /* There are no mappings for dst:d | i:d, so if the immediate is suitable
392 * set the types to :UD so the instruction can be compacted.
393 */
394 if (is_compactable_immediate(brw_inst_imm_ud(brw, inst)) &&
395 brw_inst_cond_modifier(brw, inst) == BRW_CONDITIONAL_NONE &&
396 brw_inst_src0_reg_type(brw, inst) == BRW_HW_REG_TYPE_D &&
397 brw_inst_dst_reg_type(brw, inst) == BRW_HW_REG_TYPE_D) {
398 brw_inst_set_src0_reg_type(brw, inst, BRW_HW_REG_TYPE_UD);
399 brw_inst_set_dst_reg_type(brw, inst, BRW_HW_REG_TYPE_UD);
400 }
401 } else {
402 if (reg.address_mode == BRW_ADDRESS_DIRECT) {
403 brw_inst_set_src0_da_reg_nr(brw, inst, reg.nr);
404 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
405 brw_inst_set_src0_da1_subreg_nr(brw, inst, reg.subnr);
406 } else {
407 brw_inst_set_src0_da16_subreg_nr(brw, inst, reg.subnr / 16);
408 }
409 } else {
410 brw_inst_set_src0_ia_subreg_nr(brw, inst, reg.subnr);
411
412 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
413 brw_inst_set_src0_ia1_addr_imm(brw, inst, reg.dw1.bits.indirect_offset);
414 } else {
415 brw_inst_set_src0_ia_subreg_nr(brw, inst, reg.dw1.bits.indirect_offset);
416 }
417 }
418
419 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
420 if (reg.width == BRW_WIDTH_1 &&
421 brw_inst_exec_size(brw, inst) == BRW_EXECUTE_1) {
422 brw_inst_set_src0_hstride(brw, inst, BRW_HORIZONTAL_STRIDE_0);
423 brw_inst_set_src0_width(brw, inst, BRW_WIDTH_1);
424 brw_inst_set_src0_vstride(brw, inst, BRW_VERTICAL_STRIDE_0);
425 } else {
426 brw_inst_set_src0_hstride(brw, inst, reg.hstride);
427 brw_inst_set_src0_width(brw, inst, reg.width);
428 brw_inst_set_src0_vstride(brw, inst, reg.vstride);
429 }
430 } else {
431 brw_inst_set_src0_da16_swiz_x(brw, inst,
432 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X));
433 brw_inst_set_src0_da16_swiz_y(brw, inst,
434 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y));
435 brw_inst_set_src0_da16_swiz_z(brw, inst,
436 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z));
437 brw_inst_set_src0_da16_swiz_w(brw, inst,
438 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W));
439
440 /* This is an oddity of the fact we're using the same
441 * descriptions for registers in align_16 as align_1:
442 */
443 if (reg.vstride == BRW_VERTICAL_STRIDE_8)
444 brw_inst_set_src0_vstride(brw, inst, BRW_VERTICAL_STRIDE_4);
445 else
446 brw_inst_set_src0_vstride(brw, inst, reg.vstride);
447 }
448 }
449 }
450
451
452 void
453 brw_set_src1(struct brw_compile *p, brw_inst *inst, struct brw_reg reg)
454 {
455 const struct brw_context *brw = p->brw;
456 assert(reg.file != BRW_MESSAGE_REGISTER_FILE);
457
458 if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
459 assert(reg.nr < 128);
460
461 gen7_convert_mrf_to_grf(p, &reg);
462
463 validate_reg(brw, inst, reg);
464
465 brw_inst_set_src1_reg_file(brw, inst, reg.file);
466 brw_inst_set_src1_reg_type(brw, inst,
467 brw_reg_type_to_hw_type(brw, reg.type, reg.file));
468 brw_inst_set_src1_abs(brw, inst, reg.abs);
469 brw_inst_set_src1_negate(brw, inst, reg.negate);
470
471 /* Only src1 can be immediate in two-argument instructions.
472 */
473 assert(brw_inst_src0_reg_file(brw, inst) != BRW_IMMEDIATE_VALUE);
474
475 if (reg.file == BRW_IMMEDIATE_VALUE) {
476 brw_inst_set_imm_ud(brw, inst, reg.dw1.ud);
477 } else {
478 /* This is a hardware restriction, which may or may not be lifted
479 * in the future:
480 */
481 assert (reg.address_mode == BRW_ADDRESS_DIRECT);
482 /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
483
484 brw_inst_set_src1_da_reg_nr(brw, inst, reg.nr);
485 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
486 brw_inst_set_src1_da1_subreg_nr(brw, inst, reg.subnr);
487 } else {
488 brw_inst_set_src1_da16_subreg_nr(brw, inst, reg.subnr / 16);
489 }
490
491 if (brw_inst_access_mode(brw, inst) == BRW_ALIGN_1) {
492 if (reg.width == BRW_WIDTH_1 &&
493 brw_inst_exec_size(brw, inst) == BRW_EXECUTE_1) {
494 brw_inst_set_src1_hstride(brw, inst, BRW_HORIZONTAL_STRIDE_0);
495 brw_inst_set_src1_width(brw, inst, BRW_WIDTH_1);
496 brw_inst_set_src1_vstride(brw, inst, BRW_VERTICAL_STRIDE_0);
497 } else {
498 brw_inst_set_src1_hstride(brw, inst, reg.hstride);
499 brw_inst_set_src1_width(brw, inst, reg.width);
500 brw_inst_set_src1_vstride(brw, inst, reg.vstride);
501 }
502 } else {
503 brw_inst_set_src1_da16_swiz_x(brw, inst,
504 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_X));
505 brw_inst_set_src1_da16_swiz_y(brw, inst,
506 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Y));
507 brw_inst_set_src1_da16_swiz_z(brw, inst,
508 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_Z));
509 brw_inst_set_src1_da16_swiz_w(brw, inst,
510 BRW_GET_SWZ(reg.dw1.bits.swizzle, BRW_CHANNEL_W));
511
512 /* This is an oddity of the fact we're using the same
513 * descriptions for registers in align_16 as align_1:
514 */
515 if (reg.vstride == BRW_VERTICAL_STRIDE_8)
516 brw_inst_set_src1_vstride(brw, inst, BRW_VERTICAL_STRIDE_4);
517 else
518 brw_inst_set_src1_vstride(brw, inst, reg.vstride);
519 }
520 }
521 }
522
523 /**
524 * Set the Message Descriptor and Extended Message Descriptor fields
525 * for SEND messages.
526 *
527 * \note This zeroes out the Function Control bits, so it must be called
528 * \b before filling out any message-specific data. Callers can
529 * choose not to fill in irrelevant bits; they will be zero.
530 */
531 static void
532 brw_set_message_descriptor(struct brw_compile *p,
533 brw_inst *inst,
534 enum brw_message_target sfid,
535 unsigned msg_length,
536 unsigned response_length,
537 bool header_present,
538 bool end_of_thread)
539 {
540 struct brw_context *brw = p->brw;
541
542 brw_set_src1(p, inst, brw_imm_d(0));
543
544 /* For indirect sends, `inst` will not be the SEND/SENDC instruction
545 * itself; instead, it will be a MOV/OR into the address register.
546 *
547 * In this case, we avoid setting the extended message descriptor bits,
548 * since they go on the later SEND/SENDC instead and if set here would
549 * instead clobber the conditionalmod bits.
550 */
551 unsigned opcode = brw_inst_opcode(brw, inst);
552 if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC) {
553 brw_inst_set_sfid(brw, inst, sfid);
554 }
555
556 brw_inst_set_mlen(brw, inst, msg_length);
557 brw_inst_set_rlen(brw, inst, response_length);
558 brw_inst_set_eot(brw, inst, end_of_thread);
559
560 if (brw->gen >= 5) {
561 brw_inst_set_header_present(brw, inst, header_present);
562 }
563 }
564
565 static void brw_set_math_message( struct brw_compile *p,
566 brw_inst *inst,
567 unsigned function,
568 unsigned integer_type,
569 bool low_precision,
570 unsigned dataType )
571 {
572 struct brw_context *brw = p->brw;
573 unsigned msg_length;
574 unsigned response_length;
575
576 /* Infer message length from the function */
577 switch (function) {
578 case BRW_MATH_FUNCTION_POW:
579 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT:
580 case BRW_MATH_FUNCTION_INT_DIV_REMAINDER:
581 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
582 msg_length = 2;
583 break;
584 default:
585 msg_length = 1;
586 break;
587 }
588
589 /* Infer response length from the function */
590 switch (function) {
591 case BRW_MATH_FUNCTION_SINCOS:
592 case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
593 response_length = 2;
594 break;
595 default:
596 response_length = 1;
597 break;
598 }
599
600
601 brw_set_message_descriptor(p, inst, BRW_SFID_MATH,
602 msg_length, response_length, false, false);
603 brw_inst_set_math_msg_function(brw, inst, function);
604 brw_inst_set_math_msg_signed_int(brw, inst, integer_type);
605 brw_inst_set_math_msg_precision(brw, inst, low_precision);
606 brw_inst_set_math_msg_saturate(brw, inst, brw_inst_saturate(brw, inst));
607 brw_inst_set_math_msg_data_type(brw, inst, dataType);
608 brw_inst_set_saturate(brw, inst, 0);
609 }
610
611
612 static void brw_set_ff_sync_message(struct brw_compile *p,
613 brw_inst *insn,
614 bool allocate,
615 unsigned response_length,
616 bool end_of_thread)
617 {
618 const struct brw_context *brw = p->brw;
619
620 brw_set_message_descriptor(p, insn, BRW_SFID_URB,
621 1, response_length, true, end_of_thread);
622 brw_inst_set_urb_opcode(brw, insn, 1); /* FF_SYNC */
623 brw_inst_set_urb_allocate(brw, insn, allocate);
624 /* The following fields are not used by FF_SYNC: */
625 brw_inst_set_urb_global_offset(brw, insn, 0);
626 brw_inst_set_urb_swizzle_control(brw, insn, 0);
627 brw_inst_set_urb_used(brw, insn, 0);
628 brw_inst_set_urb_complete(brw, insn, 0);
629 }
630
631 static void brw_set_urb_message( struct brw_compile *p,
632 brw_inst *insn,
633 enum brw_urb_write_flags flags,
634 unsigned msg_length,
635 unsigned response_length,
636 unsigned offset,
637 unsigned swizzle_control )
638 {
639 struct brw_context *brw = p->brw;
640
641 assert(brw->gen < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE);
642 assert(brw->gen < 7 || !(flags & BRW_URB_WRITE_ALLOCATE));
643 assert(brw->gen >= 7 || !(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
644
645 brw_set_message_descriptor(p, insn, BRW_SFID_URB,
646 msg_length, response_length, true,
647 flags & BRW_URB_WRITE_EOT);
648
649 if (flags & BRW_URB_WRITE_OWORD) {
650 assert(msg_length == 2); /* header + one OWORD of data */
651 brw_inst_set_urb_opcode(brw, insn, BRW_URB_OPCODE_WRITE_OWORD);
652 } else {
653 brw_inst_set_urb_opcode(brw, insn, BRW_URB_OPCODE_WRITE_HWORD);
654 }
655
656 brw_inst_set_urb_global_offset(brw, insn, offset);
657 brw_inst_set_urb_swizzle_control(brw, insn, swizzle_control);
658
659 if (brw->gen < 8) {
660 brw_inst_set_urb_complete(brw, insn, !!(flags & BRW_URB_WRITE_COMPLETE));
661 }
662
663 if (brw->gen < 7) {
664 brw_inst_set_urb_allocate(brw, insn, !!(flags & BRW_URB_WRITE_ALLOCATE));
665 brw_inst_set_urb_used(brw, insn, !(flags & BRW_URB_WRITE_UNUSED));
666 } else {
667 brw_inst_set_urb_per_slot_offset(brw, insn,
668 !!(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
669 }
670 }
671
672 void
673 brw_set_dp_write_message(struct brw_compile *p,
674 brw_inst *insn,
675 unsigned binding_table_index,
676 unsigned msg_control,
677 unsigned msg_type,
678 unsigned msg_length,
679 bool header_present,
680 unsigned last_render_target,
681 unsigned response_length,
682 unsigned end_of_thread,
683 unsigned send_commit_msg)
684 {
685 struct brw_context *brw = p->brw;
686 unsigned sfid;
687
688 if (brw->gen >= 7) {
689 /* Use the Render Cache for RT writes; otherwise use the Data Cache */
690 if (msg_type == GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE)
691 sfid = GEN6_SFID_DATAPORT_RENDER_CACHE;
692 else
693 sfid = GEN7_SFID_DATAPORT_DATA_CACHE;
694 } else if (brw->gen == 6) {
695 /* Use the render cache for all write messages. */
696 sfid = GEN6_SFID_DATAPORT_RENDER_CACHE;
697 } else {
698 sfid = BRW_SFID_DATAPORT_WRITE;
699 }
700
701 brw_set_message_descriptor(p, insn, sfid, msg_length, response_length,
702 header_present, end_of_thread);
703
704 brw_inst_set_binding_table_index(brw, insn, binding_table_index);
705 brw_inst_set_dp_write_msg_type(brw, insn, msg_type);
706 brw_inst_set_dp_write_msg_control(brw, insn, msg_control);
707 brw_inst_set_rt_last(brw, insn, last_render_target);
708 if (brw->gen < 7) {
709 brw_inst_set_dp_write_commit(brw, insn, send_commit_msg);
710 }
711 }
712
713 void
714 brw_set_dp_read_message(struct brw_compile *p,
715 brw_inst *insn,
716 unsigned binding_table_index,
717 unsigned msg_control,
718 unsigned msg_type,
719 unsigned target_cache,
720 unsigned msg_length,
721 bool header_present,
722 unsigned response_length)
723 {
724 struct brw_context *brw = p->brw;
725 unsigned sfid;
726
727 if (brw->gen >= 7) {
728 sfid = GEN7_SFID_DATAPORT_DATA_CACHE;
729 } else if (brw->gen == 6) {
730 if (target_cache == BRW_DATAPORT_READ_TARGET_RENDER_CACHE)
731 sfid = GEN6_SFID_DATAPORT_RENDER_CACHE;
732 else
733 sfid = GEN6_SFID_DATAPORT_SAMPLER_CACHE;
734 } else {
735 sfid = BRW_SFID_DATAPORT_READ;
736 }
737
738 brw_set_message_descriptor(p, insn, sfid, msg_length, response_length,
739 header_present, false);
740
741 brw_inst_set_binding_table_index(brw, insn, binding_table_index);
742 brw_inst_set_dp_read_msg_type(brw, insn, msg_type);
743 brw_inst_set_dp_read_msg_control(brw, insn, msg_control);
744 if (brw->gen < 6)
745 brw_inst_set_dp_read_target_cache(brw, insn, target_cache);
746 }
747
748 void
749 brw_set_sampler_message(struct brw_compile *p,
750 brw_inst *inst,
751 unsigned binding_table_index,
752 unsigned sampler,
753 unsigned msg_type,
754 unsigned response_length,
755 unsigned msg_length,
756 unsigned header_present,
757 unsigned simd_mode,
758 unsigned return_format)
759 {
760 struct brw_context *brw = p->brw;
761
762 brw_set_message_descriptor(p, inst, BRW_SFID_SAMPLER, msg_length,
763 response_length, header_present, false);
764
765 brw_inst_set_binding_table_index(brw, inst, binding_table_index);
766 brw_inst_set_sampler(brw, inst, sampler);
767 brw_inst_set_sampler_msg_type(brw, inst, msg_type);
768 if (brw->gen >= 5) {
769 brw_inst_set_sampler_simd_mode(brw, inst, simd_mode);
770 } else if (brw->gen == 4 && !brw->is_g4x) {
771 brw_inst_set_sampler_return_format(brw, inst, return_format);
772 }
773 }
774
775 void brw_set_indirect_send_descriptor(struct brw_compile *p,
776 brw_inst *insn,
777 unsigned sfid,
778 struct brw_reg descriptor)
779 {
780 /* Only a0.0 may be used as SEND's descriptor operand. */
781 assert(descriptor.file == BRW_ARCHITECTURE_REGISTER_FILE);
782 assert(descriptor.type == BRW_REGISTER_TYPE_UD);
783 assert(descriptor.nr == BRW_ARF_ADDRESS);
784 assert(descriptor.subnr == 0);
785
786 brw_set_message_descriptor(p, insn, sfid, 0, 0, false, false);
787 brw_set_src1(p, insn, descriptor);
788 }
789
790 static void
791 gen7_set_dp_scratch_message(struct brw_compile *p,
792 brw_inst *inst,
793 bool write,
794 bool dword,
795 bool invalidate_after_read,
796 unsigned num_regs,
797 unsigned addr_offset,
798 unsigned mlen,
799 unsigned rlen,
800 bool header_present)
801 {
802 const struct brw_context *brw = p->brw;
803 assert(num_regs == 1 || num_regs == 2 || num_regs == 4 ||
804 (brw->gen >= 8 && num_regs == 8));
805 brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE,
806 mlen, rlen, header_present, false);
807 brw_inst_set_dp_category(brw, inst, 1); /* Scratch Block Read/Write msgs */
808 brw_inst_set_scratch_read_write(brw, inst, write);
809 brw_inst_set_scratch_type(brw, inst, dword);
810 brw_inst_set_scratch_invalidate_after_read(brw, inst, invalidate_after_read);
811 brw_inst_set_scratch_block_size(brw, inst, ffs(num_regs) - 1);
812 brw_inst_set_scratch_addr_offset(brw, inst, addr_offset);
813 }
814
815 #define next_insn brw_next_insn
816 brw_inst *
817 brw_next_insn(struct brw_compile *p, unsigned opcode)
818 {
819 const struct brw_context *brw = p->brw;
820 brw_inst *insn;
821
822 if (p->nr_insn + 1 > p->store_size) {
823 p->store_size <<= 1;
824 p->store = reralloc(p->mem_ctx, p->store, brw_inst, p->store_size);
825 }
826
827 p->next_insn_offset += 16;
828 insn = &p->store[p->nr_insn++];
829 memcpy(insn, p->current, sizeof(*insn));
830
831 brw_inst_set_opcode(brw, insn, opcode);
832 return insn;
833 }
834
835 static brw_inst *
836 brw_alu1(struct brw_compile *p, unsigned opcode,
837 struct brw_reg dest, struct brw_reg src)
838 {
839 brw_inst *insn = next_insn(p, opcode);
840 brw_set_dest(p, insn, dest);
841 brw_set_src0(p, insn, src);
842 return insn;
843 }
844
845 static brw_inst *
846 brw_alu2(struct brw_compile *p, unsigned opcode,
847 struct brw_reg dest, struct brw_reg src0, struct brw_reg src1)
848 {
849 brw_inst *insn = next_insn(p, opcode);
850 brw_set_dest(p, insn, dest);
851 brw_set_src0(p, insn, src0);
852 brw_set_src1(p, insn, src1);
853 return insn;
854 }
855
856 static int
857 get_3src_subreg_nr(struct brw_reg reg)
858 {
859 if (reg.vstride == BRW_VERTICAL_STRIDE_0) {
860 assert(brw_is_single_value_swizzle(reg.dw1.bits.swizzle));
861 return reg.subnr / 4 + BRW_GET_SWZ(reg.dw1.bits.swizzle, 0);
862 } else {
863 return reg.subnr / 4;
864 }
865 }
866
867 static brw_inst *
868 brw_alu3(struct brw_compile *p, unsigned opcode, struct brw_reg dest,
869 struct brw_reg src0, struct brw_reg src1, struct brw_reg src2)
870 {
871 struct brw_context *brw = p->brw;
872 brw_inst *inst = next_insn(p, opcode);
873
874 gen7_convert_mrf_to_grf(p, &dest);
875
876 assert(brw_inst_access_mode(brw, inst) == BRW_ALIGN_16);
877
878 assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
879 dest.file == BRW_MESSAGE_REGISTER_FILE);
880 assert(dest.nr < 128);
881 assert(dest.address_mode == BRW_ADDRESS_DIRECT);
882 assert(dest.type == BRW_REGISTER_TYPE_F ||
883 dest.type == BRW_REGISTER_TYPE_D ||
884 dest.type == BRW_REGISTER_TYPE_UD);
885 if (brw->gen == 6) {
886 brw_inst_set_3src_dst_reg_file(brw, inst,
887 dest.file == BRW_MESSAGE_REGISTER_FILE);
888 }
889 brw_inst_set_3src_dst_reg_nr(brw, inst, dest.nr);
890 brw_inst_set_3src_dst_subreg_nr(brw, inst, dest.subnr / 16);
891 brw_inst_set_3src_dst_writemask(brw, inst, dest.dw1.bits.writemask);
892 guess_execution_size(p, inst, dest);
893
894 assert(src0.file == BRW_GENERAL_REGISTER_FILE);
895 assert(src0.address_mode == BRW_ADDRESS_DIRECT);
896 assert(src0.nr < 128);
897 brw_inst_set_3src_src0_swizzle(brw, inst, src0.dw1.bits.swizzle);
898 brw_inst_set_3src_src0_subreg_nr(brw, inst, get_3src_subreg_nr(src0));
899 brw_inst_set_3src_src0_reg_nr(brw, inst, src0.nr);
900 brw_inst_set_3src_src0_abs(brw, inst, src0.abs);
901 brw_inst_set_3src_src0_negate(brw, inst, src0.negate);
902 brw_inst_set_3src_src0_rep_ctrl(brw, inst,
903 src0.vstride == BRW_VERTICAL_STRIDE_0);
904
905 assert(src1.file == BRW_GENERAL_REGISTER_FILE);
906 assert(src1.address_mode == BRW_ADDRESS_DIRECT);
907 assert(src1.nr < 128);
908 brw_inst_set_3src_src1_swizzle(brw, inst, src1.dw1.bits.swizzle);
909 brw_inst_set_3src_src1_subreg_nr(brw, inst, get_3src_subreg_nr(src1));
910 brw_inst_set_3src_src1_reg_nr(brw, inst, src1.nr);
911 brw_inst_set_3src_src1_abs(brw, inst, src1.abs);
912 brw_inst_set_3src_src1_negate(brw, inst, src1.negate);
913 brw_inst_set_3src_src1_rep_ctrl(brw, inst,
914 src1.vstride == BRW_VERTICAL_STRIDE_0);
915
916 assert(src2.file == BRW_GENERAL_REGISTER_FILE);
917 assert(src2.address_mode == BRW_ADDRESS_DIRECT);
918 assert(src2.nr < 128);
919 brw_inst_set_3src_src2_swizzle(brw, inst, src2.dw1.bits.swizzle);
920 brw_inst_set_3src_src2_subreg_nr(brw, inst, get_3src_subreg_nr(src2));
921 brw_inst_set_3src_src2_reg_nr(brw, inst, src2.nr);
922 brw_inst_set_3src_src2_abs(brw, inst, src2.abs);
923 brw_inst_set_3src_src2_negate(brw, inst, src2.negate);
924 brw_inst_set_3src_src2_rep_ctrl(brw, inst,
925 src2.vstride == BRW_VERTICAL_STRIDE_0);
926
927 if (brw->gen >= 7) {
928 /* Set both the source and destination types based on dest.type,
929 * ignoring the source register types. The MAD and LRP emitters ensure
930 * that all four types are float. The BFE and BFI2 emitters, however,
931 * may send us mixed D and UD types and want us to ignore that and use
932 * the destination type.
933 */
934 switch (dest.type) {
935 case BRW_REGISTER_TYPE_F:
936 brw_inst_set_3src_src_type(brw, inst, BRW_3SRC_TYPE_F);
937 brw_inst_set_3src_dst_type(brw, inst, BRW_3SRC_TYPE_F);
938 break;
939 case BRW_REGISTER_TYPE_D:
940 brw_inst_set_3src_src_type(brw, inst, BRW_3SRC_TYPE_D);
941 brw_inst_set_3src_dst_type(brw, inst, BRW_3SRC_TYPE_D);
942 break;
943 case BRW_REGISTER_TYPE_UD:
944 brw_inst_set_3src_src_type(brw, inst, BRW_3SRC_TYPE_UD);
945 brw_inst_set_3src_dst_type(brw, inst, BRW_3SRC_TYPE_UD);
946 break;
947 }
948 }
949
950 return inst;
951 }
952
953
954 /***********************************************************************
955 * Convenience routines.
956 */
957 #define ALU1(OP) \
958 brw_inst *brw_##OP(struct brw_compile *p, \
959 struct brw_reg dest, \
960 struct brw_reg src0) \
961 { \
962 return brw_alu1(p, BRW_OPCODE_##OP, dest, src0); \
963 }
964
965 #define ALU2(OP) \
966 brw_inst *brw_##OP(struct brw_compile *p, \
967 struct brw_reg dest, \
968 struct brw_reg src0, \
969 struct brw_reg src1) \
970 { \
971 return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1); \
972 }
973
974 #define ALU3(OP) \
975 brw_inst *brw_##OP(struct brw_compile *p, \
976 struct brw_reg dest, \
977 struct brw_reg src0, \
978 struct brw_reg src1, \
979 struct brw_reg src2) \
980 { \
981 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
982 }
983
984 #define ALU3F(OP) \
985 brw_inst *brw_##OP(struct brw_compile *p, \
986 struct brw_reg dest, \
987 struct brw_reg src0, \
988 struct brw_reg src1, \
989 struct brw_reg src2) \
990 { \
991 assert(dest.type == BRW_REGISTER_TYPE_F); \
992 assert(src0.type == BRW_REGISTER_TYPE_F); \
993 assert(src1.type == BRW_REGISTER_TYPE_F); \
994 assert(src2.type == BRW_REGISTER_TYPE_F); \
995 return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
996 }
997
998 /* Rounding operations (other than RNDD) require two instructions - the first
999 * stores a rounded value (possibly the wrong way) in the dest register, but
1000 * also sets a per-channel "increment bit" in the flag register. A predicated
1001 * add of 1.0 fixes dest to contain the desired result.
1002 *
1003 * Sandybridge and later appear to round correctly without an ADD.
1004 */
1005 #define ROUND(OP) \
1006 void brw_##OP(struct brw_compile *p, \
1007 struct brw_reg dest, \
1008 struct brw_reg src) \
1009 { \
1010 struct brw_context *brw = p->brw; \
1011 brw_inst *rnd, *add; \
1012 rnd = next_insn(p, BRW_OPCODE_##OP); \
1013 brw_set_dest(p, rnd, dest); \
1014 brw_set_src0(p, rnd, src); \
1015 \
1016 if (brw->gen < 6) { \
1017 /* turn on round-increments */ \
1018 brw_inst_set_cond_modifier(brw, rnd, BRW_CONDITIONAL_R); \
1019 add = brw_ADD(p, dest, dest, brw_imm_f(1.0f)); \
1020 brw_inst_set_pred_control(brw, add, BRW_PREDICATE_NORMAL); \
1021 } \
1022 }
1023
1024
1025 ALU1(MOV)
1026 ALU2(SEL)
1027 ALU1(NOT)
1028 ALU2(AND)
1029 ALU2(OR)
1030 ALU2(XOR)
1031 ALU2(SHR)
1032 ALU2(SHL)
1033 ALU2(ASR)
1034 ALU1(FRC)
1035 ALU1(RNDD)
1036 ALU2(MAC)
1037 ALU2(MACH)
1038 ALU1(LZD)
1039 ALU2(DP4)
1040 ALU2(DPH)
1041 ALU2(DP3)
1042 ALU2(DP2)
1043 ALU2(PLN)
1044 ALU3F(MAD)
1045 ALU3F(LRP)
1046 ALU1(BFREV)
1047 ALU3(BFE)
1048 ALU2(BFI1)
1049 ALU3(BFI2)
1050 ALU1(FBH)
1051 ALU1(FBL)
1052 ALU1(CBIT)
1053 ALU2(ADDC)
1054 ALU2(SUBB)
1055
1056 ROUND(RNDZ)
1057 ROUND(RNDE)
1058
1059
1060 brw_inst *
1061 brw_ADD(struct brw_compile *p, struct brw_reg dest,
1062 struct brw_reg src0, struct brw_reg src1)
1063 {
1064 /* 6.2.2: add */
1065 if (src0.type == BRW_REGISTER_TYPE_F ||
1066 (src0.file == BRW_IMMEDIATE_VALUE &&
1067 src0.type == BRW_REGISTER_TYPE_VF)) {
1068 assert(src1.type != BRW_REGISTER_TYPE_UD);
1069 assert(src1.type != BRW_REGISTER_TYPE_D);
1070 }
1071
1072 if (src1.type == BRW_REGISTER_TYPE_F ||
1073 (src1.file == BRW_IMMEDIATE_VALUE &&
1074 src1.type == BRW_REGISTER_TYPE_VF)) {
1075 assert(src0.type != BRW_REGISTER_TYPE_UD);
1076 assert(src0.type != BRW_REGISTER_TYPE_D);
1077 }
1078
1079 return brw_alu2(p, BRW_OPCODE_ADD, dest, src0, src1);
1080 }
1081
1082 brw_inst *
1083 brw_AVG(struct brw_compile *p, struct brw_reg dest,
1084 struct brw_reg src0, struct brw_reg src1)
1085 {
1086 assert(dest.type == src0.type);
1087 assert(src0.type == src1.type);
1088 switch (src0.type) {
1089 case BRW_REGISTER_TYPE_B:
1090 case BRW_REGISTER_TYPE_UB:
1091 case BRW_REGISTER_TYPE_W:
1092 case BRW_REGISTER_TYPE_UW:
1093 case BRW_REGISTER_TYPE_D:
1094 case BRW_REGISTER_TYPE_UD:
1095 break;
1096 default:
1097 unreachable("Bad type for brw_AVG");
1098 }
1099
1100 return brw_alu2(p, BRW_OPCODE_AVG, dest, src0, src1);
1101 }
1102
1103 brw_inst *
1104 brw_MUL(struct brw_compile *p, struct brw_reg dest,
1105 struct brw_reg src0, struct brw_reg src1)
1106 {
1107 /* 6.32.38: mul */
1108 if (src0.type == BRW_REGISTER_TYPE_D ||
1109 src0.type == BRW_REGISTER_TYPE_UD ||
1110 src1.type == BRW_REGISTER_TYPE_D ||
1111 src1.type == BRW_REGISTER_TYPE_UD) {
1112 assert(dest.type != BRW_REGISTER_TYPE_F);
1113 }
1114
1115 if (src0.type == BRW_REGISTER_TYPE_F ||
1116 (src0.file == BRW_IMMEDIATE_VALUE &&
1117 src0.type == BRW_REGISTER_TYPE_VF)) {
1118 assert(src1.type != BRW_REGISTER_TYPE_UD);
1119 assert(src1.type != BRW_REGISTER_TYPE_D);
1120 }
1121
1122 if (src1.type == BRW_REGISTER_TYPE_F ||
1123 (src1.file == BRW_IMMEDIATE_VALUE &&
1124 src1.type == BRW_REGISTER_TYPE_VF)) {
1125 assert(src0.type != BRW_REGISTER_TYPE_UD);
1126 assert(src0.type != BRW_REGISTER_TYPE_D);
1127 }
1128
1129 assert(src0.file != BRW_ARCHITECTURE_REGISTER_FILE ||
1130 src0.nr != BRW_ARF_ACCUMULATOR);
1131 assert(src1.file != BRW_ARCHITECTURE_REGISTER_FILE ||
1132 src1.nr != BRW_ARF_ACCUMULATOR);
1133
1134 return brw_alu2(p, BRW_OPCODE_MUL, dest, src0, src1);
1135 }
1136
1137 brw_inst *
1138 brw_LINE(struct brw_compile *p, struct brw_reg dest,
1139 struct brw_reg src0, struct brw_reg src1)
1140 {
1141 src0.vstride = BRW_VERTICAL_STRIDE_0;
1142 src0.width = BRW_WIDTH_1;
1143 src0.hstride = BRW_HORIZONTAL_STRIDE_0;
1144 return brw_alu2(p, BRW_OPCODE_LINE, dest, src0, src1);
1145 }
1146
1147 brw_inst *
1148 brw_F32TO16(struct brw_compile *p, struct brw_reg dst, struct brw_reg src)
1149 {
1150 const struct brw_context *brw = p->brw;
1151 bool align16 = brw_inst_access_mode(brw, p->current) == BRW_ALIGN_16;
1152
1153 if (align16) {
1154 assert(dst.type == BRW_REGISTER_TYPE_UD);
1155 } else {
1156 assert(dst.type == BRW_REGISTER_TYPE_W ||
1157 dst.type == BRW_REGISTER_TYPE_UW ||
1158 dst.type == BRW_REGISTER_TYPE_HF);
1159 }
1160
1161 if (brw->gen >= 8) {
1162 if (align16) {
1163 /* Emulate the Gen7 zeroing bug (see comments in vec4_visitor's
1164 * emit_pack_half_2x16 method.)
1165 */
1166 brw_MOV(p, retype(dst, BRW_REGISTER_TYPE_UD), brw_imm_ud(0u));
1167 }
1168 return brw_MOV(p, retype(dst, BRW_REGISTER_TYPE_HF), src);
1169 } else {
1170 assert(brw->gen == 7);
1171 return brw_alu1(p, BRW_OPCODE_F32TO16, dst, src);
1172 }
1173 }
1174
1175 brw_inst *
1176 brw_F16TO32(struct brw_compile *p, struct brw_reg dst, struct brw_reg src)
1177 {
1178 const struct brw_context *brw = p->brw;
1179 bool align16 = brw_inst_access_mode(brw, p->current) == BRW_ALIGN_16;
1180
1181 if (align16) {
1182 assert(src.type == BRW_REGISTER_TYPE_UD);
1183 } else {
1184 assert(src.type == BRW_REGISTER_TYPE_W ||
1185 src.type == BRW_REGISTER_TYPE_UW ||
1186 src.type == BRW_REGISTER_TYPE_HF);
1187 }
1188
1189 if (brw->gen >= 8) {
1190 return brw_MOV(p, dst, retype(src, BRW_REGISTER_TYPE_HF));
1191 } else {
1192 assert(brw->gen == 7);
1193 return brw_alu1(p, BRW_OPCODE_F16TO32, dst, src);
1194 }
1195 }
1196
1197
1198 void brw_NOP(struct brw_compile *p)
1199 {
1200 brw_inst *insn = next_insn(p, BRW_OPCODE_NOP);
1201 brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1202 brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1203 brw_set_src1(p, insn, brw_imm_ud(0x0));
1204 }
1205
1206
1207
1208
1209
1210 /***********************************************************************
1211 * Comparisons, if/else/endif
1212 */
1213
1214 brw_inst *
1215 brw_JMPI(struct brw_compile *p, struct brw_reg index,
1216 unsigned predicate_control)
1217 {
1218 const struct brw_context *brw = p->brw;
1219 struct brw_reg ip = brw_ip_reg();
1220 brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index);
1221
1222 brw_inst_set_exec_size(brw, inst, BRW_EXECUTE_2);
1223 brw_inst_set_qtr_control(brw, inst, BRW_COMPRESSION_NONE);
1224 brw_inst_set_mask_control(brw, inst, BRW_MASK_DISABLE);
1225 brw_inst_set_pred_control(brw, inst, predicate_control);
1226
1227 return inst;
1228 }
1229
1230 static void
1231 push_if_stack(struct brw_compile *p, brw_inst *inst)
1232 {
1233 p->if_stack[p->if_stack_depth] = inst - p->store;
1234
1235 p->if_stack_depth++;
1236 if (p->if_stack_array_size <= p->if_stack_depth) {
1237 p->if_stack_array_size *= 2;
1238 p->if_stack = reralloc(p->mem_ctx, p->if_stack, int,
1239 p->if_stack_array_size);
1240 }
1241 }
1242
1243 static brw_inst *
1244 pop_if_stack(struct brw_compile *p)
1245 {
1246 p->if_stack_depth--;
1247 return &p->store[p->if_stack[p->if_stack_depth]];
1248 }
1249
1250 static void
1251 push_loop_stack(struct brw_compile *p, brw_inst *inst)
1252 {
1253 if (p->loop_stack_array_size < p->loop_stack_depth) {
1254 p->loop_stack_array_size *= 2;
1255 p->loop_stack = reralloc(p->mem_ctx, p->loop_stack, int,
1256 p->loop_stack_array_size);
1257 p->if_depth_in_loop = reralloc(p->mem_ctx, p->if_depth_in_loop, int,
1258 p->loop_stack_array_size);
1259 }
1260
1261 p->loop_stack[p->loop_stack_depth] = inst - p->store;
1262 p->loop_stack_depth++;
1263 p->if_depth_in_loop[p->loop_stack_depth] = 0;
1264 }
1265
1266 static brw_inst *
1267 get_inner_do_insn(struct brw_compile *p)
1268 {
1269 return &p->store[p->loop_stack[p->loop_stack_depth - 1]];
1270 }
1271
1272 /* EU takes the value from the flag register and pushes it onto some
1273 * sort of a stack (presumably merging with any flag value already on
1274 * the stack). Within an if block, the flags at the top of the stack
1275 * control execution on each channel of the unit, eg. on each of the
1276 * 16 pixel values in our wm programs.
1277 *
1278 * When the matching 'else' instruction is reached (presumably by
1279 * countdown of the instruction count patched in by our ELSE/ENDIF
1280 * functions), the relevent flags are inverted.
1281 *
1282 * When the matching 'endif' instruction is reached, the flags are
1283 * popped off. If the stack is now empty, normal execution resumes.
1284 */
1285 brw_inst *
1286 brw_IF(struct brw_compile *p, unsigned execute_size)
1287 {
1288 struct brw_context *brw = p->brw;
1289 brw_inst *insn;
1290
1291 insn = next_insn(p, BRW_OPCODE_IF);
1292
1293 /* Override the defaults for this instruction:
1294 */
1295 if (brw->gen < 6) {
1296 brw_set_dest(p, insn, brw_ip_reg());
1297 brw_set_src0(p, insn, brw_ip_reg());
1298 brw_set_src1(p, insn, brw_imm_d(0x0));
1299 } else if (brw->gen == 6) {
1300 brw_set_dest(p, insn, brw_imm_w(0));
1301 brw_inst_set_gen6_jump_count(brw, insn, 0);
1302 brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1303 brw_set_src1(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1304 } else if (brw->gen == 7) {
1305 brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1306 brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1307 brw_set_src1(p, insn, brw_imm_ud(0));
1308 brw_inst_set_jip(brw, insn, 0);
1309 brw_inst_set_uip(brw, insn, 0);
1310 } else {
1311 brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1312 brw_set_src0(p, insn, brw_imm_d(0));
1313 brw_inst_set_jip(brw, insn, 0);
1314 brw_inst_set_uip(brw, insn, 0);
1315 }
1316
1317 brw_inst_set_exec_size(brw, insn, execute_size);
1318 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1319 brw_inst_set_pred_control(brw, insn, BRW_PREDICATE_NORMAL);
1320 brw_inst_set_mask_control(brw, insn, BRW_MASK_ENABLE);
1321 if (!p->single_program_flow && brw->gen < 6)
1322 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1323
1324 push_if_stack(p, insn);
1325 p->if_depth_in_loop[p->loop_stack_depth]++;
1326 return insn;
1327 }
1328
1329 /* This function is only used for gen6-style IF instructions with an
1330 * embedded comparison (conditional modifier). It is not used on gen7.
1331 */
1332 brw_inst *
1333 gen6_IF(struct brw_compile *p, enum brw_conditional_mod conditional,
1334 struct brw_reg src0, struct brw_reg src1)
1335 {
1336 const struct brw_context *brw = p->brw;
1337 brw_inst *insn;
1338
1339 insn = next_insn(p, BRW_OPCODE_IF);
1340
1341 brw_set_dest(p, insn, brw_imm_w(0));
1342 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1343 : BRW_EXECUTE_8);
1344 brw_inst_set_gen6_jump_count(brw, insn, 0);
1345 brw_set_src0(p, insn, src0);
1346 brw_set_src1(p, insn, src1);
1347
1348 assert(brw_inst_qtr_control(brw, insn) == BRW_COMPRESSION_NONE);
1349 assert(brw_inst_pred_control(brw, insn) == BRW_PREDICATE_NONE);
1350 brw_inst_set_cond_modifier(brw, insn, conditional);
1351
1352 push_if_stack(p, insn);
1353 return insn;
1354 }
1355
1356 /**
1357 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1358 */
1359 static void
1360 convert_IF_ELSE_to_ADD(struct brw_compile *p,
1361 brw_inst *if_inst, brw_inst *else_inst)
1362 {
1363 const struct brw_context *brw = p->brw;
1364
1365 /* The next instruction (where the ENDIF would be, if it existed) */
1366 brw_inst *next_inst = &p->store[p->nr_insn];
1367
1368 assert(p->single_program_flow);
1369 assert(if_inst != NULL && brw_inst_opcode(brw, if_inst) == BRW_OPCODE_IF);
1370 assert(else_inst == NULL || brw_inst_opcode(brw, else_inst) == BRW_OPCODE_ELSE);
1371 assert(brw_inst_exec_size(brw, if_inst) == BRW_EXECUTE_1);
1372
1373 /* Convert IF to an ADD instruction that moves the instruction pointer
1374 * to the first instruction of the ELSE block. If there is no ELSE
1375 * block, point to where ENDIF would be. Reverse the predicate.
1376 *
1377 * There's no need to execute an ENDIF since we don't need to do any
1378 * stack operations, and if we're currently executing, we just want to
1379 * continue normally.
1380 */
1381 brw_inst_set_opcode(brw, if_inst, BRW_OPCODE_ADD);
1382 brw_inst_set_pred_inv(brw, if_inst, true);
1383
1384 if (else_inst != NULL) {
1385 /* Convert ELSE to an ADD instruction that points where the ENDIF
1386 * would be.
1387 */
1388 brw_inst_set_opcode(brw, else_inst, BRW_OPCODE_ADD);
1389
1390 brw_inst_set_imm_ud(brw, if_inst, (else_inst - if_inst + 1) * 16);
1391 brw_inst_set_imm_ud(brw, else_inst, (next_inst - else_inst) * 16);
1392 } else {
1393 brw_inst_set_imm_ud(brw, if_inst, (next_inst - if_inst) * 16);
1394 }
1395 }
1396
1397 /**
1398 * Patch IF and ELSE instructions with appropriate jump targets.
1399 */
1400 static void
1401 patch_IF_ELSE(struct brw_compile *p,
1402 brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst)
1403 {
1404 struct brw_context *brw = p->brw;
1405
1406 /* We shouldn't be patching IF and ELSE instructions in single program flow
1407 * mode when gen < 6, because in single program flow mode on those
1408 * platforms, we convert flow control instructions to conditional ADDs that
1409 * operate on IP (see brw_ENDIF).
1410 *
1411 * However, on Gen6, writing to IP doesn't work in single program flow mode
1412 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1413 * not be updated by non-flow control instructions."). And on later
1414 * platforms, there is no significant benefit to converting control flow
1415 * instructions to conditional ADDs. So we do patch IF and ELSE
1416 * instructions in single program flow mode on those platforms.
1417 */
1418 if (brw->gen < 6)
1419 assert(!p->single_program_flow);
1420
1421 assert(if_inst != NULL && brw_inst_opcode(brw, if_inst) == BRW_OPCODE_IF);
1422 assert(endif_inst != NULL);
1423 assert(else_inst == NULL || brw_inst_opcode(brw, else_inst) == BRW_OPCODE_ELSE);
1424
1425 unsigned br = brw_jump_scale(brw);
1426
1427 assert(brw_inst_opcode(brw, endif_inst) == BRW_OPCODE_ENDIF);
1428 brw_inst_set_exec_size(brw, endif_inst, brw_inst_exec_size(brw, if_inst));
1429
1430 if (else_inst == NULL) {
1431 /* Patch IF -> ENDIF */
1432 if (brw->gen < 6) {
1433 /* Turn it into an IFF, which means no mask stack operations for
1434 * all-false and jumping past the ENDIF.
1435 */
1436 brw_inst_set_opcode(brw, if_inst, BRW_OPCODE_IFF);
1437 brw_inst_set_gen4_jump_count(brw, if_inst,
1438 br * (endif_inst - if_inst + 1));
1439 brw_inst_set_gen4_pop_count(brw, if_inst, 0);
1440 } else if (brw->gen == 6) {
1441 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1442 brw_inst_set_gen6_jump_count(brw, if_inst, br*(endif_inst - if_inst));
1443 } else {
1444 brw_inst_set_uip(brw, if_inst, br * (endif_inst - if_inst));
1445 brw_inst_set_jip(brw, if_inst, br * (endif_inst - if_inst));
1446 }
1447 } else {
1448 brw_inst_set_exec_size(brw, else_inst, brw_inst_exec_size(brw, if_inst));
1449
1450 /* Patch IF -> ELSE */
1451 if (brw->gen < 6) {
1452 brw_inst_set_gen4_jump_count(brw, if_inst,
1453 br * (else_inst - if_inst));
1454 brw_inst_set_gen4_pop_count(brw, if_inst, 0);
1455 } else if (brw->gen == 6) {
1456 brw_inst_set_gen6_jump_count(brw, if_inst,
1457 br * (else_inst - if_inst + 1));
1458 }
1459
1460 /* Patch ELSE -> ENDIF */
1461 if (brw->gen < 6) {
1462 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1463 * matching ENDIF.
1464 */
1465 brw_inst_set_gen4_jump_count(brw, else_inst,
1466 br * (endif_inst - else_inst + 1));
1467 brw_inst_set_gen4_pop_count(brw, else_inst, 1);
1468 } else if (brw->gen == 6) {
1469 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1470 brw_inst_set_gen6_jump_count(brw, else_inst,
1471 br * (endif_inst - else_inst));
1472 } else {
1473 /* The IF instruction's JIP should point just past the ELSE */
1474 brw_inst_set_jip(brw, if_inst, br * (else_inst - if_inst + 1));
1475 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1476 brw_inst_set_uip(brw, if_inst, br * (endif_inst - if_inst));
1477 brw_inst_set_jip(brw, else_inst, br * (endif_inst - else_inst));
1478 if (brw->gen >= 8) {
1479 /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1480 * should point to ENDIF.
1481 */
1482 brw_inst_set_uip(brw, else_inst, br * (endif_inst - else_inst));
1483 }
1484 }
1485 }
1486 }
1487
1488 void
1489 brw_ELSE(struct brw_compile *p)
1490 {
1491 struct brw_context *brw = p->brw;
1492 brw_inst *insn;
1493
1494 insn = next_insn(p, BRW_OPCODE_ELSE);
1495
1496 if (brw->gen < 6) {
1497 brw_set_dest(p, insn, brw_ip_reg());
1498 brw_set_src0(p, insn, brw_ip_reg());
1499 brw_set_src1(p, insn, brw_imm_d(0x0));
1500 } else if (brw->gen == 6) {
1501 brw_set_dest(p, insn, brw_imm_w(0));
1502 brw_inst_set_gen6_jump_count(brw, insn, 0);
1503 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1504 brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1505 } else if (brw->gen == 7) {
1506 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1507 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1508 brw_set_src1(p, insn, brw_imm_ud(0));
1509 brw_inst_set_jip(brw, insn, 0);
1510 brw_inst_set_uip(brw, insn, 0);
1511 } else {
1512 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1513 brw_set_src0(p, insn, brw_imm_d(0));
1514 brw_inst_set_jip(brw, insn, 0);
1515 brw_inst_set_uip(brw, insn, 0);
1516 }
1517
1518 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1519 brw_inst_set_mask_control(brw, insn, BRW_MASK_ENABLE);
1520 if (!p->single_program_flow && brw->gen < 6)
1521 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1522
1523 push_if_stack(p, insn);
1524 }
1525
1526 void
1527 brw_ENDIF(struct brw_compile *p)
1528 {
1529 struct brw_context *brw = p->brw;
1530 brw_inst *insn = NULL;
1531 brw_inst *else_inst = NULL;
1532 brw_inst *if_inst = NULL;
1533 brw_inst *tmp;
1534 bool emit_endif = true;
1535
1536 /* In single program flow mode, we can express IF and ELSE instructions
1537 * equivalently as ADD instructions that operate on IP. On platforms prior
1538 * to Gen6, flow control instructions cause an implied thread switch, so
1539 * this is a significant savings.
1540 *
1541 * However, on Gen6, writing to IP doesn't work in single program flow mode
1542 * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1543 * not be updated by non-flow control instructions."). And on later
1544 * platforms, there is no significant benefit to converting control flow
1545 * instructions to conditional ADDs. So we only do this trick on Gen4 and
1546 * Gen5.
1547 */
1548 if (brw->gen < 6 && p->single_program_flow)
1549 emit_endif = false;
1550
1551 /*
1552 * A single next_insn() may change the base adress of instruction store
1553 * memory(p->store), so call it first before referencing the instruction
1554 * store pointer from an index
1555 */
1556 if (emit_endif)
1557 insn = next_insn(p, BRW_OPCODE_ENDIF);
1558
1559 /* Pop the IF and (optional) ELSE instructions from the stack */
1560 p->if_depth_in_loop[p->loop_stack_depth]--;
1561 tmp = pop_if_stack(p);
1562 if (brw_inst_opcode(brw, tmp) == BRW_OPCODE_ELSE) {
1563 else_inst = tmp;
1564 tmp = pop_if_stack(p);
1565 }
1566 if_inst = tmp;
1567
1568 if (!emit_endif) {
1569 /* ENDIF is useless; don't bother emitting it. */
1570 convert_IF_ELSE_to_ADD(p, if_inst, else_inst);
1571 return;
1572 }
1573
1574 if (brw->gen < 6) {
1575 brw_set_dest(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1576 brw_set_src0(p, insn, retype(brw_vec4_grf(0,0), BRW_REGISTER_TYPE_UD));
1577 brw_set_src1(p, insn, brw_imm_d(0x0));
1578 } else if (brw->gen == 6) {
1579 brw_set_dest(p, insn, brw_imm_w(0));
1580 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1581 brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1582 } else if (brw->gen == 7) {
1583 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1584 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1585 brw_set_src1(p, insn, brw_imm_ud(0));
1586 } else {
1587 brw_set_src0(p, insn, brw_imm_d(0));
1588 }
1589
1590 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1591 brw_inst_set_mask_control(brw, insn, BRW_MASK_ENABLE);
1592 if (brw->gen < 6)
1593 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1594
1595 /* Also pop item off the stack in the endif instruction: */
1596 if (brw->gen < 6) {
1597 brw_inst_set_gen4_jump_count(brw, insn, 0);
1598 brw_inst_set_gen4_pop_count(brw, insn, 1);
1599 } else if (brw->gen == 6) {
1600 brw_inst_set_gen6_jump_count(brw, insn, 2);
1601 } else {
1602 brw_inst_set_jip(brw, insn, 2);
1603 }
1604 patch_IF_ELSE(p, if_inst, else_inst, insn);
1605 }
1606
1607 brw_inst *
1608 brw_BREAK(struct brw_compile *p)
1609 {
1610 struct brw_context *brw = p->brw;
1611 brw_inst *insn;
1612
1613 insn = next_insn(p, BRW_OPCODE_BREAK);
1614 if (brw->gen >= 8) {
1615 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1616 brw_set_src0(p, insn, brw_imm_d(0x0));
1617 } else if (brw->gen >= 6) {
1618 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1619 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1620 brw_set_src1(p, insn, brw_imm_d(0x0));
1621 } else {
1622 brw_set_dest(p, insn, brw_ip_reg());
1623 brw_set_src0(p, insn, brw_ip_reg());
1624 brw_set_src1(p, insn, brw_imm_d(0x0));
1625 brw_inst_set_gen4_pop_count(brw, insn,
1626 p->if_depth_in_loop[p->loop_stack_depth]);
1627 }
1628 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1629 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1630 : BRW_EXECUTE_8);
1631
1632 return insn;
1633 }
1634
1635 brw_inst *
1636 brw_CONT(struct brw_compile *p)
1637 {
1638 const struct brw_context *brw = p->brw;
1639 brw_inst *insn;
1640
1641 insn = next_insn(p, BRW_OPCODE_CONTINUE);
1642 brw_set_dest(p, insn, brw_ip_reg());
1643 if (brw->gen >= 8) {
1644 brw_set_src0(p, insn, brw_imm_d(0x0));
1645 } else {
1646 brw_set_src0(p, insn, brw_ip_reg());
1647 brw_set_src1(p, insn, brw_imm_d(0x0));
1648 }
1649
1650 if (brw->gen < 6) {
1651 brw_inst_set_gen4_pop_count(brw, insn,
1652 p->if_depth_in_loop[p->loop_stack_depth]);
1653 }
1654 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1655 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1656 : BRW_EXECUTE_8);
1657 return insn;
1658 }
1659
1660 brw_inst *
1661 gen6_HALT(struct brw_compile *p)
1662 {
1663 const struct brw_context *brw = p->brw;
1664 brw_inst *insn;
1665
1666 insn = next_insn(p, BRW_OPCODE_HALT);
1667 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1668 if (brw->gen >= 8) {
1669 brw_set_src0(p, insn, brw_imm_d(0x0));
1670 } else {
1671 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1672 brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1673 }
1674
1675 if (p->compressed) {
1676 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_16);
1677 } else {
1678 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1679 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_8);
1680 }
1681 return insn;
1682 }
1683
1684 /* DO/WHILE loop:
1685 *
1686 * The DO/WHILE is just an unterminated loop -- break or continue are
1687 * used for control within the loop. We have a few ways they can be
1688 * done.
1689 *
1690 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1691 * jip and no DO instruction.
1692 *
1693 * For non-uniform control flow pre-gen6, there's a DO instruction to
1694 * push the mask, and a WHILE to jump back, and BREAK to get out and
1695 * pop the mask.
1696 *
1697 * For gen6, there's no more mask stack, so no need for DO. WHILE
1698 * just points back to the first instruction of the loop.
1699 */
1700 brw_inst *
1701 brw_DO(struct brw_compile *p, unsigned execute_size)
1702 {
1703 struct brw_context *brw = p->brw;
1704
1705 if (brw->gen >= 6 || p->single_program_flow) {
1706 push_loop_stack(p, &p->store[p->nr_insn]);
1707 return &p->store[p->nr_insn];
1708 } else {
1709 brw_inst *insn = next_insn(p, BRW_OPCODE_DO);
1710
1711 push_loop_stack(p, insn);
1712
1713 /* Override the defaults for this instruction:
1714 */
1715 brw_set_dest(p, insn, brw_null_reg());
1716 brw_set_src0(p, insn, brw_null_reg());
1717 brw_set_src1(p, insn, brw_null_reg());
1718
1719 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1720 brw_inst_set_exec_size(brw, insn, execute_size);
1721 brw_inst_set_pred_control(brw, insn, BRW_PREDICATE_NONE);
1722
1723 return insn;
1724 }
1725 }
1726
1727 /**
1728 * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1729 * instruction here.
1730 *
1731 * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1732 * nesting, since it can always just point to the end of the block/current loop.
1733 */
1734 static void
1735 brw_patch_break_cont(struct brw_compile *p, brw_inst *while_inst)
1736 {
1737 struct brw_context *brw = p->brw;
1738 brw_inst *do_inst = get_inner_do_insn(p);
1739 brw_inst *inst;
1740 unsigned br = brw_jump_scale(brw);
1741
1742 assert(brw->gen < 6);
1743
1744 for (inst = while_inst - 1; inst != do_inst; inst--) {
1745 /* If the jump count is != 0, that means that this instruction has already
1746 * been patched because it's part of a loop inside of the one we're
1747 * patching.
1748 */
1749 if (brw_inst_opcode(brw, inst) == BRW_OPCODE_BREAK &&
1750 brw_inst_gen4_jump_count(brw, inst) == 0) {
1751 brw_inst_set_gen4_jump_count(brw, inst, br*((while_inst - inst) + 1));
1752 } else if (brw_inst_opcode(brw, inst) == BRW_OPCODE_CONTINUE &&
1753 brw_inst_gen4_jump_count(brw, inst) == 0) {
1754 brw_inst_set_gen4_jump_count(brw, inst, br * (while_inst - inst));
1755 }
1756 }
1757 }
1758
1759 brw_inst *
1760 brw_WHILE(struct brw_compile *p)
1761 {
1762 struct brw_context *brw = p->brw;
1763 brw_inst *insn, *do_insn;
1764 unsigned br = brw_jump_scale(brw);
1765
1766 if (brw->gen >= 6) {
1767 insn = next_insn(p, BRW_OPCODE_WHILE);
1768 do_insn = get_inner_do_insn(p);
1769
1770 if (brw->gen >= 8) {
1771 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1772 brw_set_src0(p, insn, brw_imm_d(0));
1773 brw_inst_set_jip(brw, insn, br * (do_insn - insn));
1774 } else if (brw->gen == 7) {
1775 brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1776 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1777 brw_set_src1(p, insn, brw_imm_ud(0));
1778 brw_inst_set_jip(brw, insn, br * (do_insn - insn));
1779 } else {
1780 brw_set_dest(p, insn, brw_imm_w(0));
1781 brw_inst_set_gen6_jump_count(brw, insn, br * (do_insn - insn));
1782 brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1783 brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1784 }
1785
1786 brw_inst_set_exec_size(brw, insn, p->compressed ? BRW_EXECUTE_16
1787 : BRW_EXECUTE_8);
1788 } else {
1789 if (p->single_program_flow) {
1790 insn = next_insn(p, BRW_OPCODE_ADD);
1791 do_insn = get_inner_do_insn(p);
1792
1793 brw_set_dest(p, insn, brw_ip_reg());
1794 brw_set_src0(p, insn, brw_ip_reg());
1795 brw_set_src1(p, insn, brw_imm_d((do_insn - insn) * 16));
1796 brw_inst_set_exec_size(brw, insn, BRW_EXECUTE_1);
1797 } else {
1798 insn = next_insn(p, BRW_OPCODE_WHILE);
1799 do_insn = get_inner_do_insn(p);
1800
1801 assert(brw_inst_opcode(brw, do_insn) == BRW_OPCODE_DO);
1802
1803 brw_set_dest(p, insn, brw_ip_reg());
1804 brw_set_src0(p, insn, brw_ip_reg());
1805 brw_set_src1(p, insn, brw_imm_d(0));
1806
1807 brw_inst_set_exec_size(brw, insn, brw_inst_exec_size(brw, do_insn));
1808 brw_inst_set_gen4_jump_count(brw, insn, br * (do_insn - insn + 1));
1809 brw_inst_set_gen4_pop_count(brw, insn, 0);
1810
1811 brw_patch_break_cont(p, insn);
1812 }
1813 }
1814 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
1815
1816 p->loop_stack_depth--;
1817
1818 return insn;
1819 }
1820
1821 /* FORWARD JUMPS:
1822 */
1823 void brw_land_fwd_jump(struct brw_compile *p, int jmp_insn_idx)
1824 {
1825 struct brw_context *brw = p->brw;
1826 brw_inst *jmp_insn = &p->store[jmp_insn_idx];
1827 unsigned jmpi = 1;
1828
1829 if (brw->gen >= 5)
1830 jmpi = 2;
1831
1832 assert(brw_inst_opcode(brw, jmp_insn) == BRW_OPCODE_JMPI);
1833 assert(brw_inst_src1_reg_file(brw, jmp_insn) == BRW_IMMEDIATE_VALUE);
1834
1835 brw_inst_set_gen4_jump_count(brw, jmp_insn,
1836 jmpi * (p->nr_insn - jmp_insn_idx - 1));
1837 }
1838
1839 /* To integrate with the above, it makes sense that the comparison
1840 * instruction should populate the flag register. It might be simpler
1841 * just to use the flag reg for most WM tasks?
1842 */
1843 void brw_CMP(struct brw_compile *p,
1844 struct brw_reg dest,
1845 unsigned conditional,
1846 struct brw_reg src0,
1847 struct brw_reg src1)
1848 {
1849 struct brw_context *brw = p->brw;
1850 brw_inst *insn = next_insn(p, BRW_OPCODE_CMP);
1851
1852 if (brw->gen >= 8) {
1853 /* The CMP instruction appears to behave erratically for floating point
1854 * sources unless the destination type is also float. Overriding it to
1855 * match src0 makes it work in all cases.
1856 */
1857 dest.type = src0.type;
1858 }
1859
1860 brw_inst_set_cond_modifier(brw, insn, conditional);
1861 brw_set_dest(p, insn, dest);
1862 brw_set_src0(p, insn, src0);
1863 brw_set_src1(p, insn, src1);
1864
1865 /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1866 * page says:
1867 * "Any CMP instruction with a null destination must use a {switch}."
1868 *
1869 * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1870 * mentioned on their work-arounds pages.
1871 */
1872 if (brw->gen == 7) {
1873 if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE &&
1874 dest.nr == BRW_ARF_NULL) {
1875 brw_inst_set_thread_control(brw, insn, BRW_THREAD_SWITCH);
1876 }
1877 }
1878 }
1879
1880 /***********************************************************************
1881 * Helpers for the various SEND message types:
1882 */
1883
1884 /** Extended math function, float[8].
1885 */
1886 void gen4_math(struct brw_compile *p,
1887 struct brw_reg dest,
1888 unsigned function,
1889 unsigned msg_reg_nr,
1890 struct brw_reg src,
1891 unsigned precision )
1892 {
1893 struct brw_context *brw = p->brw;
1894 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
1895 unsigned data_type;
1896 if (src.vstride == BRW_VERTICAL_STRIDE_0 &&
1897 src.width == BRW_WIDTH_1 &&
1898 src.hstride == BRW_HORIZONTAL_STRIDE_0) {
1899 data_type = BRW_MATH_DATA_SCALAR;
1900 } else {
1901 data_type = BRW_MATH_DATA_VECTOR;
1902 }
1903
1904 assert(brw->gen < 6);
1905
1906 /* Example code doesn't set predicate_control for send
1907 * instructions.
1908 */
1909 brw_inst_set_pred_control(brw, insn, 0);
1910 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
1911
1912 brw_set_dest(p, insn, dest);
1913 brw_set_src0(p, insn, src);
1914 brw_set_math_message(p,
1915 insn,
1916 function,
1917 src.type == BRW_REGISTER_TYPE_D,
1918 precision,
1919 data_type);
1920 }
1921
1922 void gen6_math(struct brw_compile *p,
1923 struct brw_reg dest,
1924 unsigned function,
1925 struct brw_reg src0,
1926 struct brw_reg src1)
1927 {
1928 struct brw_context *brw = p->brw;
1929 brw_inst *insn = next_insn(p, BRW_OPCODE_MATH);
1930
1931 assert(brw->gen >= 6);
1932
1933 assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
1934 (brw->gen >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE));
1935 assert(src0.file == BRW_GENERAL_REGISTER_FILE ||
1936 (brw->gen >= 8 && src0.file == BRW_IMMEDIATE_VALUE));
1937
1938 assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1);
1939 if (brw->gen == 6) {
1940 assert(src0.hstride == BRW_HORIZONTAL_STRIDE_1);
1941 assert(src1.hstride == BRW_HORIZONTAL_STRIDE_1);
1942 }
1943
1944 if (function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT ||
1945 function == BRW_MATH_FUNCTION_INT_DIV_REMAINDER ||
1946 function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER) {
1947 assert(src0.type != BRW_REGISTER_TYPE_F);
1948 assert(src1.type != BRW_REGISTER_TYPE_F);
1949 assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
1950 (brw->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
1951 } else {
1952 assert(src0.type == BRW_REGISTER_TYPE_F);
1953 assert(src1.type == BRW_REGISTER_TYPE_F);
1954 if (function == BRW_MATH_FUNCTION_POW) {
1955 assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
1956 (brw->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
1957 } else {
1958 assert(src1.file == BRW_ARCHITECTURE_REGISTER_FILE &&
1959 src1.nr == BRW_ARF_NULL);
1960 }
1961 }
1962
1963 /* Source modifiers are ignored for extended math instructions on Gen6. */
1964 if (brw->gen == 6) {
1965 assert(!src0.negate);
1966 assert(!src0.abs);
1967 assert(!src1.negate);
1968 assert(!src1.abs);
1969 }
1970
1971 brw_inst_set_math_function(brw, insn, function);
1972
1973 brw_set_dest(p, insn, dest);
1974 brw_set_src0(p, insn, src0);
1975 brw_set_src1(p, insn, src1);
1976 }
1977
1978
1979 /**
1980 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
1981 * using a constant offset per channel.
1982 *
1983 * The offset must be aligned to oword size (16 bytes). Used for
1984 * register spilling.
1985 */
1986 void brw_oword_block_write_scratch(struct brw_compile *p,
1987 struct brw_reg mrf,
1988 int num_regs,
1989 unsigned offset)
1990 {
1991 struct brw_context *brw = p->brw;
1992 uint32_t msg_control, msg_type;
1993 int mlen;
1994
1995 if (brw->gen >= 6)
1996 offset /= 16;
1997
1998 mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
1999
2000 if (num_regs == 1) {
2001 msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS;
2002 mlen = 2;
2003 } else {
2004 msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS;
2005 mlen = 3;
2006 }
2007
2008 /* Set up the message header. This is g0, with g0.2 filled with
2009 * the offset. We don't want to leave our offset around in g0 or
2010 * it'll screw up texture samples, so set it up inside the message
2011 * reg.
2012 */
2013 {
2014 brw_push_insn_state(p);
2015 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2016 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2017
2018 brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2019
2020 /* set message header global offset field (reg 0, element 2) */
2021 brw_MOV(p,
2022 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
2023 mrf.nr,
2024 2), BRW_REGISTER_TYPE_UD),
2025 brw_imm_ud(offset));
2026
2027 brw_pop_insn_state(p);
2028 }
2029
2030 {
2031 struct brw_reg dest;
2032 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2033 int send_commit_msg;
2034 struct brw_reg src_header = retype(brw_vec8_grf(0, 0),
2035 BRW_REGISTER_TYPE_UW);
2036
2037 if (brw_inst_qtr_control(brw, insn) != BRW_COMPRESSION_NONE) {
2038 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2039 src_header = vec16(src_header);
2040 }
2041 assert(brw_inst_pred_control(brw, insn) == BRW_PREDICATE_NONE);
2042 if (brw->gen < 6)
2043 brw_inst_set_base_mrf(brw, insn, mrf.nr);
2044
2045 /* Until gen6, writes followed by reads from the same location
2046 * are not guaranteed to be ordered unless write_commit is set.
2047 * If set, then a no-op write is issued to the destination
2048 * register to set a dependency, and a read from the destination
2049 * can be used to ensure the ordering.
2050 *
2051 * For gen6, only writes between different threads need ordering
2052 * protection. Our use of DP writes is all about register
2053 * spilling within a thread.
2054 */
2055 if (brw->gen >= 6) {
2056 dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2057 send_commit_msg = 0;
2058 } else {
2059 dest = src_header;
2060 send_commit_msg = 1;
2061 }
2062
2063 brw_set_dest(p, insn, dest);
2064 if (brw->gen >= 6) {
2065 brw_set_src0(p, insn, mrf);
2066 } else {
2067 brw_set_src0(p, insn, brw_null_reg());
2068 }
2069
2070 if (brw->gen >= 6)
2071 msg_type = GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
2072 else
2073 msg_type = BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
2074
2075 brw_set_dp_write_message(p,
2076 insn,
2077 255, /* binding table index (255=stateless) */
2078 msg_control,
2079 msg_type,
2080 mlen,
2081 true, /* header_present */
2082 0, /* not a render target */
2083 send_commit_msg, /* response_length */
2084 0, /* eot */
2085 send_commit_msg);
2086 }
2087 }
2088
2089
2090 /**
2091 * Read a block of owords (half a GRF each) from the scratch buffer
2092 * using a constant index per channel.
2093 *
2094 * Offset must be aligned to oword size (16 bytes). Used for register
2095 * spilling.
2096 */
2097 void
2098 brw_oword_block_read_scratch(struct brw_compile *p,
2099 struct brw_reg dest,
2100 struct brw_reg mrf,
2101 int num_regs,
2102 unsigned offset)
2103 {
2104 struct brw_context *brw = p->brw;
2105 uint32_t msg_control;
2106 int rlen;
2107
2108 if (brw->gen >= 6)
2109 offset /= 16;
2110
2111 if (p->brw->gen >= 7) {
2112 /* On gen 7 and above, we no longer have message registers and we can
2113 * send from any register we want. By using the destination register
2114 * for the message, we guarantee that the implied message write won't
2115 * accidentally overwrite anything. This has been a problem because
2116 * the MRF registers and source for the final FB write are both fixed
2117 * and may overlap.
2118 */
2119 mrf = retype(dest, BRW_REGISTER_TYPE_UD);
2120 } else {
2121 mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2122 }
2123 dest = retype(dest, BRW_REGISTER_TYPE_UW);
2124
2125 if (num_regs == 1) {
2126 msg_control = BRW_DATAPORT_OWORD_BLOCK_2_OWORDS;
2127 rlen = 1;
2128 } else {
2129 msg_control = BRW_DATAPORT_OWORD_BLOCK_4_OWORDS;
2130 rlen = 2;
2131 }
2132
2133 {
2134 brw_push_insn_state(p);
2135 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2136 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2137
2138 brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2139
2140 /* set message header global offset field (reg 0, element 2) */
2141 brw_MOV(p, get_element_ud(mrf, 2), brw_imm_ud(offset));
2142
2143 brw_pop_insn_state(p);
2144 }
2145
2146 {
2147 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2148
2149 assert(brw_inst_pred_control(brw, insn) == 0);
2150 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2151
2152 brw_set_dest(p, insn, dest); /* UW? */
2153 if (brw->gen >= 6) {
2154 brw_set_src0(p, insn, mrf);
2155 } else {
2156 brw_set_src0(p, insn, brw_null_reg());
2157 brw_inst_set_base_mrf(brw, insn, mrf.nr);
2158 }
2159
2160 brw_set_dp_read_message(p,
2161 insn,
2162 255, /* binding table index (255=stateless) */
2163 msg_control,
2164 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ, /* msg_type */
2165 BRW_DATAPORT_READ_TARGET_RENDER_CACHE,
2166 1, /* msg_length */
2167 true, /* header_present */
2168 rlen);
2169 }
2170 }
2171
2172 void
2173 gen7_block_read_scratch(struct brw_compile *p,
2174 struct brw_reg dest,
2175 int num_regs,
2176 unsigned offset)
2177 {
2178 const struct brw_context *brw = p->brw;
2179 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2180 assert(brw_inst_pred_control(brw, insn) == BRW_PREDICATE_NONE);
2181
2182 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2183 brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW));
2184
2185 /* The HW requires that the header is present; this is to get the g0.5
2186 * scratch offset.
2187 */
2188 brw_set_src0(p, insn, brw_vec8_grf(0, 0));
2189
2190 /* According to the docs, offset is "A 12-bit HWord offset into the memory
2191 * Immediate Memory buffer as specified by binding table 0xFF." An HWORD
2192 * is 32 bytes, which happens to be the size of a register.
2193 */
2194 offset /= REG_SIZE;
2195 assert(offset < (1 << 12));
2196
2197 gen7_set_dp_scratch_message(p, insn,
2198 false, /* scratch read */
2199 false, /* OWords */
2200 false, /* invalidate after read */
2201 num_regs,
2202 offset,
2203 1, /* mlen: just g0 */
2204 num_regs, /* rlen */
2205 true); /* header present */
2206 }
2207
2208 /**
2209 * Read a float[4] vector from the data port Data Cache (const buffer).
2210 * Location (in buffer) should be a multiple of 16.
2211 * Used for fetching shader constants.
2212 */
2213 void brw_oword_block_read(struct brw_compile *p,
2214 struct brw_reg dest,
2215 struct brw_reg mrf,
2216 uint32_t offset,
2217 uint32_t bind_table_index)
2218 {
2219 struct brw_context *brw = p->brw;
2220
2221 /* On newer hardware, offset is in units of owords. */
2222 if (brw->gen >= 6)
2223 offset /= 16;
2224
2225 mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2226
2227 brw_push_insn_state(p);
2228 brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
2229 brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2230 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2231
2232 brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2233
2234 /* set message header global offset field (reg 0, element 2) */
2235 brw_MOV(p,
2236 retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
2237 mrf.nr,
2238 2), BRW_REGISTER_TYPE_UD),
2239 brw_imm_ud(offset));
2240
2241 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2242
2243 /* cast dest to a uword[8] vector */
2244 dest = retype(vec8(dest), BRW_REGISTER_TYPE_UW);
2245
2246 brw_set_dest(p, insn, dest);
2247 if (brw->gen >= 6) {
2248 brw_set_src0(p, insn, mrf);
2249 } else {
2250 brw_set_src0(p, insn, brw_null_reg());
2251 brw_inst_set_base_mrf(brw, insn, mrf.nr);
2252 }
2253
2254 brw_set_dp_read_message(p,
2255 insn,
2256 bind_table_index,
2257 BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW,
2258 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ,
2259 BRW_DATAPORT_READ_TARGET_DATA_CACHE,
2260 1, /* msg_length */
2261 true, /* header_present */
2262 1); /* response_length (1 reg, 2 owords!) */
2263
2264 brw_pop_insn_state(p);
2265 }
2266
2267
2268 void brw_fb_WRITE(struct brw_compile *p,
2269 int dispatch_width,
2270 struct brw_reg payload,
2271 struct brw_reg implied_header,
2272 unsigned msg_control,
2273 unsigned binding_table_index,
2274 unsigned msg_length,
2275 unsigned response_length,
2276 bool eot,
2277 bool header_present)
2278 {
2279 struct brw_context *brw = p->brw;
2280 brw_inst *insn;
2281 unsigned msg_type;
2282 struct brw_reg dest, src0;
2283
2284 if (dispatch_width == 16)
2285 dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2286 else
2287 dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2288
2289 if (brw->gen >= 6) {
2290 insn = next_insn(p, BRW_OPCODE_SENDC);
2291 } else {
2292 insn = next_insn(p, BRW_OPCODE_SEND);
2293 }
2294 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2295
2296 if (brw->gen >= 6) {
2297 /* headerless version, just submit color payload */
2298 src0 = payload;
2299
2300 msg_type = GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE;
2301 } else {
2302 assert(payload.file == BRW_MESSAGE_REGISTER_FILE);
2303 brw_inst_set_base_mrf(brw, insn, payload.nr);
2304 src0 = implied_header;
2305
2306 msg_type = BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE;
2307 }
2308
2309 brw_set_dest(p, insn, dest);
2310 brw_set_src0(p, insn, src0);
2311 brw_set_dp_write_message(p,
2312 insn,
2313 binding_table_index,
2314 msg_control,
2315 msg_type,
2316 msg_length,
2317 header_present,
2318 eot, /* last render target write */
2319 response_length,
2320 eot,
2321 0 /* send_commit_msg */);
2322 }
2323
2324
2325 /**
2326 * Texture sample instruction.
2327 * Note: the msg_type plus msg_length values determine exactly what kind
2328 * of sampling operation is performed. See volume 4, page 161 of docs.
2329 */
2330 void brw_SAMPLE(struct brw_compile *p,
2331 struct brw_reg dest,
2332 unsigned msg_reg_nr,
2333 struct brw_reg src0,
2334 unsigned binding_table_index,
2335 unsigned sampler,
2336 unsigned msg_type,
2337 unsigned response_length,
2338 unsigned msg_length,
2339 unsigned header_present,
2340 unsigned simd_mode,
2341 unsigned return_format)
2342 {
2343 struct brw_context *brw = p->brw;
2344 brw_inst *insn;
2345
2346 if (msg_reg_nr != -1)
2347 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2348
2349 insn = next_insn(p, BRW_OPCODE_SEND);
2350 brw_inst_set_pred_control(brw, insn, BRW_PREDICATE_NONE); /* XXX */
2351
2352 /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2353 *
2354 * "Instruction compression is not allowed for this instruction (that
2355 * is, send). The hardware behavior is undefined if this instruction is
2356 * set as compressed. However, compress control can be set to "SecHalf"
2357 * to affect the EMask generation."
2358 *
2359 * No similar wording is found in later PRMs, but there are examples
2360 * utilizing send with SecHalf. More importantly, SIMD8 sampler messages
2361 * are allowed in SIMD16 mode and they could not work without SecHalf. For
2362 * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2363 */
2364 if (brw_inst_qtr_control(brw, insn) != BRW_COMPRESSION_2NDHALF)
2365 brw_inst_set_qtr_control(brw, insn, BRW_COMPRESSION_NONE);
2366
2367 if (brw->gen < 6)
2368 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
2369
2370 brw_set_dest(p, insn, dest);
2371 brw_set_src0(p, insn, src0);
2372 brw_set_sampler_message(p, insn,
2373 binding_table_index,
2374 sampler,
2375 msg_type,
2376 response_length,
2377 msg_length,
2378 header_present,
2379 simd_mode,
2380 return_format);
2381 }
2382
2383 /* Adjust the message header's sampler state pointer to
2384 * select the correct group of 16 samplers.
2385 */
2386 void brw_adjust_sampler_state_pointer(struct brw_compile *p,
2387 struct brw_reg header,
2388 struct brw_reg sampler_index,
2389 struct brw_reg scratch)
2390 {
2391 /* The "Sampler Index" field can only store values between 0 and 15.
2392 * However, we can add an offset to the "Sampler State Pointer"
2393 * field, effectively selecting a different set of 16 samplers.
2394 *
2395 * The "Sampler State Pointer" needs to be aligned to a 32-byte
2396 * offset, and each sampler state is only 16-bytes, so we can't
2397 * exclusively use the offset - we have to use both.
2398 */
2399
2400 struct brw_context *brw = p->brw;
2401
2402 if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
2403 const int sampler_state_size = 16; /* 16 bytes */
2404 uint32_t sampler = sampler_index.dw1.ud;
2405
2406 if (sampler >= 16) {
2407 assert(brw->is_haswell || brw->gen >= 8);
2408 brw_ADD(p,
2409 get_element_ud(header, 3),
2410 get_element_ud(brw_vec8_grf(0, 0), 3),
2411 brw_imm_ud(16 * (sampler / 16) * sampler_state_size));
2412 }
2413 } else {
2414 /* Non-const sampler array indexing case */
2415 if (brw->gen < 8 && !brw->is_haswell) {
2416 return;
2417 }
2418
2419 struct brw_reg temp = vec1(retype(scratch, BRW_REGISTER_TYPE_UD));
2420
2421 brw_AND(p, temp, get_element_ud(sampler_index, 0), brw_imm_ud(0x0f0));
2422 brw_SHL(p, temp, temp, brw_imm_ud(4));
2423 brw_ADD(p,
2424 get_element_ud(header, 3),
2425 get_element_ud(brw_vec8_grf(0, 0), 3),
2426 temp);
2427 }
2428 }
2429
2430 /* All these variables are pretty confusing - we might be better off
2431 * using bitmasks and macros for this, in the old style. Or perhaps
2432 * just having the caller instantiate the fields in dword3 itself.
2433 */
2434 void brw_urb_WRITE(struct brw_compile *p,
2435 struct brw_reg dest,
2436 unsigned msg_reg_nr,
2437 struct brw_reg src0,
2438 enum brw_urb_write_flags flags,
2439 unsigned msg_length,
2440 unsigned response_length,
2441 unsigned offset,
2442 unsigned swizzle)
2443 {
2444 struct brw_context *brw = p->brw;
2445 brw_inst *insn;
2446
2447 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2448
2449 if (brw->gen >= 7 && !(flags & BRW_URB_WRITE_USE_CHANNEL_MASKS)) {
2450 /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2451 brw_push_insn_state(p);
2452 brw_set_default_access_mode(p, BRW_ALIGN_1);
2453 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2454 brw_OR(p, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE, msg_reg_nr, 5),
2455 BRW_REGISTER_TYPE_UD),
2456 retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD),
2457 brw_imm_ud(0xff00));
2458 brw_pop_insn_state(p);
2459 }
2460
2461 insn = next_insn(p, BRW_OPCODE_SEND);
2462
2463 assert(msg_length < BRW_MAX_MRF);
2464
2465 brw_set_dest(p, insn, dest);
2466 brw_set_src0(p, insn, src0);
2467 brw_set_src1(p, insn, brw_imm_d(0));
2468
2469 if (brw->gen < 6)
2470 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
2471
2472 brw_set_urb_message(p,
2473 insn,
2474 flags,
2475 msg_length,
2476 response_length,
2477 offset,
2478 swizzle);
2479 }
2480
2481 static int
2482 brw_find_next_block_end(struct brw_compile *p, int start_offset)
2483 {
2484 int offset;
2485 void *store = p->store;
2486 const struct brw_context *brw = p->brw;
2487
2488 for (offset = next_offset(brw, store, start_offset);
2489 offset < p->next_insn_offset;
2490 offset = next_offset(brw, store, offset)) {
2491 brw_inst *insn = store + offset;
2492
2493 switch (brw_inst_opcode(brw, insn)) {
2494 case BRW_OPCODE_ENDIF:
2495 case BRW_OPCODE_ELSE:
2496 case BRW_OPCODE_WHILE:
2497 case BRW_OPCODE_HALT:
2498 return offset;
2499 }
2500 }
2501
2502 return 0;
2503 }
2504
2505 /* There is no DO instruction on gen6, so to find the end of the loop
2506 * we have to see if the loop is jumping back before our start
2507 * instruction.
2508 */
2509 static int
2510 brw_find_loop_end(struct brw_compile *p, int start_offset)
2511 {
2512 struct brw_context *brw = p->brw;
2513 int offset;
2514 int scale = 16 / brw_jump_scale(brw);
2515 void *store = p->store;
2516
2517 assert(brw->gen >= 6);
2518
2519 /* Always start after the instruction (such as a WHILE) we're trying to fix
2520 * up.
2521 */
2522 for (offset = next_offset(brw, store, start_offset);
2523 offset < p->next_insn_offset;
2524 offset = next_offset(brw, store, offset)) {
2525 brw_inst *insn = store + offset;
2526
2527 if (brw_inst_opcode(brw, insn) == BRW_OPCODE_WHILE) {
2528 int jip = brw->gen == 6 ? brw_inst_gen6_jump_count(brw, insn)
2529 : brw_inst_jip(brw, insn);
2530 if (offset + jip * scale <= start_offset)
2531 return offset;
2532 }
2533 }
2534 assert(!"not reached");
2535 return start_offset;
2536 }
2537
2538 /* After program generation, go back and update the UIP and JIP of
2539 * BREAK, CONT, and HALT instructions to their correct locations.
2540 */
2541 void
2542 brw_set_uip_jip(struct brw_compile *p)
2543 {
2544 struct brw_context *brw = p->brw;
2545 int offset;
2546 int br = brw_jump_scale(brw);
2547 int scale = 16 / br;
2548 void *store = p->store;
2549
2550 if (brw->gen < 6)
2551 return;
2552
2553 for (offset = 0; offset < p->next_insn_offset;
2554 offset = next_offset(brw, store, offset)) {
2555 brw_inst *insn = store + offset;
2556
2557 if (brw_inst_cmpt_control(brw, insn)) {
2558 /* Fixups for compacted BREAK/CONTINUE not supported yet. */
2559 assert(brw_inst_opcode(brw, insn) != BRW_OPCODE_BREAK &&
2560 brw_inst_opcode(brw, insn) != BRW_OPCODE_CONTINUE &&
2561 brw_inst_opcode(brw, insn) != BRW_OPCODE_HALT);
2562 continue;
2563 }
2564
2565 int block_end_offset = brw_find_next_block_end(p, offset);
2566 switch (brw_inst_opcode(brw, insn)) {
2567 case BRW_OPCODE_BREAK:
2568 assert(block_end_offset != 0);
2569 brw_inst_set_jip(brw, insn, (block_end_offset - offset) / scale);
2570 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2571 brw_inst_set_uip(brw, insn,
2572 (brw_find_loop_end(p, offset) - offset +
2573 (brw->gen == 6 ? 16 : 0)) / scale);
2574 break;
2575 case BRW_OPCODE_CONTINUE:
2576 assert(block_end_offset != 0);
2577 brw_inst_set_jip(brw, insn, (block_end_offset - offset) / scale);
2578 brw_inst_set_uip(brw, insn,
2579 (brw_find_loop_end(p, offset) - offset) / scale);
2580
2581 assert(brw_inst_uip(brw, insn) != 0);
2582 assert(brw_inst_jip(brw, insn) != 0);
2583 break;
2584
2585 case BRW_OPCODE_ENDIF: {
2586 int32_t jump = (block_end_offset == 0) ?
2587 1 * br : (block_end_offset - offset) / scale;
2588 if (brw->gen >= 7)
2589 brw_inst_set_jip(brw, insn, jump);
2590 else
2591 brw_inst_set_gen6_jump_count(brw, insn, jump);
2592 break;
2593 }
2594
2595 case BRW_OPCODE_HALT:
2596 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2597 *
2598 * "In case of the halt instruction not inside any conditional
2599 * code block, the value of <JIP> and <UIP> should be the
2600 * same. In case of the halt instruction inside conditional code
2601 * block, the <UIP> should be the end of the program, and the
2602 * <JIP> should be end of the most inner conditional code block."
2603 *
2604 * The uip will have already been set by whoever set up the
2605 * instruction.
2606 */
2607 if (block_end_offset == 0) {
2608 brw_inst_set_jip(brw, insn, brw_inst_uip(brw, insn));
2609 } else {
2610 brw_inst_set_jip(brw, insn, (block_end_offset - offset) / scale);
2611 }
2612 assert(brw_inst_uip(brw, insn) != 0);
2613 assert(brw_inst_jip(brw, insn) != 0);
2614 break;
2615 }
2616 }
2617 }
2618
2619 void brw_ff_sync(struct brw_compile *p,
2620 struct brw_reg dest,
2621 unsigned msg_reg_nr,
2622 struct brw_reg src0,
2623 bool allocate,
2624 unsigned response_length,
2625 bool eot)
2626 {
2627 struct brw_context *brw = p->brw;
2628 brw_inst *insn;
2629
2630 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2631
2632 insn = next_insn(p, BRW_OPCODE_SEND);
2633 brw_set_dest(p, insn, dest);
2634 brw_set_src0(p, insn, src0);
2635 brw_set_src1(p, insn, brw_imm_d(0));
2636
2637 if (brw->gen < 6)
2638 brw_inst_set_base_mrf(brw, insn, msg_reg_nr);
2639
2640 brw_set_ff_sync_message(p,
2641 insn,
2642 allocate,
2643 response_length,
2644 eot);
2645 }
2646
2647 /**
2648 * Emit the SEND instruction necessary to generate stream output data on Gen6
2649 * (for transform feedback).
2650 *
2651 * If send_commit_msg is true, this is the last piece of stream output data
2652 * from this thread, so send the data as a committed write. According to the
2653 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2654 *
2655 * "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2656 * writes are complete by sending the final write as a committed write."
2657 */
2658 void
2659 brw_svb_write(struct brw_compile *p,
2660 struct brw_reg dest,
2661 unsigned msg_reg_nr,
2662 struct brw_reg src0,
2663 unsigned binding_table_index,
2664 bool send_commit_msg)
2665 {
2666 brw_inst *insn;
2667
2668 gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2669
2670 insn = next_insn(p, BRW_OPCODE_SEND);
2671 brw_set_dest(p, insn, dest);
2672 brw_set_src0(p, insn, src0);
2673 brw_set_src1(p, insn, brw_imm_d(0));
2674 brw_set_dp_write_message(p, insn,
2675 binding_table_index,
2676 0, /* msg_control: ignored */
2677 GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE,
2678 1, /* msg_length */
2679 true, /* header_present */
2680 0, /* last_render_target: ignored */
2681 send_commit_msg, /* response_length */
2682 0, /* end_of_thread */
2683 send_commit_msg); /* send_commit_msg */
2684 }
2685
2686 static void
2687 brw_set_dp_untyped_atomic_message(struct brw_compile *p,
2688 brw_inst *insn,
2689 unsigned atomic_op,
2690 unsigned bind_table_index,
2691 unsigned msg_length,
2692 unsigned response_length,
2693 bool header_present)
2694 {
2695 const struct brw_context *brw = p->brw;
2696
2697 unsigned msg_control =
2698 atomic_op | /* Atomic Operation Type: BRW_AOP_* */
2699 (response_length ? 1 << 5 : 0); /* Return data expected */
2700
2701 if (brw->gen >= 8 || brw->is_haswell) {
2702 brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1,
2703 msg_length, response_length,
2704 header_present, false);
2705
2706
2707 if (brw_inst_access_mode(brw, insn) == BRW_ALIGN_1) {
2708 if (brw_inst_exec_size(brw, insn) != BRW_EXECUTE_16)
2709 msg_control |= 1 << 4; /* SIMD8 mode */
2710
2711 brw_inst_set_dp_msg_type(brw, insn,
2712 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP);
2713 } else {
2714 brw_inst_set_dp_msg_type(brw, insn,
2715 HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2);
2716 }
2717 } else {
2718 brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
2719 msg_length, response_length,
2720 header_present, false);
2721
2722 brw_inst_set_dp_msg_type(brw, insn, GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP);
2723
2724 if (brw_inst_exec_size(brw, insn) != BRW_EXECUTE_16)
2725 msg_control |= 1 << 4; /* SIMD8 mode */
2726 }
2727
2728 brw_inst_set_binding_table_index(brw, insn, bind_table_index);
2729 brw_inst_set_dp_msg_control(brw, insn, msg_control);
2730 }
2731
2732 void
2733 brw_untyped_atomic(struct brw_compile *p,
2734 struct brw_reg dest,
2735 struct brw_reg payload,
2736 unsigned atomic_op,
2737 unsigned bind_table_index,
2738 unsigned msg_length,
2739 unsigned response_length) {
2740 const struct brw_context *brw = p->brw;
2741 brw_inst *insn = brw_next_insn(p, BRW_OPCODE_SEND);
2742
2743 brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD));
2744 brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UD));
2745 brw_set_src1(p, insn, brw_imm_d(0));
2746 brw_set_dp_untyped_atomic_message(
2747 p, insn, atomic_op, bind_table_index, msg_length, response_length,
2748 brw_inst_access_mode(brw, insn) == BRW_ALIGN_1);
2749 }
2750
2751 static void
2752 brw_set_dp_untyped_surface_read_message(struct brw_compile *p,
2753 brw_inst *insn,
2754 unsigned bind_table_index,
2755 unsigned msg_length,
2756 unsigned response_length,
2757 bool header_present)
2758 {
2759 const struct brw_context *brw = p->brw;
2760 const unsigned dispatch_width =
2761 (brw_inst_exec_size(brw, insn) == BRW_EXECUTE_16 ? 16 : 8);
2762 const unsigned num_channels = response_length / (dispatch_width / 8);
2763
2764 if (brw->gen >= 8 || brw->is_haswell) {
2765 brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1,
2766 msg_length, response_length,
2767 header_present, false);
2768
2769 brw_inst_set_dp_msg_type(brw, insn,
2770 HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ);
2771 } else {
2772 brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
2773 msg_length, response_length,
2774 header_present, false);
2775
2776 brw_inst_set_dp_msg_type(brw, insn,
2777 GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ);
2778 }
2779
2780 /* Set mask of 32-bit channels to drop. */
2781 unsigned msg_control = (0xf & (0xf << num_channels));
2782
2783 if (brw_inst_access_mode(brw, insn) == BRW_ALIGN_1) {
2784 if (dispatch_width == 16)
2785 msg_control |= 1 << 4; /* SIMD16 mode */
2786 else
2787 msg_control |= 2 << 4; /* SIMD8 mode */
2788 }
2789
2790 brw_inst_set_binding_table_index(brw, insn, bind_table_index);
2791 brw_inst_set_dp_msg_control(brw, insn, msg_control);
2792 }
2793
2794 void
2795 brw_untyped_surface_read(struct brw_compile *p,
2796 struct brw_reg dest,
2797 struct brw_reg mrf,
2798 unsigned bind_table_index,
2799 unsigned msg_length,
2800 unsigned response_length)
2801 {
2802 const struct brw_context *brw = p->brw;
2803 brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2804
2805 brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD));
2806 brw_set_src0(p, insn, retype(mrf, BRW_REGISTER_TYPE_UD));
2807 brw_set_dp_untyped_surface_read_message(
2808 p, insn, bind_table_index, msg_length, response_length,
2809 brw_inst_access_mode(brw, insn) == BRW_ALIGN_1);
2810 }
2811
2812 void
2813 brw_pixel_interpolator_query(struct brw_compile *p,
2814 struct brw_reg dest,
2815 struct brw_reg mrf,
2816 bool noperspective,
2817 unsigned mode,
2818 unsigned data,
2819 unsigned msg_length,
2820 unsigned response_length)
2821 {
2822 const struct brw_context *brw = p->brw;
2823 struct brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2824
2825 brw_set_dest(p, insn, dest);
2826 brw_set_src0(p, insn, mrf);
2827 brw_set_message_descriptor(p, insn, GEN7_SFID_PIXEL_INTERPOLATOR,
2828 msg_length, response_length,
2829 false /* header is never present for PI */,
2830 false);
2831
2832 brw_inst_set_pi_simd_mode(
2833 brw, insn, brw_inst_exec_size(brw, insn) == BRW_EXECUTE_16);
2834 brw_inst_set_pi_slot_group(brw, insn, 0); /* zero unless 32/64px dispatch */
2835 brw_inst_set_pi_nopersp(brw, insn, noperspective);
2836 brw_inst_set_pi_message_type(brw, insn, mode);
2837 brw_inst_set_pi_message_data(brw, insn, data);
2838 }
2839
2840 /**
2841 * This instruction is generated as a single-channel align1 instruction by
2842 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
2843 *
2844 * We can't use the typed atomic op in the FS because that has the execution
2845 * mask ANDed with the pixel mask, but we just want to write the one dword for
2846 * all the pixels.
2847 *
2848 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
2849 * one u32. So we use the same untyped atomic write message as the pixel
2850 * shader.
2851 *
2852 * The untyped atomic operation requires a BUFFER surface type with RAW
2853 * format, and is only accessible through the legacy DATA_CACHE dataport
2854 * messages.
2855 */
2856 void brw_shader_time_add(struct brw_compile *p,
2857 struct brw_reg payload,
2858 uint32_t surf_index)
2859 {
2860 assert(p->brw->gen >= 7);
2861
2862 brw_push_insn_state(p);
2863 brw_set_default_access_mode(p, BRW_ALIGN_1);
2864 brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2865 brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
2866 brw_pop_insn_state(p);
2867
2868 /* We use brw_vec1_reg and unmasked because we want to increment the given
2869 * offset only once.
2870 */
2871 brw_set_dest(p, send, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE,
2872 BRW_ARF_NULL, 0));
2873 brw_set_src0(p, send, brw_vec1_reg(payload.file,
2874 payload.nr, 0));
2875 brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, surf_index,
2876 2 /* message length */,
2877 0 /* response length */,
2878 false /* header present */);
2879 }